From 599d892f4836a87ccb5fe69c84d0ed5630616de5 Mon Sep 17 00:00:00 2001 From: Sebastien Ros Date: Fri, 4 Mar 2011 16:21:19 -0800 Subject: [PATCH] Removing unused methods and refactoring the batch loop --HG-- branch : indexing --- .../Indexing/IndexingTaskExecutorTests.cs | 13 +- .../Orchard.Indexing/Models/IndexSettings.cs | 2 +- .../Services/IndexingTaskExecutor.cs | 256 +++++++++--------- .../Services/IndexingTaskManager.cs | 4 - .../Tasks/Indexing/IIndexingTaskManager.cs | 5 - 5 files changed, 145 insertions(+), 135 deletions(-) diff --git a/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs b/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs index 7e6549a4a..01b77337a 100644 --- a/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs +++ b/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using Autofac; using Lucene.Services; using Moq; @@ -176,6 +175,7 @@ namespace Orchard.Tests.Modules.Indexing { // there should be nothing done _indexNotifier.UpdateIndex(IndexName); + Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; _indexNotifier.UpdateIndex(IndexName); @@ -216,6 +216,17 @@ namespace Orchard.Tests.Modules.Indexing { Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); } + + [Test] + public void ShouldIndexAllContentOverTheLoopSize() { + for (int i = 0; i < 999; i++) { + var content = _contentManager.Create(ThingDriver.ContentTypeName); + content.Text = "Lorem ipsum " + i; + } + _indexNotifier.UpdateIndex(IndexName); + Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(999)); + } + #region Stubs public class ThingHandler : ContentHandler { public ThingHandler() { diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs index c0b3e0e67..c895bc474 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs @@ -50,7 +50,7 @@ namespace Orchard.Indexing.Models new XElement(TagMode, Mode), new XElement(TagLastIndexedId, LastIndexedId), new XElement(TagLastContentId, LastContentId), - new XElement(TagLastIndexedUtc, LastIndexedUtc.ToString("s")) + new XElement(TagLastIndexedUtc, LastIndexedUtc.ToString("u")) )).ToString(); } } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs index 974392f54..14ae41bc4 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using JetBrains.Annotations; using Orchard.ContentManagement; +using Orchard.ContentManagement.Records; using Orchard.Data; using Orchard.Environment.Configuration; using Orchard.FileSystems.AppData; @@ -22,7 +23,8 @@ namespace Orchard.Indexing.Services { /// [UsedImplicitly] public class IndexingTaskExecutor : IIndexNotifierHandler, IIndexStatisticsProvider { - private readonly IRepository _repository; + private readonly IRepository _taskRepository; + private readonly IRepository _contentRepository; private IIndexProvider _indexProvider; private readonly IIndexManager _indexManager; private readonly IContentManager _contentManager; @@ -34,14 +36,16 @@ namespace Orchard.Indexing.Services { private IndexingStatus _indexingStatus = IndexingStatus.Idle; public IndexingTaskExecutor( - IRepository repository, + IRepository taskRepository, + IRepository contentRepository, IIndexManager indexManager, IContentManager contentManager, IAppDataFolder appDataFolder, ShellSettings shellSettings, ILockFileManager lockFileManager, IClock clock) { - _repository = repository; + _taskRepository = taskRepository; + _contentRepository = contentRepository; _indexManager = indexManager; _contentManager = contentManager; _appDataFolder = appDataFolder; @@ -54,6 +58,11 @@ namespace Orchard.Indexing.Services { public ILogger Logger { get; set; } public void UpdateIndex(string indexName) { + // What to do here to run next batch in a separate transaction + while (UpdateIndexBatch(indexName)) {} + } + + public bool UpdateIndexBatch(string indexName) { ILockFile lockFile = null; var settingsFilename = GetSettingsFileName(indexName); var lockFilename = settingsFilename + ".lock"; @@ -61,13 +70,12 @@ namespace Orchard.Indexing.Services { // acquire a lock file on the index if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile)) { Logger.Information("Index was requested but is already running"); - return; + return false; } - using (lockFile) - { + using (lockFile) { if (!_indexManager.HasIndexProvider()) { - return; + return false; } // load index settings to know what is the current state of indexing @@ -79,131 +87,132 @@ namespace Orchard.Indexing.Services { if (!_indexProvider.Exists(indexName)) { _indexProvider.CreateIndex(indexName); indexSettings = new IndexSettings(); + + // mark the last available task at the moment the process is started. + // once the Rebuild is done, Update will start at this point of the table + indexSettings.LastIndexedId = _taskRepository + .Table + .OrderByDescending(x => x.Id) + .Select(x => x.Id) + .FirstOrDefault(); } // execute indexing commands by batch of [ContentItemsPerLoop] content items - for (; ; ){ - var addToIndex = new List(); - var deleteFromIndex = new List(); + return BatchIndex(indexName, settingsFilename, indexSettings); + } + } - // Rebuilding the index ? - if (indexSettings.Mode == IndexingMode.Rebuild) { - Logger.Information("Rebuilding index"); - _indexingStatus = IndexingStatus.Rebuilding; + /// + /// Indexes a batch of content items + /// + /// + /// true if there are more items to process; otherwise, false. + /// + private bool BatchIndex(string indexName, string settingsFilename, IndexSettings indexSettings) { + var addToIndex = new List(); + var deleteFromIndex = new List(); - // store the last inserted task - var lastIndexId = _repository - .Fetch(x => true) - .OrderByDescending(x => x.Id) - .Select(x => x.Id) - .FirstOrDefault(); + // Rebuilding the index ? + if (indexSettings.Mode == IndexingMode.Rebuild) { + Logger.Information("Rebuilding index"); + _indexingStatus = IndexingStatus.Rebuilding; - // load all content items - var contentItemIds = _contentManager - .Query(VersionOptions.Published) - .List() - .Where(x => x.Id > indexSettings.LastContentId) - .OrderBy(x => x.Id) - .Select(x => x.Id) - .Distinct() - .Take(ContentItemsPerLoop) - .ToArray(); + // load all content items + var contentItems = _contentRepository + .Fetch( + versionRecord => versionRecord.Published && versionRecord.ContentItemRecord.Id > indexSettings.LastContentId, + order => order.Asc(versionRecord => versionRecord.Id)) + .Take(ContentItemsPerLoop) + .Select(versionRecord => _contentManager.Get(versionRecord.ContentItemRecord.Id, VersionOptions.VersionRecord(versionRecord.Id))) + .Distinct() + .ToList(); - indexSettings.LastIndexedId = lastIndexId; + // if no more elements to index, switch to update mode + if (contentItems.Count == 0) { + indexSettings.Mode = IndexingMode.Update; + } - // if no more elements to index, switch to update mode - if (contentItemIds.Length == 0) { - indexSettings.Mode = IndexingMode.Update; - } - - foreach (var id in contentItemIds) { - try { - IDocumentIndex documentIndex = ExtractDocumentIndex(id); - - if (documentIndex != null && documentIndex.IsDirty) { - addToIndex.Add(documentIndex); - } - - // store the last processed element - indexSettings.LastContentId = contentItemIds.LastOrDefault(); - } - catch (Exception ex) { - Logger.Warning(ex, "Unable to index content item #{0} during rebuild", id); - } - } - } - - if (indexSettings.Mode == IndexingMode.Update) { - Logger.Information("Updating index"); - _indexingStatus = IndexingStatus.Updating; - - // load next content items to index, by filtering and ordering on the task id - var lastIndexId = _repository - .Fetch(x => x.Id > indexSettings.LastIndexedId) - .OrderByDescending(x => x.Id) - .Select(x => x.Id) - .FirstOrDefault(); - - var contentItemIds = _repository - .Fetch(x => x.Id > indexSettings.LastIndexedId) - .OrderBy(x => x.Id) - .Take(ContentItemsPerLoop) - .Select(x => x.ContentItemRecord.Id) - .Distinct() // don't process the same content item twice - .ToArray(); - - indexSettings.LastIndexedId = lastIndexId; - - foreach (var id in contentItemIds) { - try { - IDocumentIndex documentIndex = ExtractDocumentIndex(id); - - if (documentIndex == null) { - deleteFromIndex.Add(id); - } - else if (documentIndex.IsDirty) { - addToIndex.Add(documentIndex); - } - } - catch (Exception ex) { - Logger.Warning(ex, "Unable to index content item #{0} during rebuild", id); - } - } - } - - // save current state of the index - indexSettings.LastIndexedUtc = _clock.UtcNow; - _appDataFolder.CreateFile(settingsFilename, indexSettings.ToString()); - - if (deleteFromIndex.Count == 0 && addToIndex.Count == 0) { - // nothing more to do - _indexingStatus = IndexingStatus.Idle; - return; - } - - // save new and updated documents to the index + foreach (var item in contentItems) { try { - if (addToIndex.Count > 0) { - _indexProvider.Store(indexName, addToIndex); - Logger.Information("Added content items to index: {0}", addToIndex.Count); + IDocumentIndex documentIndex = ExtractDocumentIndex(item); + + if (documentIndex != null && documentIndex.IsDirty) { + addToIndex.Add(documentIndex); } + + indexSettings.LastContentId = item.VersionRecord.Id; } catch (Exception ex) { - Logger.Warning(ex, "An error occured while adding a document to the index"); - } - - // removing documents from the index - try { - if (deleteFromIndex.Count > 0) { - _indexProvider.Delete(indexName, deleteFromIndex); - Logger.Information("Added content items to index: {0}", addToIndex.Count); - } - } - catch (Exception ex) { - Logger.Warning(ex, "An error occured while removing a document from the index"); + Logger.Warning(ex, "Unable to index content item #{0} during rebuild", item.Id); } } } + + if (indexSettings.Mode == IndexingMode.Update) { + Logger.Information("Updating index"); + _indexingStatus = IndexingStatus.Updating; + + var contentItems = _taskRepository + .Fetch(x => x.Id > indexSettings.LastIndexedId) + .OrderBy(x => x.Id) + .Take(ContentItemsPerLoop) + .GroupBy(x => x.ContentItemRecord.Id) + .Select(group => new {TaskId = group.Max(task => task.Id), ContentItem = _contentManager.Get(group.Key, VersionOptions.Published)}) + .OrderBy(x => x.TaskId) + .ToArray(); + + foreach (var item in contentItems) { + try { + IDocumentIndex documentIndex = ExtractDocumentIndex(item.ContentItem); + + if (documentIndex == null) { + deleteFromIndex.Add(item.ContentItem.Id); + } + else if (documentIndex.IsDirty) { + addToIndex.Add(documentIndex); + } + + indexSettings.LastIndexedId = item.TaskId; + } + catch (Exception ex) { + Logger.Warning(ex, "Unable to index content item #{0} during update", item.ContentItem.Id); + } + } + } + + // save current state of the index + indexSettings.LastIndexedUtc = _clock.UtcNow; + _appDataFolder.CreateFile(settingsFilename, indexSettings.ToString()); + + if (deleteFromIndex.Count == 0 && addToIndex.Count == 0) { + // nothing more to do + _indexingStatus = IndexingStatus.Idle; + return false; + } + + // save new and updated documents to the index + try { + if (addToIndex.Count > 0) { + _indexProvider.Store(indexName, addToIndex); + Logger.Information("Added content items to index: {0}", addToIndex.Count); + } + } + catch (Exception ex) { + Logger.Warning(ex, "An error occured while adding a document to the index"); + } + + // removing documents from the index + try { + if (deleteFromIndex.Count > 0) { + _indexProvider.Delete(indexName, deleteFromIndex); + Logger.Information("Added content items to index: {0}", addToIndex.Count); + } + } + catch (Exception ex) { + Logger.Warning(ex, "An error occured while removing a document from the index"); + } + + return true; } /// @@ -224,18 +233,17 @@ namespace Orchard.Indexing.Services { /// Creates a IDocumentIndex instance for a specific content item id. If the content /// item is no more published, it returns null. /// - private IDocumentIndex ExtractDocumentIndex(int id) { - var contentItem = _contentManager.Get(id, VersionOptions.Published); - + private IDocumentIndex ExtractDocumentIndex(ContentItem contentItem) { // ignore deleted or unpublished items - if(contentItem == null || !contentItem.IsPublished()) { + if (contentItem == null || !contentItem.IsPublished()) { return null; } // skip items from types which are not indexed var settings = GetTypeIndexingSettings(contentItem); - if (!settings.Included) + if (!settings.Included) { return null; + } var documentIndex = _indexProvider.New(contentItem.Id); @@ -244,11 +252,11 @@ namespace Orchard.Indexing.Services { return documentIndex; } - static TypeIndexing GetTypeIndexingSettings(ContentItem contentItem) { + private static TypeIndexing GetTypeIndexingSettings(ContentItem contentItem) { if (contentItem == null || contentItem.TypeDefinition == null || contentItem.TypeDefinition.Settings == null) { - return new TypeIndexing { Included = false }; + return new TypeIndexing {Included = false}; } return contentItem.TypeDefinition.Settings.GetModel(); } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs index 993f962e9..2726b0715 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs @@ -51,9 +51,5 @@ namespace Orchard.Indexing.Services { CreateTask(contentItem, IndexingTaskRecord.Delete); Logger.Information("Deleting index task created for [{0}:{1}]", contentItem.ContentType, contentItem.Id); } - - public DateTime GetLastTaskDateTime() { - return _repository.Table.Max(t => t.CreatedUtc) ?? new DateTime(1980, 1, 1); - } } } diff --git a/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs b/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs index 85b6b02f2..aab78c650 100644 --- a/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs +++ b/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs @@ -12,10 +12,5 @@ namespace Orchard.Tasks.Indexing { /// Adds a new entry in the index task table in order to delete an existing index for the specified content item. /// void CreateDeleteIndexTask(ContentItem contentItem); - - /// - /// Returns the Date Time of the last task created - /// - DateTime GetLastTaskDateTime(); } } \ No newline at end of file