From 9eebb483ae8957b0a3fcb0538b30fa373ddebc6b Mon Sep 17 00:00:00 2001 From: Sebastien Ros Date: Fri, 4 Jun 2010 17:42:20 -0700 Subject: [PATCH] Implemented RebuildIndex plus UpdateIndex signal to backgroung tasks. Adapted UI. --HG-- branch : dev --- .../Indexing/DefaultIndexProviderTests.cs | 25 ++- .../Indexing/Lucene/DefaultIndexProvider.cs | 16 +- .../Indexing/Services/IndexingTaskExecutor.cs | 165 ++++++++++++------ .../Indexing/Services/IndexingTaskManager.cs | 8 +- .../Controllers/AdminController.cs | 5 +- .../Orchard.Search/Services/ISearchService.cs | 4 +- .../Orchard.Search/Services/SearchService.cs | 25 ++- .../Orchard.Search/Views/Admin/Index.ascx | 2 +- src/Orchard/Indexing/IIndexNotifierHandler.cs | 5 + src/Orchard/Indexing/IIndexProvider.cs | 5 + src/Orchard/Orchard.Framework.csproj | 1 + .../Tasks/Indexing/IIndexingTaskManager.cs | 6 +- 12 files changed, 193 insertions(+), 74 deletions(-) create mode 100644 src/Orchard/Indexing/IIndexNotifierHandler.cs diff --git a/src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs b/src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs index 35626848d..f92333d91 100644 --- a/src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs +++ b/src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs @@ -58,7 +58,11 @@ namespace Orchard.Tests.Indexing { [Test] public void IndexProviderShouldOverwriteAlreadyExistingIndex() { _provider.CreateIndex("default"); - _provider.CreateIndex("default"); + _provider.Store("default", _provider.New(1).Add("body", null)); + Assert.That(_provider.IsEmpty("default"), Is.False); + + _provider.CreateIndex("default"); + Assert.That(_provider.IsEmpty("default"), Is.True); } [Test] @@ -193,5 +197,24 @@ namespace Orchard.Tests.Indexing { _provider.SetLastIndexUtc("default", new DateTime(1901, 1, 1, 1, 1, 1, 1)); Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(DefaultIndexProvider.DefaultMinDateTime)); } + + [Test] + public void IsEmptyShouldBeTrueForNoneExistingIndexes() { + _provider.IsEmpty("dummy"); + Assert.That(_provider.IsEmpty("default"), Is.True); + } + + [Test] + public void IsEmptyShouldBeTrueForJustNewIndexes() { + _provider.CreateIndex("default"); + Assert.That(_provider.IsEmpty("default"), Is.True); + } + + [Test] + public void IsEmptyShouldBeFalseWhenThereIsADocument() { + _provider.CreateIndex("default"); + _provider.Store("default", _provider.New(1).Add("body", null)); + Assert.That(_provider.IsEmpty("default"), Is.False); + } } } diff --git a/src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs b/src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs index c449c75e3..f283aed45 100644 --- a/src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs +++ b/src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs @@ -70,6 +70,21 @@ namespace Orchard.Core.Indexing.Lucene { return new DirectoryInfo(_appDataFolder.MapPath(Path.Combine(_basePath, indexName))).Exists; } + public bool IsEmpty(string indexName) { + if(!Exists(indexName)) { + return true; + } + + var reader = IndexReader.Open(GetDirectory(indexName), true); + + try { + return reader.NumDocs() == 0; + } + finally { + reader.Close(); + } + } + public void CreateIndex(string indexName) { var writer = new IndexWriter(GetDirectory(indexName), _analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED); writer.Close(); @@ -118,7 +133,6 @@ namespace Orchard.Core.Indexing.Lucene { writer.Optimize(); writer.Close(); } - } public void Delete(string indexName, int documentId) { diff --git a/src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs b/src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs index 0fcaff198..baa9b1b8d 100644 --- a/src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs +++ b/src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs @@ -10,105 +10,164 @@ using Orchard.Logging; using Orchard.Services; using Orchard.Tasks; using Orchard.Core.Indexing.Models; +using Orchard.Tasks.Indexing; +using Orchard.Indexing; namespace Orchard.Core.Indexing.Services { /// /// Contains the logic which is regularly executed to retrieve index information from multiple content handlers. /// [UsedImplicitly] - public class IndexingTaskExecutor : IBackgroundTask { + public class IndexingTaskExecutor : IBackgroundTask, IIndexNotifierHandler { private readonly IClock _clock; private readonly IRepository _repository; private readonly IEnumerable _handlers; private IIndexProvider _indexProvider; private readonly IIndexManager _indexManager; + private readonly IIndexingTaskManager _indexingTaskManager; private readonly IContentManager _contentManager; private const string SearchIndexName = "Search"; + + private readonly object _synLock = new object(); public IndexingTaskExecutor( IClock clock, IRepository repository, IEnumerable handlers, IIndexManager indexManager, + IIndexingTaskManager indexingTaskManager, IContentManager contentManager) { _clock = clock; _repository = repository; _indexManager = indexManager; _handlers = handlers; + _indexingTaskManager = indexingTaskManager; _contentManager = contentManager; Logger = NullLogger.Instance; } public ILogger Logger { get; set; } + public void UpdateIndex(string indexName) { + if (indexName == SearchIndexName) { + Sweep(); + } + } + public void Sweep() { - if (!_indexManager.HasIndexProvider()) { + if ( !System.Threading.Monitor.TryEnter(_synLock) ) { + Logger.Information("Index was requested but was already running"); return; } - _indexProvider = _indexManager.GetSearchIndexProvider(); - - // retrieve last processed index time - var lastIndexing = _indexProvider.GetLastIndexUtc(SearchIndexName); - _indexProvider.SetLastIndexUtc(SearchIndexName, _clock.UtcNow); - - // retrieve not yet processed tasks - var taskRecords = _repository.Fetch(x => x.CreatedUtc >= lastIndexing) - .ToArray(); - - if (taskRecords.Length == 0) - return; - - Logger.Information("Processing {0} indexing tasks", taskRecords.Length); - - if (!_indexProvider.Exists(SearchIndexName)) { - _indexProvider.CreateIndex(SearchIndexName); - } - - var updateIndexDocuments = new List(); - - // process Delete tasks try { - _indexProvider.Delete(SearchIndexName, taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete).Select(t => t.Id)); - } - catch (Exception ex) { - Logger.Warning(ex, "An error occured while removing a document from the index"); - } - // process Update tasks - foreach (var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update)) { - var task = new IndexingTask(_contentManager, taskRecord); + if (!_indexManager.HasIndexProvider()) { + return; + } - try { - var context = new IndexContentContext { - ContentItem = task.ContentItem, - IndexDocument = _indexProvider.New(task.ContentItem.Id) - }; + _indexProvider = _indexManager.GetSearchIndexProvider(); + var updateIndexDocuments = new List(); + var lastIndexing = DateTime.UtcNow; - // dispatch to handlers to retrieve index information - foreach (var handler in _handlers) { - handler.Indexing(context); + // Do we need to rebuild the full index (first time module is used, or rebuild index requested) ? + if (_indexProvider.IsEmpty(SearchIndexName)) { + Logger.Information("Rebuild index started"); + + // mark current last task, as we should process older ones (in case of rebuild index only) + lastIndexing = _indexingTaskManager.GetLastTaskDateTime(); + + // get every existing content item to index it + foreach (var contentItem in _contentManager.Query(VersionOptions.Published).List()) { + try { + var context = new IndexContentContext { + ContentItem = contentItem, + IndexDocument = _indexProvider.New(contentItem.Id) + }; + + // dispatch to handlers to retrieve index information + foreach (var handler in _handlers) { + handler.Indexing(context); + } + + updateIndexDocuments.Add(context.IndexDocument); + + foreach (var handler in _handlers) { + handler.Indexed(context); + } + } + catch (Exception ex) { + Logger.Warning(ex, "Unable to index content item #{0} during rebuild", contentItem.Id); + } } - updateIndexDocuments.Add(context.IndexDocument); + } + else { + // retrieve last processed index time + lastIndexing = _indexProvider.GetLastIndexUtc(SearchIndexName); + } - foreach (var handler in _handlers) { - handler.Indexed(context); + _indexProvider.SetLastIndexUtc(SearchIndexName, _clock.UtcNow); + + // retrieve not yet processed tasks + var taskRecords = _repository.Fetch(x => x.CreatedUtc >= lastIndexing) + .ToArray(); + + if (taskRecords.Length == 0) + return; + + Logger.Information("Processing {0} indexing tasks", taskRecords.Length); + + if (!_indexProvider.Exists(SearchIndexName)) { + _indexProvider.CreateIndex(SearchIndexName); + } + + // process Delete tasks + try { + _indexProvider.Delete(SearchIndexName, taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete).Select(t => t.Id)); + } + catch (Exception ex) { + Logger.Warning(ex, "An error occured while removing a document from the index"); + } + + // process Update tasks + foreach (var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update)) { + var task = new IndexingTask(_contentManager, taskRecord); + + try { + var context = new IndexContentContext { + ContentItem = task.ContentItem, + IndexDocument = _indexProvider.New(task.ContentItem.Id) + }; + + // dispatch to handlers to retrieve index information + foreach (var handler in _handlers) { + handler.Indexing(context); + } + + updateIndexDocuments.Add(context.IndexDocument); + + foreach (var handler in _handlers) { + handler.Indexed(context); + } + } + catch (Exception ex) { + Logger.Warning(ex, "Unable to process indexing task #{0}", taskRecord.Id); } } - catch (Exception ex) { - Logger.Warning(ex, "Unable to process indexing task #{0}", taskRecord.Id); + + if (updateIndexDocuments.Count > 0) { + try { + _indexProvider.Store(SearchIndexName, updateIndexDocuments); + } + catch (Exception ex) { + Logger.Warning(ex, "An error occured while adding a document to the index"); + } } } - - if (updateIndexDocuments.Count > 0) { - try { - _indexProvider.Store(SearchIndexName, updateIndexDocuments); - } - catch (Exception ex) { - Logger.Warning(ex, "An error occured while adding a document to the index"); - } + finally { + System.Threading.Monitor.Exit(_synLock); } } } diff --git a/src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs b/src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs index 30eac6a59..9969920c3 100644 --- a/src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs +++ b/src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs @@ -59,12 +59,8 @@ namespace Orchard.Core.Indexing.Services { Logger.Information("Deleting index task created for [{0}:{1}]", contentItem.ContentType, contentItem.Id); } - public IEnumerable GetTasks(DateTime? createdAfter) { - return _repository - .Fetch(x => x.CreatedUtc > createdAfter) - .Select(x => new IndexingTask(_contentManager, x)) - .Cast() - .ToReadOnlyCollection(); + public DateTime GetLastTaskDateTime() { + return _repository.Table.Max(t => t.CreatedUtc) ?? DateTime.MinValue; } /// diff --git a/src/Orchard.Web/Modules/Orchard.Search/Controllers/AdminController.cs b/src/Orchard.Web/Modules/Orchard.Search/Controllers/AdminController.cs index 852509b81..217e1db53 100644 --- a/src/Orchard.Web/Modules/Orchard.Search/Controllers/AdminController.cs +++ b/src/Orchard.Web/Modules/Orchard.Search/Controllers/AdminController.cs @@ -1,4 +1,5 @@ -using System.Web.Mvc; +using System; +using System.Web.Mvc; using Orchard.Localization; using Orchard.Search.Services; using Orchard.Search.ViewModels; @@ -18,7 +19,7 @@ namespace Orchard.Search.Controllers { public Localizer T { get; set; } public ActionResult Index() { - var viewModel = new SearchIndexViewModel {HasIndexToManage = _searchService.HasIndexToManage}; + var viewModel = new SearchIndexViewModel {HasIndexToManage = _searchService.HasIndexToManage, IndexUpdatedUtc = _searchService.GetIndexUpdatedUtc()}; if (!viewModel.HasIndexToManage) Services.Notifier.Information(T("There is not search index to manage for this site.")); diff --git a/src/Orchard.Web/Modules/Orchard.Search/Services/ISearchService.cs b/src/Orchard.Web/Modules/Orchard.Search/Services/ISearchService.cs index aba8ecdb0..5f23fb778 100644 --- a/src/Orchard.Web/Modules/Orchard.Search/Services/ISearchService.cs +++ b/src/Orchard.Web/Modules/Orchard.Search/Services/ISearchService.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using Orchard.Indexing; namespace Orchard.Search.Services { @@ -7,5 +8,6 @@ namespace Orchard.Search.Services { IEnumerable Query(string term); void RebuildIndex(); void UpdateIndex(); + DateTime GetIndexUpdatedUtc(); } } \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Search/Services/SearchService.cs b/src/Orchard.Web/Modules/Orchard.Search/Services/SearchService.cs index 257be8946..52454fb72 100644 --- a/src/Orchard.Web/Modules/Orchard.Search/Services/SearchService.cs +++ b/src/Orchard.Web/Modules/Orchard.Search/Services/SearchService.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using Orchard.Indexing; using Orchard.Localization; @@ -8,12 +9,14 @@ namespace Orchard.Search.Services { public class SearchService : ISearchService { - private const string SearchIndexName = "search"; + private const string SearchIndexName = "Search"; private readonly IIndexManager _indexManager; + private readonly IEnumerable _indexNotifierHandlers; - public SearchService(IOrchardServices services, IIndexManager indexManager) { + public SearchService(IOrchardServices services, IIndexManager indexManager, IEnumerable indexNotifierHandlers) { Services = services; _indexManager = indexManager; + _indexNotifierHandlers = indexNotifierHandlers; T = NullLocalizer.Instance; } @@ -49,10 +52,20 @@ namespace Orchard.Search.Services } public void UpdateIndex() { - //todo: this - //if (_indexManager.HasIndexProvider()) - // _indexManager.GetSearchIndexProvider().UpdateIndex(SearchIndexName); + + foreach(var handler in _indexNotifierHandlers) { + handler.UpdateIndex(SearchIndexName); + } + Services.Notifier.Information(T("The search index has been updated.")); } + + public DateTime GetIndexUpdatedUtc() { + if(!HasIndexToManage) { + return DateTime.MinValue; + } + + return _indexManager.GetSearchIndexProvider().GetLastIndexUtc(SearchIndexName); + } } } \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Search/Views/Admin/Index.ascx b/src/Orchard.Web/Modules/Orchard.Search/Views/Admin/Index.ascx index 7726c2ef2..f68e7e59b 100644 --- a/src/Orchard.Web/Modules/Orchard.Search/Views/Admin/Index.ascx +++ b/src/Orchard.Web/Modules/Orchard.Search/Views/Admin/Index.ascx @@ -10,7 +10,7 @@ using (Html.BeginForm("update", "admin", FormMethod.Post, new {area = "Orchard.S } using (Html.BeginForm("rebuild", "admin", FormMethod.Post, new {area = "Orchard.Search"})) { %>
-

<%=T("Rebuild the search index for a fresh start. ") %>

+

<%=T("Rebuild the search index for a fresh start. ") %>

<%=Html.AntiForgeryTokenOrchard() %>
<% } %> \ No newline at end of file diff --git a/src/Orchard/Indexing/IIndexNotifierHandler.cs b/src/Orchard/Indexing/IIndexNotifierHandler.cs new file mode 100644 index 000000000..f115544d9 --- /dev/null +++ b/src/Orchard/Indexing/IIndexNotifierHandler.cs @@ -0,0 +1,5 @@ +namespace Orchard.Indexing { + public interface IIndexNotifierHandler : IEvents { + void UpdateIndex(string indexName); + } +} diff --git a/src/Orchard/Indexing/IIndexProvider.cs b/src/Orchard/Indexing/IIndexProvider.cs index 63a9380e7..121089bf7 100644 --- a/src/Orchard/Indexing/IIndexProvider.cs +++ b/src/Orchard/Indexing/IIndexProvider.cs @@ -18,6 +18,11 @@ namespace Orchard.Indexing { /// void DeleteIndex(string name); + /// + /// Whether an index is empty or not + /// + bool IsEmpty(string indexName); + /// /// Creates an empty document /// diff --git a/src/Orchard/Orchard.Framework.csproj b/src/Orchard/Orchard.Framework.csproj index 80aea76e8..29b723a66 100644 --- a/src/Orchard/Orchard.Framework.csproj +++ b/src/Orchard/Orchard.Framework.csproj @@ -349,6 +349,7 @@ + diff --git a/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs b/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs index c469ed6e9..1bd06d54c 100644 --- a/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs +++ b/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs @@ -15,10 +15,10 @@ namespace Orchard.Tasks.Indexing { void CreateDeleteIndexTask(ContentItem contentItem); /// - /// Loads all indexing tasks created after to a specific date and time + /// Returns the Date Time of the last task created /// - IEnumerable GetTasks(DateTime? createdAfter); - + DateTime GetLastTaskDateTime(); + /// /// Deletes all indexing tasks assigned to a specific content item ///