From d93c9274c875e64a8582f7e8f9062aa7457644d5 Mon Sep 17 00:00:00 2001 From: Sebastien Ros Date: Fri, 4 Jun 2010 16:29:50 -0700 Subject: [PATCH] Store index settings on local file system instead of db, for farm scenarios --HG-- branch : dev --- .../Indexing/DefaultIndexProviderTests.cs | 12 ++++ .../Indexing/Lucene/DefaultIndexProvider.cs | 46 +++++++++++++ .../Indexing/Models/IndexingSettingsRecord.cs | 8 --- .../Indexing/Services/IndexingTaskExecutor.cs | 64 ++++++------------- .../Indexing/Services/IndexingTaskManager.cs | 43 ++----------- src/Orchard.Web/Core/Orchard.Core.csproj | 1 - src/Orchard/Indexing/IIndexProvider.cs | 14 +++- .../Tasks/Indexing/IIndexingTaskManager.cs | 5 -- 8 files changed, 95 insertions(+), 98 deletions(-) delete mode 100644 src/Orchard.Web/Core/Indexing/Models/IndexingSettingsRecord.cs diff --git a/src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs b/src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs index 443f8dad6..35626848d 100644 --- a/src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs +++ b/src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs @@ -181,5 +181,17 @@ namespace Orchard.Tests.Indexing { Assert.That(searchBuilder.Get(2).Id, Is.EqualTo(2)); Assert.That(searchBuilder.Get(3).Id, Is.EqualTo(3)); } + + [Test] + public void ProviderShouldStoreSettings() { + _provider.CreateIndex("default"); + Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(DefaultIndexProvider.DefaultMinDateTime)); + + _provider.SetLastIndexUtc("default", new DateTime(2010, 1, 1, 1, 1, 1, 1)); + Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(new DateTime(2010, 1, 1, 1, 1, 1, 0))); + + _provider.SetLastIndexUtc("default", new DateTime(1901, 1, 1, 1, 1, 1, 1)); + Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(DefaultIndexProvider.DefaultMinDateTime)); + } } } diff --git a/src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs b/src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs index 9a5e01174..c449c75e3 100644 --- a/src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs +++ b/src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs @@ -13,6 +13,7 @@ using Orchard.Indexing; using Directory = Lucene.Net.Store.Directory; using Version = Lucene.Net.Util.Version; using Orchard.Logging; +using System.Xml.Linq; namespace Orchard.Core.Indexing.Lucene { /// @@ -24,6 +25,9 @@ namespace Orchard.Core.Indexing.Lucene { public static readonly Version LuceneVersion = Version.LUCENE_29; private readonly Analyzer _analyzer = new StandardAnalyzer(LuceneVersion); private readonly string _basePath; + public static readonly DateTime DefaultMinDateTime = new DateTime(1980, 1, 1); + public static readonly string Settings = "Settings"; + public static readonly string LastIndexUtc = "LastIndexedUtc"; public ILogger Logger { get; set; } @@ -37,6 +41,10 @@ namespace Orchard.Core.Indexing.Lucene { Logger = NullLogger.Instance; // Ensures the directory exists + EnsureDirectoryExists(); + } + + private void EnsureDirectoryExists() { var directory = new DirectoryInfo(_appDataFolder.MapPath(_basePath)); if(!directory.Exists) { directory.Create(); @@ -72,6 +80,11 @@ namespace Orchard.Core.Indexing.Lucene { public void DeleteIndex(string indexName) { new DirectoryInfo(Path.Combine(_appDataFolder.MapPath(Path.Combine(_basePath, indexName)))) .Delete(true); + + var settingsFileName = GetSettingsFileName(indexName); + if(File.Exists(settingsFileName)) { + File.Delete(settingsFileName); + } } public void Store(string indexName, IIndexDocument indexDocument) { @@ -148,5 +161,38 @@ namespace Orchard.Core.Indexing.Lucene { return new DefaultSearchBuilder(GetDirectory(indexName)); } + private string GetSettingsFileName(string indexName) { + return Path.Combine(_appDataFolder.MapPath(_basePath), indexName + ".settings.xml"); + } + + public DateTime GetLastIndexUtc(string indexName) { + var settingsFileName = GetSettingsFileName(indexName); + + return File.Exists(settingsFileName) + ? DateTime.Parse(XDocument.Load(settingsFileName).Descendants(LastIndexUtc).First().Value) + : DefaultMinDateTime; + } + + public void SetLastIndexUtc(string indexName, DateTime lastIndexUtc) { + if ( lastIndexUtc < DefaultMinDateTime ) { + lastIndexUtc = DefaultMinDateTime; + } + + XDocument doc; + var settingsFileName = GetSettingsFileName(indexName); + if ( !File.Exists(settingsFileName) ) { + EnsureDirectoryExists(); + doc = new XDocument( + new XElement(Settings, + new XElement(LastIndexUtc, lastIndexUtc.ToString("s")))); + } + else { + doc = XDocument.Load(settingsFileName); + doc.Element(Settings).Element(LastIndexUtc).Value = lastIndexUtc.ToString("s"); + } + + doc.Save(settingsFileName); + } + } } diff --git a/src/Orchard.Web/Core/Indexing/Models/IndexingSettingsRecord.cs b/src/Orchard.Web/Core/Indexing/Models/IndexingSettingsRecord.cs deleted file mode 100644 index 28cd06b9d..000000000 --- a/src/Orchard.Web/Core/Indexing/Models/IndexingSettingsRecord.cs +++ /dev/null @@ -1,8 +0,0 @@ -using System; - -namespace Orchard.Core.Indexing.Models { - public class IndexingSettingsRecord { - public virtual int Id { get; set; } - public virtual DateTime? LatestIndexingUtc { get; set; } - } -} \ No newline at end of file diff --git a/src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs b/src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs index 1b4d53035..0fcaff198 100644 --- a/src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs +++ b/src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs @@ -19,7 +19,6 @@ namespace Orchard.Core.Indexing.Services { public class IndexingTaskExecutor : IBackgroundTask { private readonly IClock _clock; private readonly IRepository _repository; - private readonly IRepository _settings; private readonly IEnumerable _handlers; private IIndexProvider _indexProvider; private readonly IIndexManager _indexManager; @@ -29,13 +28,11 @@ namespace Orchard.Core.Indexing.Services { public IndexingTaskExecutor( IClock clock, IRepository repository, - IRepository settings, IEnumerable handlers, IIndexManager indexManager, IContentManager contentManager) { _clock = clock; _repository = repository; - _settings = settings; _indexManager = indexManager; _handlers = handlers; _contentManager = contentManager; @@ -46,64 +43,41 @@ namespace Orchard.Core.Indexing.Services { public void Sweep() { - if ( !_indexManager.HasIndexProvider() ) { + if (!_indexManager.HasIndexProvider()) { return; } _indexProvider = _indexManager.GetSearchIndexProvider(); // retrieve last processed index time - var settingsRecord = _settings.Table.FirstOrDefault(); + var lastIndexing = _indexProvider.GetLastIndexUtc(SearchIndexName); + _indexProvider.SetLastIndexUtc(SearchIndexName, _clock.UtcNow); - if ( settingsRecord == null ) { - _settings.Create(settingsRecord = new IndexingSettingsRecord { LatestIndexingUtc = new DateTime(1980, 1, 1) }); - } - - var lastIndexing = settingsRecord.LatestIndexingUtc; - settingsRecord.LatestIndexingUtc = _clock.UtcNow; - - // retrieved not yet processed tasks + // retrieve not yet processed tasks var taskRecords = _repository.Fetch(x => x.CreatedUtc >= lastIndexing) .ToArray(); - if ( taskRecords.Length == 0 ) + if (taskRecords.Length == 0) return; Logger.Information("Processing {0} indexing tasks", taskRecords.Length); - - if ( !_indexProvider.Exists(SearchIndexName) ) { + if (!_indexProvider.Exists(SearchIndexName)) { _indexProvider.CreateIndex(SearchIndexName); } var updateIndexDocuments = new List(); - var deleteIndexDocuments = new List(); // process Delete tasks - foreach ( var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete) ) { - var task = new IndexingTask(_contentManager, taskRecord); - deleteIndexDocuments.Add(taskRecord.ContentItemRecord.Id); - - try { - _repository.Delete(taskRecord); - } - catch ( Exception ex ) { - Logger.Error(ex, "Could not delete task #{0}", taskRecord.Id); - } - } - - try { - if ( deleteIndexDocuments.Count > 0 ) { - _indexProvider.Delete(SearchIndexName, deleteIndexDocuments); - } + _indexProvider.Delete(SearchIndexName, taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete).Select(t => t.Id)); } - catch ( Exception ex ) { - Logger.Warning(ex, "An error occured while remove a document from the index"); + catch (Exception ex) { + Logger.Warning(ex, "An error occured while removing a document from the index"); } // process Update tasks - foreach ( var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update) ) { + foreach (var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update)) { var task = new IndexingTask(_contentManager, taskRecord); try { @@ -113,31 +87,29 @@ namespace Orchard.Core.Indexing.Services { }; // dispatch to handlers to retrieve index information - foreach ( var handler in _handlers ) { + foreach (var handler in _handlers) { handler.Indexing(context); } updateIndexDocuments.Add(context.IndexDocument); - foreach ( var handler in _handlers ) { + foreach (var handler in _handlers) { handler.Indexed(context); } } - catch ( Exception ex ) { + catch (Exception ex) { Logger.Warning(ex, "Unable to process indexing task #{0}", taskRecord.Id); } } - try { - if ( updateIndexDocuments.Count > 0 ) { + if (updateIndexDocuments.Count > 0) { + try { _indexProvider.Store(SearchIndexName, updateIndexDocuments); } + catch (Exception ex) { + Logger.Warning(ex, "An error occured while adding a document to the index"); + } } - catch ( Exception ex ) { - Logger.Warning(ex, "An error occured while adding a document to the index"); - } - - _settings.Update(settingsRecord); } } } diff --git a/src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs b/src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs index 1dde1d257..30eac6a59 100644 --- a/src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs +++ b/src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs @@ -16,18 +16,15 @@ namespace Orchard.Core.Indexing.Services { public class IndexingTaskManager : IIndexingTaskManager { private readonly IContentManager _contentManager; private readonly IRepository _repository; - private readonly IRepository _settings; private readonly IClock _clock; public IndexingTaskManager( IContentManager contentManager, IRepository repository, - IRepository settings, IClock clock) { _clock = clock; _repository = repository; _contentManager = contentManager; - _settings = settings; Logger = NullLogger.Instance; } @@ -38,14 +35,7 @@ namespace Orchard.Core.Indexing.Services { throw new ArgumentNullException("contentItem"); } - // remove previous tasks for the same content item - var tasks = _repository - .Fetch(x => x.ContentItemRecord.Id == contentItem.Id) - .ToArray(); - - foreach ( var task in tasks ) { - _repository.Delete(task); - } + DeleteTasks(contentItem); var taskRecord = new IndexingTaskRecord { CreatedUtc = _clock.UtcNow, @@ -77,37 +67,16 @@ namespace Orchard.Core.Indexing.Services { .ToReadOnlyCollection(); } - public void DeleteTasks(DateTime? createdBefore) { - Logger.Debug("Deleting Indexing tasks created before {0}", createdBefore); - - var tasks = _repository - .Fetch(x => x.CreatedUtc <= createdBefore); - - foreach (var task in tasks) { - _repository.Delete(task); - } - } - + /// + /// Removes existing tasks for the specified content item + /// public void DeleteTasks(ContentItem contentItem) { - Logger.Debug("Deleting Indexing tasks for ContentItem [{0}:{1}]", contentItem.ContentType, contentItem.Id); - var tasks = _repository - .Fetch(x => x.Id == contentItem.Id); - + .Fetch(x => x.ContentItemRecord.Id == contentItem.Id) + .ToArray(); foreach (var task in tasks) { _repository.Delete(task); } } - - public void RebuildIndex() { - var settingsRecord = _settings.Table.FirstOrDefault(); - if (settingsRecord == null) { - _settings.Create(settingsRecord = new IndexingSettingsRecord() ); - } - - settingsRecord.LatestIndexingUtc = new DateTime(1980, 1, 1); - _settings.Update(settingsRecord); - } - } } diff --git a/src/Orchard.Web/Core/Orchard.Core.csproj b/src/Orchard.Web/Core/Orchard.Core.csproj index b29aa2515..0132a8947 100644 --- a/src/Orchard.Web/Core/Orchard.Core.csproj +++ b/src/Orchard.Web/Core/Orchard.Core.csproj @@ -113,7 +113,6 @@ - diff --git a/src/Orchard/Indexing/IIndexProvider.cs b/src/Orchard/Indexing/IIndexProvider.cs index 4045931e8..63a9380e7 100644 --- a/src/Orchard/Indexing/IIndexProvider.cs +++ b/src/Orchard/Indexing/IIndexProvider.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; namespace Orchard.Indexing { public interface IIndexProvider : IDependency { @@ -48,5 +49,16 @@ namespace Orchard.Indexing { /// /// A search builder instance ISearchBuilder CreateSearchBuilder(string indexName); + + /// + /// Returns the date and time when the index was last processed + /// + DateTime GetLastIndexUtc(string indexName); + + /// + /// Sets the date and time when the index was last processed + /// + void SetLastIndexUtc(string indexName, DateTime lastIndexUtc); + } } \ No newline at end of file diff --git a/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs b/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs index cdb950e0f..c469ed6e9 100644 --- a/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs +++ b/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs @@ -19,11 +19,6 @@ namespace Orchard.Tasks.Indexing { /// IEnumerable GetTasks(DateTime? createdAfter); - /// - /// Deletes all indexing tasks previous to a specific date and time - /// - void DeleteTasks(DateTime? createdBefore); - /// /// Deletes all indexing tasks assigned to a specific content item ///