Store index settings on local file system instead of db, for farm scenarios

--HG--
branch : dev

src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs

@@ -181,5 +181,17 @@ namespace Orchard.Tests.Indexing {
             Assert.That(searchBuilder.Get(2).Id, Is.EqualTo(2));
             Assert.That(searchBuilder.Get(3).Id, Is.EqualTo(3));
         }
+
+        [Test]
+        public void ProviderShouldStoreSettings() {
+            _provider.CreateIndex("default");
+            Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(DefaultIndexProvider.DefaultMinDateTime));
+
+            _provider.SetLastIndexUtc("default", new DateTime(2010, 1, 1, 1, 1, 1, 1));
+            Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(new DateTime(2010, 1, 1, 1, 1, 1, 0)));
+
+            _provider.SetLastIndexUtc("default", new DateTime(1901, 1, 1, 1, 1, 1, 1));
+            Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(DefaultIndexProvider.DefaultMinDateTime));
+        }
     }
 }

src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs

@@ -13,6 +13,7 @@ using Orchard.Indexing;
 using Directory = Lucene.Net.Store.Directory;
 using Version = Lucene.Net.Util.Version;
 using Orchard.Logging;
+using System.Xml.Linq;
 
 namespace Orchard.Core.Indexing.Lucene {
     /// <summary>
@@ -24,6 +25,9 @@ namespace Orchard.Core.Indexing.Lucene {
         public static readonly Version LuceneVersion = Version.LUCENE_29;
         private readonly Analyzer _analyzer = new StandardAnalyzer(LuceneVersion);
         private readonly string _basePath;
+        public static readonly DateTime DefaultMinDateTime = new DateTime(1980, 1, 1);
+        public static readonly string Settings = "Settings";
+        public static readonly string LastIndexUtc = "LastIndexedUtc";
 
         public ILogger Logger { get; set; }
 
@@ -37,6 +41,10 @@ namespace Orchard.Core.Indexing.Lucene {
             Logger = NullLogger.Instance;
 
             // Ensures the directory exists
+            EnsureDirectoryExists();
+        }
+
+        private void EnsureDirectoryExists() {
             var directory = new DirectoryInfo(_appDataFolder.MapPath(_basePath));
             if(!directory.Exists) {
                 directory.Create();
@@ -72,6 +80,11 @@ namespace Orchard.Core.Indexing.Lucene {
         public void DeleteIndex(string indexName) {
             new DirectoryInfo(Path.Combine(_appDataFolder.MapPath(Path.Combine(_basePath, indexName))))
                 .Delete(true);
+
+            var settingsFileName = GetSettingsFileName(indexName);
+            if(File.Exists(settingsFileName)) {
+                File.Delete(settingsFileName);
+            }
         }
 
         public void Store(string indexName, IIndexDocument indexDocument) {
@@ -148,5 +161,38 @@ namespace Orchard.Core.Indexing.Lucene {
             return new DefaultSearchBuilder(GetDirectory(indexName));
         }
 
+        private string GetSettingsFileName(string indexName) {
+            return Path.Combine(_appDataFolder.MapPath(_basePath), indexName + ".settings.xml");
+        }
+
+        public DateTime GetLastIndexUtc(string indexName) {
+            var settingsFileName = GetSettingsFileName(indexName);
+
+            return File.Exists(settingsFileName) 
+                ? DateTime.Parse(XDocument.Load(settingsFileName).Descendants(LastIndexUtc).First().Value)
+                : DefaultMinDateTime;
+        }
+
+        public void SetLastIndexUtc(string indexName, DateTime lastIndexUtc) {
+            if ( lastIndexUtc < DefaultMinDateTime ) {
+                lastIndexUtc = DefaultMinDateTime;
+            }
+
+            XDocument doc;
+            var settingsFileName = GetSettingsFileName(indexName);
+            if ( !File.Exists(settingsFileName) ) {
+                EnsureDirectoryExists();
+                doc = new XDocument(
+                        new XElement(Settings,
+                            new XElement(LastIndexUtc, lastIndexUtc.ToString("s"))));
+            }
+            else {
+                doc = XDocument.Load(settingsFileName);
+                doc.Element(Settings).Element(LastIndexUtc).Value = lastIndexUtc.ToString("s");
+            }
+
+            doc.Save(settingsFileName);
+        }
+
     }
 }

src/Orchard.Web/Core/Indexing/Models/IndexingSettingsRecord.cs

@@ -1,8 +0,0 @@
-using System;
-
-namespace Orchard.Core.Indexing.Models {
-    public class IndexingSettingsRecord {
-        public virtual int Id { get; set; }
-        public virtual DateTime? LatestIndexingUtc { get; set; }
-    }
-}

src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs

@@ -19,7 +19,6 @@ namespace Orchard.Core.Indexing.Services {
     public class IndexingTaskExecutor : IBackgroundTask {
         private readonly IClock _clock;
         private readonly IRepository<IndexingTaskRecord> _repository;
-        private readonly IRepository<IndexingSettingsRecord> _settings;
         private readonly IEnumerable<IContentHandler> _handlers;
         private IIndexProvider _indexProvider;
         private readonly IIndexManager _indexManager;
@@ -29,13 +28,11 @@ namespace Orchard.Core.Indexing.Services {
         public IndexingTaskExecutor(
             IClock clock,
             IRepository<IndexingTaskRecord> repository,
-            IRepository<IndexingSettingsRecord> settings,
             IEnumerable<IContentHandler> handlers,
             IIndexManager indexManager,
             IContentManager contentManager) {
             _clock = clock;
             _repository = repository;
-            _settings = settings;
             _indexManager = indexManager;
             _handlers = handlers;
             _contentManager = contentManager;
@@ -46,64 +43,41 @@ namespace Orchard.Core.Indexing.Services {
 
         public void Sweep() {
 
-            if ( !_indexManager.HasIndexProvider() ) {
+            if (!_indexManager.HasIndexProvider()) {
                 return;
             }
 
             _indexProvider = _indexManager.GetSearchIndexProvider();
 
             // retrieve last processed index time
-            var settingsRecord = _settings.Table.FirstOrDefault();
+            var lastIndexing = _indexProvider.GetLastIndexUtc(SearchIndexName);
+            _indexProvider.SetLastIndexUtc(SearchIndexName, _clock.UtcNow);
 
-            if ( settingsRecord == null ) {
+            // retrieve not yet processed tasks
-                _settings.Create(settingsRecord = new IndexingSettingsRecord { LatestIndexingUtc = new DateTime(1980, 1, 1) });
-            }
-
-            var lastIndexing = settingsRecord.LatestIndexingUtc;
-            settingsRecord.LatestIndexingUtc = _clock.UtcNow;
-
-            // retrieved not yet processed tasks
             var taskRecords = _repository.Fetch(x => x.CreatedUtc >= lastIndexing)
                 .ToArray();
 
-            if ( taskRecords.Length == 0 )
+            if (taskRecords.Length == 0)
                 return;
 
             Logger.Information("Processing {0} indexing tasks", taskRecords.Length);
 
-
+            if (!_indexProvider.Exists(SearchIndexName)) {
-            if ( !_indexProvider.Exists(SearchIndexName) ) {
                 _indexProvider.CreateIndex(SearchIndexName);
             }
 
             var updateIndexDocuments = new List<IIndexDocument>();
-            var deleteIndexDocuments = new List<int>();
 
             // process Delete tasks
-            foreach ( var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete) ) {
-                var task = new IndexingTask(_contentManager, taskRecord);
-                deleteIndexDocuments.Add(taskRecord.ContentItemRecord.Id);
-
-                try {
-                    _repository.Delete(taskRecord);
-                }
-                catch ( Exception ex ) {
-                    Logger.Error(ex, "Could not delete task #{0}", taskRecord.Id);
-                }
-            }
-
-
             try {
-                if ( deleteIndexDocuments.Count > 0 ) {
+                _indexProvider.Delete(SearchIndexName, taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete).Select(t => t.Id));
-                    _indexProvider.Delete(SearchIndexName, deleteIndexDocuments);
-                }
             }
-            catch ( Exception ex ) {
+            catch (Exception ex) {
-                Logger.Warning(ex, "An error occured while remove a document from the index");
+                Logger.Warning(ex, "An error occured while removing a document from the index");
             }
 
             // process Update tasks
-            foreach ( var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update) ) {
+            foreach (var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update)) {
                 var task = new IndexingTask(_contentManager, taskRecord);
 
                 try {
@@ -113,31 +87,29 @@ namespace Orchard.Core.Indexing.Services {
                     };
 
                     // dispatch to handlers to retrieve index information
-                    foreach ( var handler in _handlers ) {
+                    foreach (var handler in _handlers) {
                         handler.Indexing(context);
                     }
 
                     updateIndexDocuments.Add(context.IndexDocument);
 
-                    foreach ( var handler in _handlers ) {
+                    foreach (var handler in _handlers) {
                         handler.Indexed(context);
                     }
                 }
-                catch ( Exception ex ) {
+                catch (Exception ex) {
                     Logger.Warning(ex, "Unable to process indexing task #{0}", taskRecord.Id);
                 }
             }
 
-            try {
+            if (updateIndexDocuments.Count > 0) {
-                if ( updateIndexDocuments.Count > 0 ) {
+                try {
                     _indexProvider.Store(SearchIndexName, updateIndexDocuments);
                 }
+                catch (Exception ex) {
+                    Logger.Warning(ex, "An error occured while adding a document to the index");
+                }
             }
-            catch ( Exception ex ) {
-                Logger.Warning(ex, "An error occured while adding a document to the index");
-            }
-
-            _settings.Update(settingsRecord);
         }
     }
 }

src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs

@@ -16,18 +16,15 @@ namespace Orchard.Core.Indexing.Services {
     public class IndexingTaskManager : IIndexingTaskManager {
         private readonly IContentManager _contentManager;
         private readonly IRepository<IndexingTaskRecord> _repository;
-        private readonly IRepository<IndexingSettingsRecord> _settings;
         private readonly IClock _clock;
 
         public IndexingTaskManager(
             IContentManager contentManager,
             IRepository<IndexingTaskRecord> repository,
-            IRepository<IndexingSettingsRecord> settings,
             IClock clock) {
             _clock = clock;
             _repository = repository;
             _contentManager = contentManager;
-            _settings = settings;
             Logger = NullLogger.Instance;
         }
 
@@ -38,14 +35,7 @@ namespace Orchard.Core.Indexing.Services {
                 throw new ArgumentNullException("contentItem");
             }
 
-            // remove previous tasks for the same content item
+            DeleteTasks(contentItem);
-            var tasks = _repository
-                .Fetch(x => x.ContentItemRecord.Id == contentItem.Id)
-                .ToArray();
-
-            foreach ( var task in tasks ) {
-                _repository.Delete(task);
-            }
 
             var taskRecord = new IndexingTaskRecord {
                 CreatedUtc = _clock.UtcNow,
@@ -77,37 +67,16 @@ namespace Orchard.Core.Indexing.Services {
                 .ToReadOnlyCollection();
         }
 
-        public void DeleteTasks(DateTime? createdBefore) {
+        /// <summary>
-            Logger.Debug("Deleting Indexing tasks created before {0}", createdBefore);
+        /// Removes existing tasks for the specified content item
-
+        /// </summary>
-            var tasks = _repository
-                .Fetch(x => x.CreatedUtc <= createdBefore);
-
-            foreach (var task in tasks) {
-                _repository.Delete(task);
-            }
-        }
-
         public void DeleteTasks(ContentItem contentItem) {
-            Logger.Debug("Deleting Indexing tasks for ContentItem [{0}:{1}]", contentItem.ContentType, contentItem.Id);
-
             var tasks = _repository
-                .Fetch(x => x.Id == contentItem.Id);
+                .Fetch(x => x.ContentItemRecord.Id == contentItem.Id)
-
+                .ToArray();
             foreach (var task in tasks) {
                 _repository.Delete(task);
             }
         }
-
-        public void RebuildIndex() {
-            var settingsRecord = _settings.Table.FirstOrDefault();
-            if (settingsRecord == null) {
-                _settings.Create(settingsRecord = new IndexingSettingsRecord() );
-            }
-            
-            settingsRecord.LatestIndexingUtc = new DateTime(1980, 1, 1);
-            _settings.Update(settingsRecord);
-        }
-
     }
 }

src/Orchard.Web/Core/Orchard.Core.csproj

@@ -113,7 +113,6 @@
     <Compile Include="Indexing\Lucene\DefaultIndexProvider.cs" />
     <Compile Include="Indexing\Lucene\DefaultSearchBuilder.cs" />
     <Compile Include="Indexing\Lucene\DefaultSearchHit.cs" />
-    <Compile Include="Indexing\Models\IndexingSettingsRecord.cs" />
     <Compile Include="Indexing\Models\IndexingTask.cs" />
     <Compile Include="Indexing\Models\IndexingTaskRecord.cs" />
     <Compile Include="Indexing\Services\CreateIndexingTaskHandler.cs" />

src/Orchard/Indexing/IIndexProvider.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 
 namespace Orchard.Indexing {
     public interface IIndexProvider : IDependency {
@@ -48,5 +49,16 @@ namespace Orchard.Indexing {
         /// </summary>
         /// <returns>A search builder instance</returns>
         ISearchBuilder CreateSearchBuilder(string indexName);
+
+        /// <summary>
+        /// Returns the date and time when the index was last processed
+        /// </summary>
+        DateTime GetLastIndexUtc(string indexName);
+
+        /// <summary>
+        /// Sets the date and time when the index was last processed
+        /// </summary>
+        void SetLastIndexUtc(string indexName, DateTime lastIndexUtc);
+
     }
 }

src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs

@@ -19,11 +19,6 @@ namespace Orchard.Tasks.Indexing {
         /// </summary>
         IEnumerable<IIndexingTask> GetTasks(DateTime? createdAfter);
         
-        /// <summary>
-        /// Deletes all indexing tasks previous to a specific date and time
-        /// </summary>
-        void DeleteTasks(DateTime? createdBefore);
-
         /// <summary>
         /// Deletes all indexing tasks assigned to a specific content item
         /// </summary>