Using IProcessEngine to schedule batched indexing; Modifying

IIndexingTaskExecutor to handle index deletiong in order to prevent
concurrency issues.

--HG--
branch : indexing

src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexingTaskExecutor.cs

@@ -0,0 +1,6 @@
+namespace Orchard.Indexing.Services {
+    public interface IIndexingTaskExecutor : IDependency {
+        bool DeleteIndex(string indexName);
+        bool UpdateIndexBatch(string indexName);
+    }
+}

src/Orchard.Web/Modules/Orchard.Indexing/Services/IUpdateIndexScheduler.cs

@@ -0,0 +1,5 @@
+namespace Orchard.Indexing.Services {
+    public interface IUpdateIndexScheduler : IDependency {
+        void Schedule(string indexName);
+    }
+}

src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs

@@ -1,5 +1,4 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using Orchard.Localization;
 using Orchard.UI.Notify;
 
@@ -9,38 +8,41 @@ namespace Orchard.Indexing.Services
         private readonly IIndexManager _indexManager;
         private readonly IEnumerable<IIndexNotifierHandler> _indexNotifierHandlers;
         private readonly IIndexStatisticsProvider _indexStatisticsProvider;
+        private readonly IIndexingTaskExecutor _indexingTaskExecutor;
 
         public IndexingService(
             IOrchardServices services, 
             IIndexManager indexManager, 
             IEnumerable<IIndexNotifierHandler> indexNotifierHandlers,
-            IIndexStatisticsProvider indexStatisticsProvider) {
+            IIndexStatisticsProvider indexStatisticsProvider,
+            IIndexingTaskExecutor indexingTaskExecutor) {
             Services = services;
             _indexManager = indexManager;
             _indexNotifierHandlers = indexNotifierHandlers;
             _indexStatisticsProvider = indexStatisticsProvider;
+            _indexingTaskExecutor = indexingTaskExecutor;
             T = NullLocalizer.Instance;
         }
 
         public IOrchardServices Services { get; set; }
         public Localizer T { get; set; }
 
-        void IIndexingService.RebuildIndex(string indexName) {
+        public void RebuildIndex(string indexName) {
             if (!_indexManager.HasIndexProvider()) {
                 Services.Notifier.Warning(T("There is no search index to rebuild."));
                 return;
             }
 
-            var searchProvider = _indexManager.GetSearchIndexProvider();
-            if (searchProvider.Exists(indexName))
-                searchProvider.DeleteIndex(indexName);
-
-            searchProvider.CreateIndex(indexName); // or just reset the updated date and let the background process recreate the index
-
-            Services.Notifier.Information(T("The index {0} has been rebuilt.", indexName));
+            if(_indexingTaskExecutor.DeleteIndex(indexName)) {
+                Services.Notifier.Information(T("The index {0} has been rebuilt.", indexName));
+                UpdateIndex(indexName);
+            }
+            else {
+                Services.Notifier.Warning(T("The index {0} could no ben rebuilt. It might already be in use, please try again later.", indexName));
+            }
         }
 
-        void IIndexingService.UpdateIndex(string indexName) {
+        public void UpdateIndex(string indexName) {
             
             foreach(var handler in _indexNotifierHandlers) {
                 handler.UpdateIndex(indexName);

src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs

@@ -22,7 +22,8 @@ namespace Orchard.Indexing.Services {
     /// and singleton locks would not be shared accross those two.
     /// </remarks>
     [UsedImplicitly]
-    public class IndexingTaskExecutor : IIndexNotifierHandler, IIndexStatisticsProvider {
+    public class IndexingTaskExecutor : IIndexingTaskExecutor, IIndexStatisticsProvider
+    {
         private readonly IRepository<IndexingTaskRecord> _taskRepository;
         private readonly IRepository<ContentItemVersionRecord> _contentRepository;
         private IIndexProvider _indexProvider;
@@ -32,7 +33,7 @@ namespace Orchard.Indexing.Services {
         private readonly ShellSettings _shellSettings;
         private readonly ILockFileManager _lockFileManager;
         private readonly IClock _clock;
-        private const int ContentItemsPerLoop = 100;
+        private const int ContentItemsPerLoop = 50;
         private IndexingStatus _indexingStatus = IndexingStatus.Idle;
 
         public IndexingTaskExecutor(
@@ -57,9 +58,31 @@ namespace Orchard.Indexing.Services {
 
         public ILogger Logger { get; set; }
 
-        public void UpdateIndex(string indexName) {
-            // What to do here to run next batch in a separate transaction
-            while (UpdateIndexBatch(indexName)) {}
+        public bool DeleteIndex(string indexName) {
+            ILockFile lockFile = null;
+            var settingsFilename = GetSettingsFileName(indexName);
+            var lockFilename = settingsFilename + ".lock";
+
+            // acquire a lock file on the index
+            if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile)) {
+                Logger.Information("Could not delete the index. Already in use.");
+                return false;
+            }
+
+            using (lockFile) {
+                if (!_indexManager.HasIndexProvider()) {
+                    return false;
+                }
+
+                var searchProvider = _indexManager.GetSearchIndexProvider();
+                if (searchProvider.Exists(indexName)) {
+                    searchProvider.DeleteIndex(indexName);
+                }
+
+                DeleteSettings(indexName);
+            }
+
+            return true;
         }
 
         public bool UpdateIndexBatch(string indexName) {
@@ -83,10 +106,8 @@ namespace Orchard.Indexing.Services {
 
                 _indexProvider = _indexManager.GetSearchIndexProvider();
 
-                // should the index be rebuilt
-                if (!_indexProvider.Exists(indexName)) {
+                if (indexSettings.Mode == IndexingMode.Rebuild && indexSettings.LastContentId == 0) {
                     _indexProvider.CreateIndex(indexName);
-                    indexSettings = new IndexSettings();
 
                     // mark the last available task at the moment the process is started.
                     // once the Rebuild is done, Update will start at this point of the table
@@ -120,7 +141,7 @@ namespace Orchard.Indexing.Services {
                 // load all content items
                 var contentItems = _contentRepository
                     .Fetch(
-                        versionRecord => versionRecord.Published && versionRecord.ContentItemRecord.Id > indexSettings.LastContentId,
+                        versionRecord => versionRecord.Published && versionRecord.Id > indexSettings.LastContentId,
                         order => order.Asc(versionRecord => versionRecord.Id))
                     .Take(ContentItemsPerLoop)
                     .Select(versionRecord => _contentManager.Get(versionRecord.ContentItemRecord.Id, VersionOptions.VersionRecord(versionRecord.Id)))
@@ -182,7 +203,7 @@ namespace Orchard.Indexing.Services {
 
             // save current state of the index
             indexSettings.LastIndexedUtc = _clock.UtcNow;
-            _appDataFolder.CreateFile(settingsFilename, indexSettings.ToString());
+            _appDataFolder.CreateFile(settingsFilename, indexSettings.ToXml());
 
             if (deleteFromIndex.Count == 0 && addToIndex.Count == 0) {
                 // nothing more to do
@@ -218,10 +239,12 @@ namespace Orchard.Indexing.Services {
         /// <summary>
         /// Loads the settings file or create a new default one if it doesn't exist
         /// </summary>
-        public IndexSettings LoadSettings(string indexName) {
+        public IndexSettings LoadSettings(string indexName)
+        {
             var indexSettings = new IndexSettings();
             var settingsFilename = GetSettingsFileName(indexName);
-            if (_appDataFolder.FileExists(settingsFilename)) {
+            if (_appDataFolder.FileExists(settingsFilename))
+            {
                 var content = _appDataFolder.ReadFile(settingsFilename);
                 indexSettings = IndexSettings.Parse(content);
             }
@@ -229,6 +252,16 @@ namespace Orchard.Indexing.Services {
             return indexSettings;
         }
 
+        /// <summary>
+        /// Deletes the settings file
+        /// </summary>
+        public void DeleteSettings(string indexName) {
+            var settingsFilename = GetSettingsFileName(indexName);
+            if (_appDataFolder.FileExists(settingsFilename)) {
+                _appDataFolder.DeleteFile(settingsFilename);
+            }
+        }
+
         /// <summary>
         /// Creates a IDocumentIndex instance for a specific content item id. If the content 
         /// item is no more published, it returns null.

src/Orchard.Web/Modules/Orchard.Indexing/Services/UpdateIndexScheduler.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using Orchard.Environment.Configuration;
+using Orchard.Environment.Descriptor;
+using Orchard.Environment.State;
+
+namespace Orchard.Indexing.Services {
+    public class UpdateIndexScheduler : IUpdateIndexScheduler, IIndexNotifierHandler {
+        private readonly IProcessingEngine _processingEngine;
+        private readonly ShellSettings _shellSettings;
+        private readonly IShellDescriptorManager _shellDescriptorManager;
+        private readonly Lazy<IIndexingTaskExecutor> _indexingTaskExecutor;
+
+        public UpdateIndexScheduler(
+            IProcessingEngine processingEngine,
+            ShellSettings shellSettings,
+            IShellDescriptorManager shellDescriptorManager,
+            Lazy<IIndexingTaskExecutor> indexingTaskExecutor
+            ) {
+            _processingEngine = processingEngine;
+            _shellSettings = shellSettings;
+            _shellDescriptorManager = shellDescriptorManager;
+            _indexingTaskExecutor = indexingTaskExecutor;
+        }
+
+        public void Schedule(string indexName) {
+            var shellDescriptor = _shellDescriptorManager.GetShellDescriptor();
+            _processingEngine.AddTask(
+                _shellSettings,
+                shellDescriptor,
+                "IIndexNotifierHandler.UpdateIndex",
+                new Dictionary<string, object> { { "indexName", indexName } }
+            );
+        }
+
+        public void UpdateIndex(string indexName) {
+            if(_indexingTaskExecutor.Value.UpdateIndexBatch(indexName)) {
+                Schedule(indexName);
+            }
+        }
+    }
+}