Using IProcessEngine to schedule batched indexing; Modifying

IIndexingTaskExecutor to handle index deletiong in order to prevent
concurrency issues.

--HG--
branch : indexing

src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs

@@ -34,7 +34,7 @@ namespace Orchard.Tests.Modules.Indexing {
         private IIndexProvider _provider;
         private IAppDataFolder _appDataFolder;
         private ShellSettings _shellSettings;
-        private IIndexNotifierHandler _indexNotifier;
+        private IIndexingTaskExecutor _indexTaskExecutor;
         private IContentManager _contentManager;
         private Mock<IContentDefinitionManager> _contentDefinitionManager;
         private StubLogger _logger;
@@ -43,7 +43,6 @@ namespace Orchard.Tests.Modules.Indexing {
         private const string IndexName = "Search";
         private readonly string _basePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
 
-
         [TestFixtureTearDown]
         public void Clean() {
             if (Directory.Exists(_basePath)) {
@@ -61,7 +60,7 @@ namespace Orchard.Tests.Modules.Indexing {
             
             builder.RegisterType<LuceneIndexProvider>().As<IIndexProvider>();
             builder.RegisterInstance(_appDataFolder).As<IAppDataFolder>();
-            builder.RegisterType<IndexingTaskExecutor>().As<IIndexNotifierHandler>();
+            builder.RegisterType<IndexingTaskExecutor>().As<IIndexingTaskExecutor>();
             builder.RegisterType<DefaultIndexManager>().As<IIndexManager>();
             builder.RegisterType<IndexingTaskManager>().As<IIndexingTaskManager>();
             builder.RegisterType<DefaultContentManager>().As<IContentManager>();
@@ -105,9 +104,9 @@ namespace Orchard.Tests.Modules.Indexing {
             base.Init();
             _lockFileManager = _container.Resolve<ILockFileManager>();
             _provider = _container.Resolve<IIndexProvider>();
-            _indexNotifier = _container.Resolve<IIndexNotifierHandler>();
+            _indexTaskExecutor = _container.Resolve<IIndexingTaskExecutor>();
             _contentManager = _container.Resolve<IContentManager>();
-            ((IndexingTaskExecutor)_indexNotifier).Logger = _logger = new StubLogger();
+            ((IndexingTaskExecutor)_indexTaskExecutor).Logger = _logger = new StubLogger();
 
             var thingType = new ContentTypeDefinitionBuilder()
                 .Named(ThingDriver.ContentTypeName)
@@ -121,7 +120,7 @@ namespace Orchard.Tests.Modules.Indexing {
 
         [Test]
         public void IndexShouldBeEmptyWhenThereIsNoContent() {
-            _indexNotifier.UpdateIndex(IndexName);
+            while(_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
             Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0));
         }
 
@@ -137,7 +136,7 @@ namespace Orchard.Tests.Modules.Indexing {
 
             _contentManager.Create("alpha");
 
-            _indexNotifier.UpdateIndex(IndexName);
+            while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
             Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0));
         }
 
@@ -154,7 +153,7 @@ namespace Orchard.Tests.Modules.Indexing {
 
             _contentManager.Create("alpha");
 
-            _indexNotifier.UpdateIndex(IndexName);
+            while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
             Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0));
         }
 
@@ -163,22 +162,22 @@ namespace Orchard.Tests.Modules.Indexing {
             var content = _contentManager.Create<Thing>(ThingDriver.ContentTypeName);
             content.Text = "Lorem ipsum";
 
-            _indexNotifier.UpdateIndex(IndexName);
+            while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
             Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
         }
 
         [Test]
         public void ShouldUpdateTheIndexWhenContentIsPublished() {
             _contentManager.Create<Thing>(ThingDriver.ContentTypeName).Text = "Lorem ipsum";
-            _indexNotifier.UpdateIndex(IndexName);
+            while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
             Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
 
             // there should be nothing done
-            _indexNotifier.UpdateIndex(IndexName);
+            while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
             Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
 
             _contentManager.Create<Thing>(ThingDriver.ContentTypeName).Text = "Lorem ipsum";
-            _indexNotifier.UpdateIndex(IndexName);
+            while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
             Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2));
         }
 
@@ -187,13 +186,13 @@ namespace Orchard.Tests.Modules.Indexing {
             ILockFile lockFile = null;
             _lockFileManager.TryAcquireLock("Sites/My Site/Search.settings.xml.lock", ref lockFile);
             using (lockFile) {
-                _indexNotifier.UpdateIndex(IndexName);
+                while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
                 Assert.That(_logger.LogEntries.Count, Is.EqualTo(1));
                 Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Index was requested but is already running"));
             }
 
             _logger.LogEntries.Clear();
-            _indexNotifier.UpdateIndex(IndexName);
+            while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
             Assert.That(_logger.LogEntries, Has.None.Matches<LogEntry>(entry => entry.LogFormat == "Index was requested but is already running"));
         }
 
@@ -201,18 +200,18 @@ namespace Orchard.Tests.Modules.Indexing {
         public void ShouldUpdateTheIndexWhenContentIsUnPublished() {
             _contentManager.Create<Thing>(ThingDriver.ContentTypeName).Text = "Lorem ipsum";
 
-            _indexNotifier.UpdateIndex(IndexName);
+            while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
             Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
 
             var content = _contentManager.Create<Thing>(ThingDriver.ContentTypeName);
             content.Text = "Lorem ipsum";
-            
+
-            _indexNotifier.UpdateIndex(IndexName);
+            while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
             Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2));
 
             _contentManager.Unpublish(content.ContentItem);
-            
+
-            _indexNotifier.UpdateIndex(IndexName);
+            while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
             Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
         }
 
@@ -223,7 +222,7 @@ namespace Orchard.Tests.Modules.Indexing {
                 var content = _contentManager.Create<Thing>(ThingDriver.ContentTypeName);
                 content.Text = "Lorem ipsum " + i;
             }
-            _indexNotifier.UpdateIndex(IndexName);
+            while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
             Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(999));
         }
 

src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs

@@ -43,7 +43,6 @@ namespace Orchard.Indexing.Commands {
         [CommandHelp("index rebuild \r\n\t" + "Rebuilds the search index")]
         public string Rebuild() {
             _indexingService.RebuildIndex(SearchIndexName);
-            _indexingService.UpdateIndex(SearchIndexName);
 
             return T("Index is now being rebuilt...").Text;
         }

src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs

@@ -44,7 +44,6 @@ namespace Orchard.Indexing.Controllers {
                 return new HttpUnauthorizedResult();
 
             _indexingService.RebuildIndex(DefaultIndexName);
-            _indexingService.UpdateIndex(DefaultIndexName);
 
             return RedirectToAction("Index");
         }

src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs

@@ -36,7 +36,7 @@ namespace Orchard.Indexing.Models
                     Mode = (IndexingMode) Enum.Parse(typeof (IndexingMode), doc.Descendants(TagMode).First().Value),
                     LastIndexedId = Int32.Parse(doc.Descendants(TagLastIndexedId).First().Value),
                     LastContentId = Int32.Parse(doc.Descendants(TagLastContentId).First().Value),
-                    LastIndexedUtc = DateTime.Parse(doc.Descendants(TagLastIndexedUtc).First().Value)
+                    LastIndexedUtc = DateTime.Parse(doc.Descendants(TagLastIndexedUtc).First().Value).ToUniversalTime()
                 };
             }
             catch {
@@ -44,7 +44,7 @@ namespace Orchard.Indexing.Models
             }
         }
 
-        public override string ToString() {
+        public string ToXml() {
             return new XDocument(
                     new XElement(TagSettings,
                         new XElement(TagMode, Mode),

src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj

@@ -60,6 +60,9 @@
     <Compile Include="Models\IndexSettings.cs">
       <SubType>Code</SubType>
     </Compile>
+    <Compile Include="Services\UpdateIndexScheduler.cs" />
+    <Compile Include="Services\IIndexingTaskExecutor.cs" />
+    <Compile Include="Services\IUpdateIndexScheduler.cs" />
     <Compile Include="Services\IIndexStatisticsProvider.cs" />
     <Compile Include="Services\IndexServiceNotificationProvider.cs" />
     <Compile Include="Services\IndexingBackgroundTask.cs" />

src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexingTaskExecutor.cs

@@ -0,0 +1,6 @@
+namespace Orchard.Indexing.Services {
+    public interface IIndexingTaskExecutor : IDependency {
+        bool DeleteIndex(string indexName);
+        bool UpdateIndexBatch(string indexName);
+    }
+}

src/Orchard.Web/Modules/Orchard.Indexing/Services/IUpdateIndexScheduler.cs

@@ -0,0 +1,5 @@
+namespace Orchard.Indexing.Services {
+    public interface IUpdateIndexScheduler : IDependency {
+        void Schedule(string indexName);
+    }
+}

src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs

@@ -1,5 +1,4 @@
-using System;
+using System.Collections.Generic;
-using System.Collections.Generic;
 using Orchard.Localization;
 using Orchard.UI.Notify;
 
@@ -9,38 +8,41 @@ namespace Orchard.Indexing.Services
         private readonly IIndexManager _indexManager;
         private readonly IEnumerable<IIndexNotifierHandler> _indexNotifierHandlers;
         private readonly IIndexStatisticsProvider _indexStatisticsProvider;
+        private readonly IIndexingTaskExecutor _indexingTaskExecutor;
 
         public IndexingService(
             IOrchardServices services, 
             IIndexManager indexManager, 
             IEnumerable<IIndexNotifierHandler> indexNotifierHandlers,
-            IIndexStatisticsProvider indexStatisticsProvider) {
+            IIndexStatisticsProvider indexStatisticsProvider,
+            IIndexingTaskExecutor indexingTaskExecutor) {
             Services = services;
             _indexManager = indexManager;
             _indexNotifierHandlers = indexNotifierHandlers;
             _indexStatisticsProvider = indexStatisticsProvider;
+            _indexingTaskExecutor = indexingTaskExecutor;
             T = NullLocalizer.Instance;
         }
 
         public IOrchardServices Services { get; set; }
         public Localizer T { get; set; }
 
-        void IIndexingService.RebuildIndex(string indexName) {
+        public void RebuildIndex(string indexName) {
             if (!_indexManager.HasIndexProvider()) {
                 Services.Notifier.Warning(T("There is no search index to rebuild."));
                 return;
             }
 
-            var searchProvider = _indexManager.GetSearchIndexProvider();
+            if(_indexingTaskExecutor.DeleteIndex(indexName)) {
-            if (searchProvider.Exists(indexName))
+                Services.Notifier.Information(T("The index {0} has been rebuilt.", indexName));
-                searchProvider.DeleteIndex(indexName);
+                UpdateIndex(indexName);
-
+            }
-            searchProvider.CreateIndex(indexName); // or just reset the updated date and let the background process recreate the index
+            else {
-
+                Services.Notifier.Warning(T("The index {0} could no ben rebuilt. It might already be in use, please try again later.", indexName));
-            Services.Notifier.Information(T("The index {0} has been rebuilt.", indexName));
+            }
         }
 
-        void IIndexingService.UpdateIndex(string indexName) {
+        public void UpdateIndex(string indexName) {
             
             foreach(var handler in _indexNotifierHandlers) {
                 handler.UpdateIndex(indexName);

src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs

@@ -22,7 +22,8 @@ namespace Orchard.Indexing.Services {
     /// and singleton locks would not be shared accross those two.
     /// </remarks>
     [UsedImplicitly]
-    public class IndexingTaskExecutor : IIndexNotifierHandler, IIndexStatisticsProvider {
+    public class IndexingTaskExecutor : IIndexingTaskExecutor, IIndexStatisticsProvider
+    {
         private readonly IRepository<IndexingTaskRecord> _taskRepository;
         private readonly IRepository<ContentItemVersionRecord> _contentRepository;
         private IIndexProvider _indexProvider;
@@ -32,7 +33,7 @@ namespace Orchard.Indexing.Services {
         private readonly ShellSettings _shellSettings;
         private readonly ILockFileManager _lockFileManager;
         private readonly IClock _clock;
-        private const int ContentItemsPerLoop = 100;
+        private const int ContentItemsPerLoop = 50;
         private IndexingStatus _indexingStatus = IndexingStatus.Idle;
 
         public IndexingTaskExecutor(
@@ -57,9 +58,31 @@ namespace Orchard.Indexing.Services {
 
         public ILogger Logger { get; set; }
 
-        public void UpdateIndex(string indexName) {
+        public bool DeleteIndex(string indexName) {
-            // What to do here to run next batch in a separate transaction
+            ILockFile lockFile = null;
-            while (UpdateIndexBatch(indexName)) {}
+            var settingsFilename = GetSettingsFileName(indexName);
+            var lockFilename = settingsFilename + ".lock";
+
+            // acquire a lock file on the index
+            if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile)) {
+                Logger.Information("Could not delete the index. Already in use.");
+                return false;
+            }
+
+            using (lockFile) {
+                if (!_indexManager.HasIndexProvider()) {
+                    return false;
+                }
+
+                var searchProvider = _indexManager.GetSearchIndexProvider();
+                if (searchProvider.Exists(indexName)) {
+                    searchProvider.DeleteIndex(indexName);
+                }
+
+                DeleteSettings(indexName);
+            }
+
+            return true;
         }
 
         public bool UpdateIndexBatch(string indexName) {
@@ -83,10 +106,8 @@ namespace Orchard.Indexing.Services {
 
                 _indexProvider = _indexManager.GetSearchIndexProvider();
 
-                // should the index be rebuilt
+                if (indexSettings.Mode == IndexingMode.Rebuild && indexSettings.LastContentId == 0) {
-                if (!_indexProvider.Exists(indexName)) {
                     _indexProvider.CreateIndex(indexName);
-                    indexSettings = new IndexSettings();
 
                     // mark the last available task at the moment the process is started.
                     // once the Rebuild is done, Update will start at this point of the table
@@ -120,7 +141,7 @@ namespace Orchard.Indexing.Services {
                 // load all content items
                 var contentItems = _contentRepository
                     .Fetch(
-                        versionRecord => versionRecord.Published && versionRecord.ContentItemRecord.Id > indexSettings.LastContentId,
+                        versionRecord => versionRecord.Published && versionRecord.Id > indexSettings.LastContentId,
                         order => order.Asc(versionRecord => versionRecord.Id))
                     .Take(ContentItemsPerLoop)
                     .Select(versionRecord => _contentManager.Get(versionRecord.ContentItemRecord.Id, VersionOptions.VersionRecord(versionRecord.Id)))
@@ -182,7 +203,7 @@ namespace Orchard.Indexing.Services {
 
             // save current state of the index
             indexSettings.LastIndexedUtc = _clock.UtcNow;
-            _appDataFolder.CreateFile(settingsFilename, indexSettings.ToString());
+            _appDataFolder.CreateFile(settingsFilename, indexSettings.ToXml());
 
             if (deleteFromIndex.Count == 0 && addToIndex.Count == 0) {
                 // nothing more to do
@@ -218,10 +239,12 @@ namespace Orchard.Indexing.Services {
         /// <summary>
         /// Loads the settings file or create a new default one if it doesn't exist
         /// </summary>
-        public IndexSettings LoadSettings(string indexName) {
+        public IndexSettings LoadSettings(string indexName)
+        {
             var indexSettings = new IndexSettings();
             var settingsFilename = GetSettingsFileName(indexName);
-            if (_appDataFolder.FileExists(settingsFilename)) {
+            if (_appDataFolder.FileExists(settingsFilename))
+            {
                 var content = _appDataFolder.ReadFile(settingsFilename);
                 indexSettings = IndexSettings.Parse(content);
             }
@@ -229,6 +252,16 @@ namespace Orchard.Indexing.Services {
             return indexSettings;
         }
 
+        /// <summary>
+        /// Deletes the settings file
+        /// </summary>
+        public void DeleteSettings(string indexName) {
+            var settingsFilename = GetSettingsFileName(indexName);
+            if (_appDataFolder.FileExists(settingsFilename)) {
+                _appDataFolder.DeleteFile(settingsFilename);
+            }
+        }
+
         /// <summary>
         /// Creates a IDocumentIndex instance for a specific content item id. If the content 
         /// item is no more published, it returns null.

src/Orchard.Web/Modules/Orchard.Indexing/Services/UpdateIndexScheduler.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using Orchard.Environment.Configuration;
+using Orchard.Environment.Descriptor;
+using Orchard.Environment.State;
+
+namespace Orchard.Indexing.Services {
+    public class UpdateIndexScheduler : IUpdateIndexScheduler, IIndexNotifierHandler {
+        private readonly IProcessingEngine _processingEngine;
+        private readonly ShellSettings _shellSettings;
+        private readonly IShellDescriptorManager _shellDescriptorManager;
+        private readonly Lazy<IIndexingTaskExecutor> _indexingTaskExecutor;
+
+        public UpdateIndexScheduler(
+            IProcessingEngine processingEngine,
+            ShellSettings shellSettings,
+            IShellDescriptorManager shellDescriptorManager,
+            Lazy<IIndexingTaskExecutor> indexingTaskExecutor
+            ) {
+            _processingEngine = processingEngine;
+            _shellSettings = shellSettings;
+            _shellDescriptorManager = shellDescriptorManager;
+            _indexingTaskExecutor = indexingTaskExecutor;
+        }
+
+        public void Schedule(string indexName) {
+            var shellDescriptor = _shellDescriptorManager.GetShellDescriptor();
+            _processingEngine.AddTask(
+                _shellSettings,
+                shellDescriptor,
+                "IIndexNotifierHandler.UpdateIndex",
+                new Dictionary<string, object> { { "indexName", indexName } }
+            );
+        }
+
+        public void UpdateIndex(string indexName) {
+            if(_indexingTaskExecutor.Value.UpdateIndexBatch(indexName)) {
+                Schedule(indexName);
+            }
+        }
+    }
+}