Background indexing tasks can now delete existing indexes

Added new column to store the action type (update, delete)
Added new overloads to handle multiple indexing actions in the same transaction

--HG--
branch : dev

src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs

@@ -168,5 +168,18 @@ namespace Orchard.Tests.Indexing {
             Assert.That(searchBuilder.WithField("body", "hr").Search().Count(), Is.EqualTo(1));
             Assert.That(searchBuilder.WithField("body", "hr").Search().First().Id, Is.EqualTo(1));
         }
+
+        [Test] public void ShouldAllowNullOrEmptyStrings() {
+            _provider.CreateIndex("default");
+            _provider.Store("default", _provider.New(1).Add("body", null));
+            _provider.Store("default", _provider.New(2).Add("body", ""));
+            _provider.Store("default", _provider.New(3).Add("body", "<hr></hr>", true));
+
+            var searchBuilder = _provider.CreateSearchBuilder("default");
+
+            Assert.That(searchBuilder.Get(1).Id, Is.EqualTo(1));
+            Assert.That(searchBuilder.Get(2).Id, Is.EqualTo(2));
+            Assert.That(searchBuilder.Get(3).Id, Is.EqualTo(3));
+        }
     }
 }

src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexDocument.cs

@@ -25,6 +25,11 @@ namespace Orchard.Core.Indexing.Lucene {
 
         public IIndexDocument Add(string name, string value, bool removeTags) {
             AppendPreviousField();
+            
+            if(value == null) {
+                value = String.Empty;
+            }
+            
             if(removeTags) {
                 value = value.RemoveTags();
             }

src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs

@@ -1,5 +1,7 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using Lucene.Net.Analysis;
 using Lucene.Net.Analysis.Standard;
 using Lucene.Net.Documents;
@@ -14,7 +16,7 @@ using Orchard.Logging;
 
 namespace Orchard.Core.Indexing.Lucene {
     /// <summary>
-    /// Represents the default implementation of an IIndexProvider based on Lucene
+    /// Represents the default implementation of an IIndexProvider, based on Lucene
     /// </summary>
     public class DefaultIndexProvider : IIndexProvider {
         private readonly IAppDataFolder _appDataFolder;
@@ -73,41 +75,66 @@ namespace Orchard.Core.Indexing.Lucene {
         }
 
         public void Store(string indexName, IIndexDocument indexDocument) {
-            Store(indexName, (DefaultIndexDocument)indexDocument);
+            Store(indexName, new [] { (DefaultIndexDocument)indexDocument });
+        }
+
+        public void Store(string indexName, IEnumerable<IIndexDocument> indexDocuments) {
+            Store(indexName, indexDocuments.Cast<DefaultIndexDocument>());
+        }
+
+        public void Store(string indexName, IEnumerable<DefaultIndexDocument> indexDocuments) {
+            if(indexDocuments.AsQueryable().Count() == 0) {
+                return;
             }
 
-        public void Store(string indexName, DefaultIndexDocument indexDocument) {
             var writer = new IndexWriter(GetDirectory(indexName), _analyzer, false, IndexWriter.MaxFieldLength.UNLIMITED);
+            DefaultIndexDocument current = null;
 
             try {
+                foreach ( var indexDocument in indexDocuments ) {
+                    current = indexDocument;
                     var doc = CreateDocument(indexDocument);
                     writer.AddDocument(doc);
                     Logger.Debug("Document [{0}] indexed", indexDocument.Id);
                 }
+            }
             catch ( Exception ex ) {
-                Logger.Error(ex, "An unexpected error occured while removing the document [{0}] from the index [{1}].", indexDocument.Id, indexName);
+                Logger.Error(ex, "An unexpected error occured while add the document [{0}] from the index [{1}].", current.Id, indexName);
             }
             finally {
+                writer.Optimize();
                 writer.Close();
             }
 
         }
 
-        public void Delete(string indexName, int id) {
+        public void Delete(string indexName, int documentId) {
+            Delete(indexName, new[] { documentId });
+        }
+
+        public void Delete(string indexName, IEnumerable<int> documentIds) {
+            if ( documentIds.AsQueryable().Count() == 0 ) {
+                return;
+            }
+            
             var reader = IndexReader.Open(GetDirectory(indexName), false);
 
+            try {
+                foreach (var id in documentIds) {
                     try {
                         var term = new Term("id", id.ToString());
-                if ( reader.DeleteDocuments(term) != 0 ) {
+                        if (reader.DeleteDocuments(term) != 0) {
                             Logger.Error("The document [{0}] could not be removed from the index [{1}]", id, indexName);
                         }
                         else {
                             Logger.Debug("Document [{0}] removed from index", id);
                         }
                     }
-            catch ( Exception ex ) {
+                    catch (Exception ex) {
                         Logger.Error(ex, "An unexpected error occured while removing the document [{0}] from the index [{1}].", id, indexName);
                     }
+                }
+            }
             finally {
                 reader.Close();
             }
@@ -121,8 +148,5 @@ namespace Orchard.Core.Indexing.Lucene {
             return new DefaultSearchBuilder(GetDirectory(indexName));
         }
 
-        public IIndexDocument Get(string indexName, int id) {
-            throw new NotImplementedException();
-        }
     }
 }

src/Orchard.Web/Core/Indexing/Models/IndexingTaskRecord.cs

@@ -3,7 +3,12 @@ using Orchard.ContentManagement.Records;
 
 namespace Orchard.Core.Indexing.Models {
     public class IndexingTaskRecord {
+
+        public const int Update = 0;
+        public const int Delete = 1;
+
         public virtual int Id { get; set; }
+        public virtual int Action { get; set; }
         public virtual DateTime? CreatedUtc { get; set; }
         public virtual ContentItemRecord ContentItemRecord { get; set; }
     }

src/Orchard.Web/Core/Indexing/Services/CreateIndexingTaskHandler.cs

@@ -19,11 +19,11 @@ namespace Orchard.Core.Indexing.Services {
         }
 
         void CreateIndexingTask(PublishContentContext context, ContentPart<CommonRecord> part) {
-            _indexingTaskManager.CreateTask(context.ContentItem);
+            _indexingTaskManager.CreateUpdateIndexTask(context.ContentItem);
         }
 
         void RemoveIndexingTask(RemoveContentContext context, ContentPart<CommonRecord> part) {
-            _indexingTaskManager.DeleteTasks(context.ContentItem);
+            _indexingTaskManager.CreateDeleteIndexTask(context.ContentItem);
         }
 
     }

src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs

@@ -22,9 +22,9 @@ namespace Orchard.Core.Indexing.Services {
         private readonly IRepository<IndexingSettingsRecord> _settings;
         private readonly IEnumerable<IContentHandler> _handlers;
         private IIndexProvider _indexProvider;
-        private IIndexManager _indexManager;
+        private readonly IIndexManager _indexManager;
         private readonly IContentManager _contentManager;
-        private const string SearchIndexName = "search";
+        private const string SearchIndexName = "Search";
 
         public IndexingTaskExecutor(
             IClock clock,
@@ -46,7 +46,7 @@ namespace Orchard.Core.Indexing.Services {
 
         public void Sweep() {
 
-            if(!_indexManager.HasIndexProvider()) {
+            if ( !_indexManager.HasIndexProvider() ) {
                 return;
             }
 
@@ -55,8 +55,8 @@ namespace Orchard.Core.Indexing.Services {
             // retrieve last processed index time
             var settingsRecord = _settings.Table.FirstOrDefault();
 
-            if (settingsRecord == null) {
+            if ( settingsRecord == null ) {
-                _settings.Create(settingsRecord = new IndexingSettingsRecord { LatestIndexingUtc = new DateTime(1980, 1, 1)});
+                _settings.Create(settingsRecord = new IndexingSettingsRecord { LatestIndexingUtc = new DateTime(1980, 1, 1) });
             }
 
             var lastIndexing = settingsRecord.LatestIndexingUtc;
@@ -66,40 +66,75 @@ namespace Orchard.Core.Indexing.Services {
             var taskRecords = _repository.Fetch(x => x.CreatedUtc >= lastIndexing)
                 .ToArray();
 
-            if (taskRecords.Length == 0)
+            if ( taskRecords.Length == 0 )
                 return;
 
             Logger.Information("Processing {0} indexing tasks", taskRecords.Length);
 
 
-            if(!_indexProvider.Exists(SearchIndexName)) {
+            if ( !_indexProvider.Exists(SearchIndexName) ) {
                 _indexProvider.CreateIndex(SearchIndexName);
             }
 
-            foreach (var taskRecord in taskRecords) {
+            var updateIndexDocuments = new List<IIndexDocument>();
+            var deleteIndexDocuments = new List<int>();
+
+            // process Delete tasks
+            foreach ( var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete) ) {
+                var task = new IndexingTask(_contentManager, taskRecord);
+                deleteIndexDocuments.Add(taskRecord.ContentItemRecord.Id);
 
                 try {
+                    _repository.Delete(taskRecord);
+                }
+                catch ( Exception ex ) {
+                    Logger.Error(ex, "Could not delete task #{0}", taskRecord.Id);
+                }
+            }
+
+
+            try {
+                if ( deleteIndexDocuments.Count > 0 ) {
+                    _indexProvider.Delete(SearchIndexName, deleteIndexDocuments);
+                }
+            }
+            catch ( Exception ex ) {
+                Logger.Warning(ex, "An error occured while remove a document from the index");
+            }
+
+            // process Update tasks
+            foreach ( var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update) ) {
                 var task = new IndexingTask(_contentManager, taskRecord);
+
+                try {
                     var context = new IndexContentContext {
                         ContentItem = task.ContentItem,
                         IndexDocument = _indexProvider.New(task.ContentItem.Id)
                     };
 
                     // dispatch to handlers to retrieve index information
-                    foreach (var handler in _handlers) {
+                    foreach ( var handler in _handlers ) {
                         handler.Indexing(context);
                     }
 
-                    _indexProvider.Store(SearchIndexName, context.IndexDocument);
+                    updateIndexDocuments.Add(context.IndexDocument);
 
                     foreach ( var handler in _handlers ) {
                         handler.Indexed(context);
                     }
                 }
-                catch (Exception ex) {
+                catch ( Exception ex ) {
                     Logger.Warning(ex, "Unable to process indexing task #{0}", taskRecord.Id);
                 }
+            }
 
+            try {
+                if ( updateIndexDocuments.Count > 0 ) {
+                    _indexProvider.Store(SearchIndexName, updateIndexDocuments);
+                }
+            }
+            catch ( Exception ex ) {
+                Logger.Warning(ex, "An error occured while adding a document to the index");
             }
 
             _settings.Update(settingsRecord);

src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs

@@ -27,34 +27,46 @@ namespace Orchard.Core.Indexing.Services {
             _clock = clock;
             _repository = repository;
             _contentManager = contentManager;
+            _settings = settings;
             Logger = NullLogger.Instance;
         }
 
         public ILogger Logger { get; set; }
 
-        public void CreateTask(ContentItem contentItem) {
+        private void CreateTask(ContentItem contentItem, int action) {
-            if (contentItem == null) {
+            if ( contentItem == null ) {
                 throw new ArgumentNullException("contentItem");
             }
 
             // remove previous tasks for the same content item
             var tasks = _repository
-                .Fetch(x => x.Id == contentItem.Id )
+                .Fetch(x => x.ContentItemRecord.Id == contentItem.Id)
                 .ToArray();
 
-            foreach (var task in tasks) {
+            foreach ( var task in tasks ) {
                 _repository.Delete(task);
             }
 
             var taskRecord = new IndexingTaskRecord {
                 CreatedUtc = _clock.UtcNow,
-                                                        ContentItemRecord = contentItem.Record
+                ContentItemRecord = contentItem.Record,
+                Action = action
             };
 
             _repository.Create(taskRecord);
             
-            Logger.Information("Indexing task created for [{0}:{1}]", contentItem.ContentType, contentItem.Id);
+        }
 
+        public void CreateUpdateIndexTask(ContentItem contentItem) {
+
+            CreateTask(contentItem, IndexingTaskRecord.Update);
+            Logger.Information("Indexing task created for [{0}:{1}]", contentItem.ContentType, contentItem.Id);
+        }
+
+        public void CreateDeleteIndexTask(ContentItem contentItem) {
+
+            CreateTask(contentItem, IndexingTaskRecord.Delete);
+            Logger.Information("Deleting index task created for [{0}:{1}]", contentItem.ContentType, contentItem.Id);
         }
 
         public IEnumerable<IIndexingTask> GetTasks(DateTime? createdAfter) {

src/Orchard/Indexing/IIndexProvider.cs

@@ -1,4 +1,6 @@
-namespace Orchard.Indexing {
+using System.Collections.Generic;
+
+namespace Orchard.Indexing {
     public interface IIndexProvider : IDependency {
         /// <summary>
         /// Creates a new index
@@ -15,11 +17,6 @@
         /// </summary>
         void DeleteIndex(string name);
 
-        /// <summary>
-        /// Loads an existing document 
-        /// </summary>
-        IIndexDocument Get(string indexName, int documentId);
-
         /// <summary>
         /// Creates an empty document
         /// </summary>
@@ -31,10 +28,20 @@
         /// </summary>
         void Store(string indexName, IIndexDocument indexDocument);
 
+        /// <summary>
+        /// Adds a set of new document to the index
+        /// </summary>
+        void Store(string indexName, IEnumerable<IIndexDocument> indexDocuments);
+
         /// <summary>
         /// Removes an existing document from the index
         /// </summary>
-        void Delete(string indexName, int id);
+        void Delete(string indexName, int documentId);
+
+        /// <summary>
+        /// Removes a set of existing document from the index
+        /// </summary>
+        void Delete(string indexName, IEnumerable<int> documentIds);
 
         /// <summary>
         /// Creates a search builder for this provider

src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs

@@ -4,9 +4,30 @@ using Orchard.ContentManagement;
 
 namespace Orchard.Tasks.Indexing {
     public interface IIndexingTaskManager : IDependency {
-        void CreateTask(ContentItem contentItem);
+        /// <summary>
+        /// Adds a new entry in the index task table in order to create an index for the specified content item.
+        /// </summary>
+        void CreateUpdateIndexTask(ContentItem contentItem);
+
+        /// <summary>
+        /// Adds a new entry in the index task table in order to delete an existing index for the specified content item.
+        /// </summary>
+        void CreateDeleteIndexTask(ContentItem contentItem);
+
+        /// <summary>
+        /// Loads all indexing tasks created after to a specific date and time
+        /// </summary>
         IEnumerable<IIndexingTask> GetTasks(DateTime? createdAfter);
+        
+        /// <summary>
+        /// Deletes all indexing tasks previous to a specific date and time
+        /// </summary>
         void DeleteTasks(DateTime? createdBefore);
+
+        /// <summary>
+        /// Deletes all indexing tasks assigned to a specific content item
+        /// </summary>
+        /// <param name="contentItem"></param>
         void DeleteTasks(ContentItem contentItem);
     }
 }

src/Orchard/Tasks/SweepGenerator.cs

@@ -14,7 +14,7 @@ namespace Orchard.Tasks {
             _timer = new Timer();
             _timer.Elapsed += Elapsed;
             Logger = NullLogger.Instance;
-            Interval = TimeSpan.FromMinutes(5);
+            Interval = TimeSpan.FromMinutes(1);
         }
 
         public ILogger Logger { get; set; }