Indexing task creation now happens gradually in a background task, preventing timeout/out of memory when editing a content type with many items. Fixes #4729.

src/Orchard.Web/Modules/Orchard.Indexing/Migrations.cs

@@ -20,7 +20,18 @@ namespace Orchard.Indexing {
                     .Column<int>("ContentItemRecord_id")
                 );
 
-            return 2;
+            SchemaBuilder.CreateTable("IndexTaskBatchRecord",
+                table => table
+                    .Column<int>("Id", column => column.PrimaryKey().Identity())
+                    .Column<int>("BatchStartIndex")
+                    .Column<string>("ContentType")
+                )
+            .AlterTable("IndexTaskBatchRecord",
+                table =>
+                    table.CreateIndex("IDX_ContentType", "ContentType")
+                );
+
+            return 3;
         }
 
         public int UpdateFrom1() {
@@ -35,5 +46,21 @@ namespace Orchard.Indexing {
 
             return 2;
         }
+
+        public int UpdateFrom2() {
+
+            SchemaBuilder.CreateTable("IndexTaskBatchRecord",
+                table => table
+                    .Column<int>("Id", column => column.PrimaryKey().Identity())
+                    .Column<int>("BatchStartIndex")
+                    .Column<string>("ContentType")
+                )
+            .AlterTable("IndexTaskBatchRecord",
+                table =>
+                    table.CreateIndex("IDX_ContentType", "ContentType")
+                );
+
+            return 3;
+        }
     }
 }

src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexTaskBatchRecord.cs

@@ -0,0 +1,7 @@
+namespace Orchard.Indexing.Models {
+    public class IndexTaskBatchRecord {
+        public virtual int Id { get; set; }
+        public virtual int BatchStartIndex { get; set; }
+        public virtual string ContentType { get; set; }
+    }
+}

src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj

@@ -78,6 +78,9 @@
     <Compile Include="Models\IndexSettings.cs">
       <SubType>Code</SubType>
     </Compile>
+    <Compile Include="Models\IndexTaskBatchRecord.cs" />
+    <Compile Include="Services\IIndexTaskBatchManagementService.cs" />
+    <Compile Include="Services\IndexTaskBatchManagementService.cs" />
     <Compile Include="Services\UpdateIndexScheduler.cs" />
     <Compile Include="Services\IIndexingTaskExecutor.cs" />
     <Compile Include="Services\IUpdateIndexScheduler.cs" />
@@ -91,6 +94,7 @@
     <Compile Include="Services\IndexingService.cs" />
     <Compile Include="Settings\EditorEvents.cs" />
     <Compile Include="Settings\FieldIndexing.cs" />
+    <Compile Include="Services\CreateUpdateIndexTaskBackgroundTask.cs" />
     <Compile Include="Settings\TypeIndexing.cs" />
     <Compile Include="ViewModels\IndexViewModel.cs" />
   </ItemGroup>

src/Orchard.Web/Modules/Orchard.Indexing/Services/CreateUpdateIndexTaskBackgroundTask.cs

@@ -0,0 +1,25 @@
+using System.Linq;
+using Orchard.Tasks;
+using Orchard.Tasks.Indexing;
+
+namespace Orchard.Indexing.Services {
+    public class CreateUpdateIndexTaskBackgroundTask : IBackgroundTask {
+        private readonly IIndexTaskBatchManagementService _indexTaskBatchManagementService;
+        private readonly IIndexingTaskManager _indexingTaskManager;
+
+        public CreateUpdateIndexTaskBackgroundTask(IIndexTaskBatchManagementService indexTaskBatchManagementService, IIndexingTaskManager indexingTaskManager) {
+            _indexTaskBatchManagementService = indexTaskBatchManagementService;
+            _indexingTaskManager = indexingTaskManager;
+        }
+
+        public void Sweep() {
+            var contentItemsLists = _indexTaskBatchManagementService.GetNextBatchOfContentItemsToIndex();
+
+            foreach (var contentItemsList in contentItemsLists) {
+                foreach (var contentItem in contentItemsList) {
+                    _indexingTaskManager.CreateUpdateIndexTask(contentItem);
+                }
+            }
+        }
+    }
+}

src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexTaskBatchManagementService.cs

@@ -0,0 +1,21 @@
+using System.Collections.Generic;
+using Orchard.ContentManagement;
+
+namespace Orchard.Indexing.Services {
+    /// <summary>
+    /// Manages the batches for indexing tasks.
+    /// </summary>
+    public interface IIndexTaskBatchManagementService : IDependency {
+
+        /// <summary>
+        /// Registers a content type for the <see cref="CreateUpdateIndexTaskBackgroundTask"/>.
+        /// </summary>
+        /// <param name="contentType">The content type.</param>
+        void RegisterContentType(string contentType);
+        
+        /// <summary>
+        /// Returns a list of a list of the registered content items.
+        /// </summary>
+        IEnumerable<IEnumerable<ContentItem>> GetNextBatchOfContentItemsToIndex();
+    }
+}

src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexTaskBatchManagementService.cs

@@ -0,0 +1,53 @@
+using System.Collections.Generic;
+using System.Linq;
+using Orchard.ContentManagement;
+using Orchard.Data;
+using Orchard.Indexing.Models;
+
+namespace Orchard.Indexing.Services {
+    public class IndexTaskBatchManagementService : IIndexTaskBatchManagementService {
+        private readonly IRepository<IndexTaskBatchRecord> _indexTaskBatchRecordRepository;
+        private readonly IContentManager _contentManager;
+
+        private const int BatchSize = 50;
+
+        public IndexTaskBatchManagementService(IRepository<IndexTaskBatchRecord> indexTaskBatchRecordRepository, IContentManager contentManager) {
+            _indexTaskBatchRecordRepository = indexTaskBatchRecordRepository;
+            _contentManager = contentManager;
+        }
+
+        public void RegisterContentType(string contentType) {
+            var registeredContentType = _indexTaskBatchRecordRepository.Table.Where(i => i.ContentType == contentType).FirstOrDefault();
+
+            if (registeredContentType == null) {
+                _indexTaskBatchRecordRepository.Create(new IndexTaskBatchRecord { ContentType = contentType, BatchStartIndex = 0 });
+            }
+            else {
+                registeredContentType.BatchStartIndex = 0;
+            }
+
+        }
+
+        public IEnumerable<IEnumerable<ContentItem>> GetNextBatchOfContentItemsToIndex() {
+            var indexTaskBatchRecords = _indexTaskBatchRecordRepository.Table;
+            if (indexTaskBatchRecords == null) return null;
+
+            var contentItemsList = new List<List<ContentItem>>();
+
+            foreach (var indexTaskBatchRecord in indexTaskBatchRecords) {
+                var contentItems = _contentManager.Query(indexTaskBatchRecord.ContentType).Slice(indexTaskBatchRecord.BatchStartIndex, BatchSize).ToList();
+
+                if (contentItems.Any()) contentItemsList.Add(contentItems);
+
+                if (contentItems.Count == 0 || contentItems.Count < BatchSize) {
+                    _indexTaskBatchRecordRepository.Delete(_indexTaskBatchRecordRepository.Table.FirstOrDefault(i => i.ContentType == indexTaskBatchRecord.ContentType));
+                }
+                else {
+                    indexTaskBatchRecord.BatchStartIndex += BatchSize;
+                }
+            }
+
+            return contentItemsList;
+        }
+    }
+}

src/Orchard.Web/Modules/Orchard.Indexing/Settings/EditorEvents.cs

@@ -1,27 +1,23 @@
-using System.Collections.Generic;
+using System;
-using System.Linq;
+using System.Collections.Generic;
 using Orchard.ContentManagement;
 using Orchard.ContentManagement.MetaData;
 using Orchard.ContentManagement.MetaData.Builders;
 using Orchard.ContentManagement.MetaData.Models;
 using Orchard.ContentManagement.ViewModels;
-using Orchard.Tasks.Indexing;
+using Orchard.Indexing.Services;
 
 namespace Orchard.Indexing.Settings {
     public class EditorEvents : ContentDefinitionEditorEventsBase {
-        private readonly IIndexingTaskManager _indexingTaskManager;
+        private readonly IIndexTaskBatchManagementService _indexTaskBatchManagementService;
-        private readonly IContentManager _contentManager;
-
-        private const int PageSize = 50;
-
-        public EditorEvents(IIndexingTaskManager indexingTaskManager, IContentManager contentManager){
-            _indexingTaskManager = indexingTaskManager;
-            _contentManager = contentManager;
-        }
 
         private string _contentTypeName;
         private bool _tasksCreated;
 
+        public EditorEvents(IIndexTaskBatchManagementService indexTaskBatchManagementService) {
+            _indexTaskBatchManagementService = indexTaskBatchManagementService;
+        }
+
         public override IEnumerable<TemplateViewModel> TypeEditor(ContentTypeDefinition definition) {
             var model = definition.Settings.GetModel<TypeIndexing>();
             _contentTypeName = definition.Name;
@@ -41,12 +37,12 @@ namespace Orchard.Indexing.Settings {
                 // if a an index is added, all existing content items need to be re-indexed
                 CreateIndexingTasks();
             }
-            
+
             yield return DefinitionTemplate(model);
         }
 
         private string Clean(string value) {
-            if (string.IsNullOrEmpty(value))
+            if (String.IsNullOrEmpty(value))
                 return value;
 
             return value.Trim(',', ' ');
@@ -57,7 +53,8 @@ namespace Orchard.Indexing.Settings {
         /// </summary>
         private void CreateIndexingTasks() {
             if (!_tasksCreated) {
-                CreateTasksForType(_contentTypeName);
+                // Creating tasks in batches is needed because editing content type settings for a type with many items causes OutOfMemoryException, see issue: https://github.com/OrchardCMS/Orchard/issues/4729
+                _indexTaskBatchManagementService.RegisterContentType(_contentTypeName);
                 _tasksCreated = true;
             }
         }
@@ -83,26 +80,5 @@ namespace Orchard.Indexing.Settings {
 
             yield return DefinitionTemplate(model);
         }
-
-        private void CreateTasksForType(string type) {
-            var index = 0;
-            bool contentItemProcessed;
-
-            // todo: load ids only, or create a queued job
-            // we create a task even for draft items, and the executor will filter based on the settings
-
-            do {
-                contentItemProcessed = false;
-                var contentItemsToIndex = _contentManager.Query(VersionOptions.Latest, new [] { type }).Slice(index, PageSize);
-
-                foreach (var contentItem in contentItemsToIndex) {
-                    contentItemProcessed = true;
-                    _indexingTaskManager.CreateUpdateIndexTask(contentItem);
-                }
-
-                index += PageSize;
-
-            } while (contentItemProcessed);
-        }
     }
 }