diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Migrations.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Migrations.cs index 5905ff0d2..1d57200ba 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Migrations.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Migrations.cs @@ -20,7 +20,18 @@ namespace Orchard.Indexing { .Column("ContentItemRecord_id") ); - return 2; + SchemaBuilder.CreateTable("IndexTaskBatchRecord", + table => table + .Column("Id", column => column.PrimaryKey().Identity()) + .Column("BatchStartIndex") + .Column("ContentType") + ) + .AlterTable("IndexTaskBatchRecord", + table => + table.CreateIndex("IDX_ContentType", "ContentType") + ); + + return 3; } public int UpdateFrom1() { @@ -35,5 +46,21 @@ namespace Orchard.Indexing { return 2; } + + public int UpdateFrom2() { + + SchemaBuilder.CreateTable("IndexTaskBatchRecord", + table => table + .Column("Id", column => column.PrimaryKey().Identity()) + .Column("BatchStartIndex") + .Column("ContentType") + ) + .AlterTable("IndexTaskBatchRecord", + table => + table.CreateIndex("IDX_ContentType", "ContentType") + ); + + return 3; + } } } \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexTaskBatchRecord.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexTaskBatchRecord.cs new file mode 100644 index 000000000..2ba820b46 --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexTaskBatchRecord.cs @@ -0,0 +1,7 @@ +namespace Orchard.Indexing.Models { + public class IndexTaskBatchRecord { + public virtual int Id { get; set; } + public virtual int BatchStartIndex { get; set; } + public virtual string ContentType { get; set; } + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj b/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj index eae542b6a..c783ef9c5 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj @@ -78,6 +78,9 @@ Code + + + @@ -91,6 +94,7 @@ + diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/CreateUpdateIndexTaskBackgroundTask.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/CreateUpdateIndexTaskBackgroundTask.cs new file mode 100644 index 000000000..1418ad37f --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/CreateUpdateIndexTaskBackgroundTask.cs @@ -0,0 +1,25 @@ +using System.Linq; +using Orchard.Tasks; +using Orchard.Tasks.Indexing; + +namespace Orchard.Indexing.Services { + public class CreateUpdateIndexTaskBackgroundTask : IBackgroundTask { + private readonly IIndexTaskBatchManagementService _indexTaskBatchManagementService; + private readonly IIndexingTaskManager _indexingTaskManager; + + public CreateUpdateIndexTaskBackgroundTask(IIndexTaskBatchManagementService indexTaskBatchManagementService, IIndexingTaskManager indexingTaskManager) { + _indexTaskBatchManagementService = indexTaskBatchManagementService; + _indexingTaskManager = indexingTaskManager; + } + + public void Sweep() { + var contentItemsLists = _indexTaskBatchManagementService.GetNextBatchOfContentItemsToIndex(); + + foreach (var contentItemsList in contentItemsLists) { + foreach (var contentItem in contentItemsList) { + _indexingTaskManager.CreateUpdateIndexTask(contentItem); + } + } + } + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexTaskBatchManagementService.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexTaskBatchManagementService.cs new file mode 100644 index 000000000..233b52262 --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexTaskBatchManagementService.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using Orchard.ContentManagement; + +namespace Orchard.Indexing.Services { + /// + /// Manages the batches for indexing tasks. + /// + public interface IIndexTaskBatchManagementService : IDependency { + + /// + /// Registers a content type for the . + /// + /// The content type. + void RegisterContentType(string contentType); + + /// + /// Returns a list of a list of the registered content items. + /// + IEnumerable> GetNextBatchOfContentItemsToIndex(); + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexTaskBatchManagementService.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexTaskBatchManagementService.cs new file mode 100644 index 000000000..963499012 --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexTaskBatchManagementService.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using System.Linq; +using Orchard.ContentManagement; +using Orchard.Data; +using Orchard.Indexing.Models; + +namespace Orchard.Indexing.Services { + public class IndexTaskBatchManagementService : IIndexTaskBatchManagementService { + private readonly IRepository _indexTaskBatchRecordRepository; + private readonly IContentManager _contentManager; + + private const int BatchSize = 50; + + public IndexTaskBatchManagementService(IRepository indexTaskBatchRecordRepository, IContentManager contentManager) { + _indexTaskBatchRecordRepository = indexTaskBatchRecordRepository; + _contentManager = contentManager; + } + + public void RegisterContentType(string contentType) { + var registeredContentType = _indexTaskBatchRecordRepository.Table.Where(i => i.ContentType == contentType).FirstOrDefault(); + + if (registeredContentType == null) { + _indexTaskBatchRecordRepository.Create(new IndexTaskBatchRecord { ContentType = contentType, BatchStartIndex = 0 }); + } + else { + registeredContentType.BatchStartIndex = 0; + } + + } + + public IEnumerable> GetNextBatchOfContentItemsToIndex() { + var indexTaskBatchRecords = _indexTaskBatchRecordRepository.Table; + if (indexTaskBatchRecords == null) return null; + + var contentItemsList = new List>(); + + foreach (var indexTaskBatchRecord in indexTaskBatchRecords) { + var contentItems = _contentManager.Query(indexTaskBatchRecord.ContentType).Slice(indexTaskBatchRecord.BatchStartIndex, BatchSize).ToList(); + + if (contentItems.Any()) contentItemsList.Add(contentItems); + + if (contentItems.Count == 0 || contentItems.Count < BatchSize) { + _indexTaskBatchRecordRepository.Delete(_indexTaskBatchRecordRepository.Table.FirstOrDefault(i => i.ContentType == indexTaskBatchRecord.ContentType)); + } + else { + indexTaskBatchRecord.BatchStartIndex += BatchSize; + } + } + + return contentItemsList; + } + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Settings/EditorEvents.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Settings/EditorEvents.cs index d4de846b8..37a6d3426 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Settings/EditorEvents.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Settings/EditorEvents.cs @@ -1,27 +1,23 @@ -using System.Collections.Generic; -using System.Linq; +using System; +using System.Collections.Generic; using Orchard.ContentManagement; using Orchard.ContentManagement.MetaData; using Orchard.ContentManagement.MetaData.Builders; using Orchard.ContentManagement.MetaData.Models; using Orchard.ContentManagement.ViewModels; -using Orchard.Tasks.Indexing; +using Orchard.Indexing.Services; namespace Orchard.Indexing.Settings { public class EditorEvents : ContentDefinitionEditorEventsBase { - private readonly IIndexingTaskManager _indexingTaskManager; - private readonly IContentManager _contentManager; - - private const int PageSize = 50; - - public EditorEvents(IIndexingTaskManager indexingTaskManager, IContentManager contentManager){ - _indexingTaskManager = indexingTaskManager; - _contentManager = contentManager; - } + private readonly IIndexTaskBatchManagementService _indexTaskBatchManagementService; private string _contentTypeName; private bool _tasksCreated; + public EditorEvents(IIndexTaskBatchManagementService indexTaskBatchManagementService) { + _indexTaskBatchManagementService = indexTaskBatchManagementService; + } + public override IEnumerable TypeEditor(ContentTypeDefinition definition) { var model = definition.Settings.GetModel(); _contentTypeName = definition.Name; @@ -41,12 +37,12 @@ namespace Orchard.Indexing.Settings { // if a an index is added, all existing content items need to be re-indexed CreateIndexingTasks(); } - + yield return DefinitionTemplate(model); } private string Clean(string value) { - if (string.IsNullOrEmpty(value)) + if (String.IsNullOrEmpty(value)) return value; return value.Trim(',', ' '); @@ -57,7 +53,8 @@ namespace Orchard.Indexing.Settings { /// private void CreateIndexingTasks() { if (!_tasksCreated) { - CreateTasksForType(_contentTypeName); + // Creating tasks in batches is needed because editing content type settings for a type with many items causes OutOfMemoryException, see issue: https://github.com/OrchardCMS/Orchard/issues/4729 + _indexTaskBatchManagementService.RegisterContentType(_contentTypeName); _tasksCreated = true; } } @@ -83,26 +80,5 @@ namespace Orchard.Indexing.Settings { yield return DefinitionTemplate(model); } - - private void CreateTasksForType(string type) { - var index = 0; - bool contentItemProcessed; - - // todo: load ids only, or create a queued job - // we create a task even for draft items, and the executor will filter based on the settings - - do { - contentItemProcessed = false; - var contentItemsToIndex = _contentManager.Query(VersionOptions.Latest, new [] { type }).Slice(index, PageSize); - - foreach (var contentItem in contentItemsToIndex) { - contentItemProcessed = true; - _indexingTaskManager.CreateUpdateIndexTask(contentItem); - } - - index += PageSize; - - } while (contentItemProcessed); - } } } \ No newline at end of file