From b98d35305a21c690847c369580ff8cc1a55fd286 Mon Sep 17 00:00:00 2001 From: Lombiq Date: Thu, 14 May 2015 12:54:02 +0200 Subject: [PATCH] Revert "Indexing task creation now happens gradually in a background task, preventing timeout/out of memory when editing a content type with many items. Fixes #4729." Commit moved to dev branch as it's not a completely safe change. This reverts commit 40243f2224452033a4826d22f56fd31da2368c66. --- .../Modules/Orchard.Indexing/Migrations.cs | 29 +--------- .../Models/IndexTaskBatchRecord.cs | 7 --- .../Orchard.Indexing/Orchard.Indexing.csproj | 4 -- .../CreateUpdateIndexTaskBackgroundTask.cs | 25 --------- .../IIndexTaskBatchManagementService.cs | 21 -------- .../IndexTaskBatchManagementService.cs | 53 ------------------- .../Orchard.Indexing/Settings/EditorEvents.cs | 48 ++++++++++++----- 7 files changed, 37 insertions(+), 150 deletions(-) delete mode 100644 src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexTaskBatchRecord.cs delete mode 100644 src/Orchard.Web/Modules/Orchard.Indexing/Services/CreateUpdateIndexTaskBackgroundTask.cs delete mode 100644 src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexTaskBatchManagementService.cs delete mode 100644 src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexTaskBatchManagementService.cs diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Migrations.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Migrations.cs index 1d57200ba..5905ff0d2 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Migrations.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Migrations.cs @@ -20,18 +20,7 @@ namespace Orchard.Indexing { .Column("ContentItemRecord_id") ); - SchemaBuilder.CreateTable("IndexTaskBatchRecord", - table => table - .Column("Id", column => column.PrimaryKey().Identity()) - .Column("BatchStartIndex") - .Column("ContentType") - ) - .AlterTable("IndexTaskBatchRecord", - table => - table.CreateIndex("IDX_ContentType", "ContentType") - ); - - return 3; + return 2; } public int UpdateFrom1() { @@ -46,21 +35,5 @@ namespace Orchard.Indexing { return 2; } - - public int UpdateFrom2() { - - SchemaBuilder.CreateTable("IndexTaskBatchRecord", - table => table - .Column("Id", column => column.PrimaryKey().Identity()) - .Column("BatchStartIndex") - .Column("ContentType") - ) - .AlterTable("IndexTaskBatchRecord", - table => - table.CreateIndex("IDX_ContentType", "ContentType") - ); - - return 3; - } } } \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexTaskBatchRecord.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexTaskBatchRecord.cs deleted file mode 100644 index 2ba820b46..000000000 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexTaskBatchRecord.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Orchard.Indexing.Models { - public class IndexTaskBatchRecord { - public virtual int Id { get; set; } - public virtual int BatchStartIndex { get; set; } - public virtual string ContentType { get; set; } - } -} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj b/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj index c783ef9c5..eae542b6a 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj @@ -78,9 +78,6 @@ Code - - - @@ -94,7 +91,6 @@ - diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/CreateUpdateIndexTaskBackgroundTask.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/CreateUpdateIndexTaskBackgroundTask.cs deleted file mode 100644 index 1418ad37f..000000000 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/CreateUpdateIndexTaskBackgroundTask.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Linq; -using Orchard.Tasks; -using Orchard.Tasks.Indexing; - -namespace Orchard.Indexing.Services { - public class CreateUpdateIndexTaskBackgroundTask : IBackgroundTask { - private readonly IIndexTaskBatchManagementService _indexTaskBatchManagementService; - private readonly IIndexingTaskManager _indexingTaskManager; - - public CreateUpdateIndexTaskBackgroundTask(IIndexTaskBatchManagementService indexTaskBatchManagementService, IIndexingTaskManager indexingTaskManager) { - _indexTaskBatchManagementService = indexTaskBatchManagementService; - _indexingTaskManager = indexingTaskManager; - } - - public void Sweep() { - var contentItemsLists = _indexTaskBatchManagementService.GetNextBatchOfContentItemsToIndex(); - - foreach (var contentItemsList in contentItemsLists) { - foreach (var contentItem in contentItemsList) { - _indexingTaskManager.CreateUpdateIndexTask(contentItem); - } - } - } - } -} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexTaskBatchManagementService.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexTaskBatchManagementService.cs deleted file mode 100644 index 233b52262..000000000 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexTaskBatchManagementService.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System.Collections.Generic; -using Orchard.ContentManagement; - -namespace Orchard.Indexing.Services { - /// - /// Manages the batches for indexing tasks. - /// - public interface IIndexTaskBatchManagementService : IDependency { - - /// - /// Registers a content type for the . - /// - /// The content type. - void RegisterContentType(string contentType); - - /// - /// Returns a list of a list of the registered content items. - /// - IEnumerable> GetNextBatchOfContentItemsToIndex(); - } -} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexTaskBatchManagementService.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexTaskBatchManagementService.cs deleted file mode 100644 index 963499012..000000000 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexTaskBatchManagementService.cs +++ /dev/null @@ -1,53 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using Orchard.ContentManagement; -using Orchard.Data; -using Orchard.Indexing.Models; - -namespace Orchard.Indexing.Services { - public class IndexTaskBatchManagementService : IIndexTaskBatchManagementService { - private readonly IRepository _indexTaskBatchRecordRepository; - private readonly IContentManager _contentManager; - - private const int BatchSize = 50; - - public IndexTaskBatchManagementService(IRepository indexTaskBatchRecordRepository, IContentManager contentManager) { - _indexTaskBatchRecordRepository = indexTaskBatchRecordRepository; - _contentManager = contentManager; - } - - public void RegisterContentType(string contentType) { - var registeredContentType = _indexTaskBatchRecordRepository.Table.Where(i => i.ContentType == contentType).FirstOrDefault(); - - if (registeredContentType == null) { - _indexTaskBatchRecordRepository.Create(new IndexTaskBatchRecord { ContentType = contentType, BatchStartIndex = 0 }); - } - else { - registeredContentType.BatchStartIndex = 0; - } - - } - - public IEnumerable> GetNextBatchOfContentItemsToIndex() { - var indexTaskBatchRecords = _indexTaskBatchRecordRepository.Table; - if (indexTaskBatchRecords == null) return null; - - var contentItemsList = new List>(); - - foreach (var indexTaskBatchRecord in indexTaskBatchRecords) { - var contentItems = _contentManager.Query(indexTaskBatchRecord.ContentType).Slice(indexTaskBatchRecord.BatchStartIndex, BatchSize).ToList(); - - if (contentItems.Any()) contentItemsList.Add(contentItems); - - if (contentItems.Count == 0 || contentItems.Count < BatchSize) { - _indexTaskBatchRecordRepository.Delete(_indexTaskBatchRecordRepository.Table.FirstOrDefault(i => i.ContentType == indexTaskBatchRecord.ContentType)); - } - else { - indexTaskBatchRecord.BatchStartIndex += BatchSize; - } - } - - return contentItemsList; - } - } -} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Settings/EditorEvents.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Settings/EditorEvents.cs index 37a6d3426..d4de846b8 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Settings/EditorEvents.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Settings/EditorEvents.cs @@ -1,23 +1,27 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; +using System.Linq; using Orchard.ContentManagement; using Orchard.ContentManagement.MetaData; using Orchard.ContentManagement.MetaData.Builders; using Orchard.ContentManagement.MetaData.Models; using Orchard.ContentManagement.ViewModels; -using Orchard.Indexing.Services; +using Orchard.Tasks.Indexing; namespace Orchard.Indexing.Settings { public class EditorEvents : ContentDefinitionEditorEventsBase { - private readonly IIndexTaskBatchManagementService _indexTaskBatchManagementService; + private readonly IIndexingTaskManager _indexingTaskManager; + private readonly IContentManager _contentManager; + + private const int PageSize = 50; + + public EditorEvents(IIndexingTaskManager indexingTaskManager, IContentManager contentManager){ + _indexingTaskManager = indexingTaskManager; + _contentManager = contentManager; + } private string _contentTypeName; private bool _tasksCreated; - public EditorEvents(IIndexTaskBatchManagementService indexTaskBatchManagementService) { - _indexTaskBatchManagementService = indexTaskBatchManagementService; - } - public override IEnumerable TypeEditor(ContentTypeDefinition definition) { var model = definition.Settings.GetModel(); _contentTypeName = definition.Name; @@ -37,12 +41,12 @@ namespace Orchard.Indexing.Settings { // if a an index is added, all existing content items need to be re-indexed CreateIndexingTasks(); } - + yield return DefinitionTemplate(model); } private string Clean(string value) { - if (String.IsNullOrEmpty(value)) + if (string.IsNullOrEmpty(value)) return value; return value.Trim(',', ' '); @@ -53,8 +57,7 @@ namespace Orchard.Indexing.Settings { /// private void CreateIndexingTasks() { if (!_tasksCreated) { - // Creating tasks in batches is needed because editing content type settings for a type with many items causes OutOfMemoryException, see issue: https://github.com/OrchardCMS/Orchard/issues/4729 - _indexTaskBatchManagementService.RegisterContentType(_contentTypeName); + CreateTasksForType(_contentTypeName); _tasksCreated = true; } } @@ -80,5 +83,26 @@ namespace Orchard.Indexing.Settings { yield return DefinitionTemplate(model); } + + private void CreateTasksForType(string type) { + var index = 0; + bool contentItemProcessed; + + // todo: load ids only, or create a queued job + // we create a task even for draft items, and the executor will filter based on the settings + + do { + contentItemProcessed = false; + var contentItemsToIndex = _contentManager.Query(VersionOptions.Latest, new [] { type }).Slice(index, PageSize); + + foreach (var contentItem in contentItemsToIndex) { + contentItemProcessed = true; + _indexingTaskManager.CreateUpdateIndexTask(contentItem); + } + + index += PageSize; + + } while (contentItemProcessed); + } } } \ No newline at end of file