lsm/Orchard
1
0
Fork 0
mirror of https://github.com/OrchardCMS/Orchard.git synced 2026-02-09 09:16:41 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Improving indexing process

Browse Source
This commit is contained in:
Sebastien Ros
2014-08-27 12:22:23 -07:00
parent 42189e9297
commit 5f92beb0fb
1 changed files with 84 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs
+84 -55
View File
@@ -33,6 +33,7 @@ namespace Orchard.Indexing.Services {
private readonly ShellSettings _shellSettings; private readonly ShellSettings _shellSettings;
private readonly ILockFileManager _lockFileManager; private readonly ILockFileManager _lockFileManager;
private readonly IClock _clock; private readonly IClock _clock;
private readonly ITransactionManager _transactionManager;
private const int ContentItemsPerLoop = 50; private const int ContentItemsPerLoop = 50;
private IndexingStatus _indexingStatus = IndexingStatus.Idle; private IndexingStatus _indexingStatus = IndexingStatus.Idle;
@@ -44,7 +45,8 @@ namespace Orchard.Indexing.Services {
IAppDataFolder appDataFolder, IAppDataFolder appDataFolder,
ShellSettings shellSettings, ShellSettings shellSettings,
ILockFileManager lockFileManager, ILockFileManager lockFileManager,
IClock clock) { IClock clock,
ITransactionManager transactionManager) {
_taskRepository = taskRepository; _taskRepository = taskRepository;
_contentRepository = contentRepository; _contentRepository = contentRepository;
_indexManager = indexManager; _indexManager = indexManager;
@@ -52,6 +54,7 @@ namespace Orchard.Indexing.Services {
_appDataFolder = appDataFolder; _appDataFolder = appDataFolder;
_shellSettings = shellSettings; _shellSettings = shellSettings;
_lockFileManager = lockFileManager; _lockFileManager = lockFileManager;
_transactionManager = transactionManager;
_clock = clock; _clock = clock;
Logger = NullLogger.Instance; Logger = NullLogger.Instance;
} }
@@ -143,89 +146,115 @@ namespace Orchard.Indexing.Services {
private bool BatchIndex(string indexName, string settingsFilename, IndexSettings indexSettings) { private bool BatchIndex(string indexName, string settingsFilename, IndexSettings indexSettings) {
var addToIndex = new List<IDocumentIndex>(); var addToIndex = new List<IDocumentIndex>();
var deleteFromIndex = new List<int>(); var deleteFromIndex = new List<int>();
bool loop = false;
// Rebuilding the index ? // Rebuilding the index ?
if (indexSettings.Mode == IndexingMode.Rebuild) { if (indexSettings.Mode == IndexingMode.Rebuild) {
Logger.Information("Rebuilding index"); Logger.Information("Rebuilding index");
_indexingStatus = IndexingStatus.Rebuilding; _indexingStatus = IndexingStatus.Rebuilding;
// load all content items do {
var contentItems = _contentRepository loop = true;
.Table.Where(versionRecord => versionRecord.Published && versionRecord.Id > indexSettings.LastContentId)
.OrderBy(versionRecord => versionRecord.Id)
.Take(ContentItemsPerLoop)
.ToList()
.Select(versionRecord => _contentManager.Get(versionRecord.ContentItemRecord.Id, VersionOptions.VersionRecord(versionRecord.Id)))
.Distinct()
.ToList();
// if no more elements to index, switch to update mode // load all content items
if (contentItems.Count == 0) { var contentItems = _contentRepository
indexSettings.Mode = IndexingMode.Update; .Table.Where(versionRecord => versionRecord.Published && versionRecord.Id > indexSettings.LastContentId)
} .OrderBy(versionRecord => versionRecord.Id)
.Take(ContentItemsPerLoop)
.ToList()
.Select(versionRecord => _contentManager.Get(versionRecord.ContentItemRecord.Id, VersionOptions.VersionRecord(versionRecord.Id)))
.Distinct()
.ToList();
foreach (var item in contentItems) { // if no more elements to index, switch to update mode
try { if (contentItems.Count == 0) {
indexSettings.Mode = IndexingMode.Update;
}
// skip items from types which are not indexed foreach (var item in contentItems) {
var settings = GetTypeIndexingSettings(item); try {
if (settings.List.Contains(indexName)) {
IDocumentIndex documentIndex = ExtractDocumentIndex(item);
if (documentIndex != null && documentIndex.IsDirty) { // skip items from types which are not indexed
addToIndex.Add(documentIndex); var settings = GetTypeIndexingSettings(item);
if (settings.List.Contains(indexName)) {
IDocumentIndex documentIndex = ExtractDocumentIndex(item);
if (documentIndex != null && documentIndex.IsDirty) {
addToIndex.Add(documentIndex);
}
} }
}
indexSettings.LastContentId = item.VersionRecord.Id; indexSettings.LastContentId = item.VersionRecord.Id;
}
catch (Exception ex) {
Logger.Warning(ex, "Unable to index content item #{0} during rebuild", item.Id);
}
} }
catch (Exception ex) {
Logger.Warning(ex, "Unable to index content item #{0} during rebuild", item.Id); if (contentItems.Count < ContentItemsPerLoop) {
loop = false;
} }
} else {
_contentManager.Clear();
_transactionManager.RequireNew();
}
} while (loop);
} }
if (indexSettings.Mode == IndexingMode.Update) { if (indexSettings.Mode == IndexingMode.Update) {
Logger.Information("Updating index"); Logger.Information("Updating index");
_indexingStatus = IndexingStatus.Updating; _indexingStatus = IndexingStatus.Updating;
var indexingTasks = _taskRepository do {
.Table.Where(x => x.Id > indexSettings.LastIndexedId) var indexingTasks = _taskRepository
.OrderBy(x => x.Id) .Table.Where(x => x.Id > indexSettings.LastIndexedId)
.Take(ContentItemsPerLoop) .OrderBy(x => x.Id)
.ToList() .Take(ContentItemsPerLoop)
.GroupBy(x => x.ContentItemRecord.Id) .ToList()
.Select(group => new {TaskId = group.Max(task => task.Id), Delete = group.Last().Action == IndexingTaskRecord.Delete, Id = group.Key, ContentItem = _contentManager.Get(group.Key, VersionOptions.Published)}) .GroupBy(x => x.ContentItemRecord.Id)
.OrderBy(x => x.TaskId) .Select(group => new { TaskId = group.Max(task => task.Id), Delete = group.Last().Action == IndexingTaskRecord.Delete, Id = group.Key, ContentItem = _contentManager.Get(group.Key, VersionOptions.Published) })
.ToArray(); .OrderBy(x => x.TaskId)
.ToArray();
foreach (var item in indexingTasks) { foreach (var item in indexingTasks) {
try { try {
IDocumentIndex documentIndex = null; IDocumentIndex documentIndex = null;
// item.ContentItem can be null if the content item has been deleted // item.ContentItem can be null if the content item has been deleted
if (item.ContentItem != null) { if (item.ContentItem != null) {
// skip items from types which are not indexed // skip items from types which are not indexed
var settings = GetTypeIndexingSettings(item.ContentItem); var settings = GetTypeIndexingSettings(item.ContentItem);
if (settings.List.Contains(indexName)) { if (settings.List.Contains(indexName)) {
documentIndex = ExtractDocumentIndex(item.ContentItem); documentIndex = ExtractDocumentIndex(item.ContentItem);
}
} }
}
if (documentIndex == null || item.Delete) { if (documentIndex == null || item.Delete) {
deleteFromIndex.Add(item.Id); deleteFromIndex.Add(item.Id);
} }
else if (documentIndex.IsDirty) { else if (documentIndex.IsDirty) {
addToIndex.Add(documentIndex); addToIndex.Add(documentIndex);
} }
indexSettings.LastIndexedId = item.TaskId; indexSettings.LastIndexedId = item.TaskId;
}
catch (Exception ex) {
Logger.Warning(ex, "Unable to index content item #{0} during update", item.Id);
}
} }
catch (Exception ex) {
Logger.Warning(ex, "Unable to index content item #{0} during update", item.Id); if (indexingTasks.Length < ContentItemsPerLoop) {
loop = false;
}
else {
_contentManager.Clear();
_transactionManager.RequireNew();
} }
} }
while (loop);
} }
// save current state of the index // save current state of the index