lsm/Orchard
1
0
Fork 0
mirror of https://github.com/OrchardCMS/Orchard.git synced 2025-10-08 00:14:31 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Using IProcessEngine to schedule batched indexing; Modifying

Browse Source 
IIndexingTaskExecutor to handle index deletiong in order to prevent
concurrency issues.

--HG--
branch : indexing
This commit is contained in:
Sebastien Ros
2011-03-05 10:47:20 -08:00
parent 599d892f48
commit 7e16aebada
10 changed files with 136 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs
View File

@@ -34,7 +34,7 @@ namespace Orchard.Tests.Modules.Indexing {
private IIndexProvider _provider;
private IAppDataFolder _appDataFolder;
private ShellSettings _shellSettings;
private IIndexNotifierHandler _indexNotifier;
private IIndexingTaskExecutor _indexTaskExecutor;
private IContentManager _contentManager;
private Mock<IContentDefinitionManager> _contentDefinitionManager;
private StubLogger _logger;
@@ -43,7 +43,6 @@ namespace Orchard.Tests.Modules.Indexing {
private const string IndexName = "Search";
private readonly string _basePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
[TestFixtureTearDown]
public void Clean() {
if (Directory.Exists(_basePath)) {
@@ -61,7 +60,7 @@ namespace Orchard.Tests.Modules.Indexing {
builder.RegisterType<LuceneIndexProvider>().As<IIndexProvider>();
builder.RegisterInstance(_appDataFolder).As<IAppDataFolder>();
builder.RegisterType<IndexingTaskExecutor>().As<IIndexNotifierHandler>();
builder.RegisterType<IndexingTaskExecutor>().As<IIndexingTaskExecutor>();
builder.RegisterType<DefaultIndexManager>().As<IIndexManager>();
builder.RegisterType<IndexingTaskManager>().As<IIndexingTaskManager>();
builder.RegisterType<DefaultContentManager>().As<IContentManager>();
@@ -105,9 +104,9 @@ namespace Orchard.Tests.Modules.Indexing {
base.Init();
_lockFileManager = _container.Resolve<ILockFileManager>();
_provider = _container.Resolve<IIndexProvider>();
_indexNotifier = _container.Resolve<IIndexNotifierHandler>();
_indexTaskExecutor = _container.Resolve<IIndexingTaskExecutor>();
_contentManager = _container.Resolve<IContentManager>();
((IndexingTaskExecutor)_indexNotifier).Logger = _logger = new StubLogger();
((IndexingTaskExecutor)_indexTaskExecutor).Logger = _logger = new StubLogger();
var thingType = new ContentTypeDefinitionBuilder()
.Named(ThingDriver.ContentTypeName)
@@ -121,7 +120,7 @@ namespace Orchard.Tests.Modules.Indexing {
[Test]
public void IndexShouldBeEmptyWhenThereIsNoContent() {
_indexNotifier.UpdateIndex(IndexName);
while(_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0));
}
@@ -137,7 +136,7 @@ namespace Orchard.Tests.Modules.Indexing {
_contentManager.Create("alpha");
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0));
}
@@ -154,7 +153,7 @@ namespace Orchard.Tests.Modules.Indexing {
_contentManager.Create("alpha");
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0));
}
@@ -163,22 +162,22 @@ namespace Orchard.Tests.Modules.Indexing {
var content = _contentManager.Create<Thing>(ThingDriver.ContentTypeName);
content.Text = "Lorem ipsum";
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
}
[Test]
public void ShouldUpdateTheIndexWhenContentIsPublished() {
_contentManager.Create<Thing>(ThingDriver.ContentTypeName).Text = "Lorem ipsum";
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
// there should be nothing done
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
_contentManager.Create<Thing>(ThingDriver.ContentTypeName).Text = "Lorem ipsum";
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2));
}
@@ -187,13 +186,13 @@ namespace Orchard.Tests.Modules.Indexing {
ILockFile lockFile = null;
_lockFileManager.TryAcquireLock("Sites/My Site/Search.settings.xml.lock", ref lockFile);
using (lockFile) {
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_logger.LogEntries.Count, Is.EqualTo(1));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Index was requested but is already running"));
}
_logger.LogEntries.Clear();
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_logger.LogEntries, Has.None.Matches<LogEntry>(entry => entry.LogFormat == "Index was requested but is already running"));
}
@@ -201,18 +200,18 @@ namespace Orchard.Tests.Modules.Indexing {
public void ShouldUpdateTheIndexWhenContentIsUnPublished() {
_contentManager.Create<Thing>(ThingDriver.ContentTypeName).Text = "Lorem ipsum";
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
var content = _contentManager.Create<Thing>(ThingDriver.ContentTypeName);
content.Text = "Lorem ipsum";
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2));
_contentManager.Unpublish(content.ContentItem);
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
}
@@ -223,7 +222,7 @@ namespace Orchard.Tests.Modules.Indexing {
var content = _contentManager.Create<Thing>(ThingDriver.ContentTypeName);
content.Text = "Lorem ipsum " + i;
}
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(999));
}

1
src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs
View File

@@ -43,7 +43,6 @@ namespace Orchard.Indexing.Commands {
[CommandHelp("index rebuild \r\n\t" + "Rebuilds the search index")]
public string Rebuild() {
_indexingService.RebuildIndex(SearchIndexName);
_indexingService.UpdateIndex(SearchIndexName);
return T("Index is now being rebuilt...").Text;
}

1
src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs
View File

@@ -44,7 +44,6 @@ namespace Orchard.Indexing.Controllers {
return new HttpUnauthorizedResult();
_indexingService.RebuildIndex(DefaultIndexName);
_indexingService.UpdateIndex(DefaultIndexName);
return RedirectToAction("Index");
}

4
src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs
View File

@@ -36,7 +36,7 @@ namespace Orchard.Indexing.Models
Mode = (IndexingMode) Enum.Parse(typeof (IndexingMode), doc.Descendants(TagMode).First().Value),
LastIndexedId = Int32.Parse(doc.Descendants(TagLastIndexedId).First().Value),
LastContentId = Int32.Parse(doc.Descendants(TagLastContentId).First().Value),
LastIndexedUtc = DateTime.Parse(doc.Descendants(TagLastIndexedUtc).First().Value)
LastIndexedUtc = DateTime.Parse(doc.Descendants(TagLastIndexedUtc).First().Value).ToUniversalTime()
};
}
catch {
@@ -44,7 +44,7 @@ namespace Orchard.Indexing.Models
}
}
public override string ToString() {
public string ToXml() {
return new XDocument(
new XElement(TagSettings,
new XElement(TagMode, Mode),

3
src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj
View File

@@ -60,6 +60,9 @@
<Compile Include="Models\IndexSettings.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="Services\UpdateIndexScheduler.cs" />
<Compile Include="Services\IIndexingTaskExecutor.cs" />
<Compile Include="Services\IUpdateIndexScheduler.cs" />
<Compile Include="Services\IIndexStatisticsProvider.cs" />
<Compile Include="Services\IndexServiceNotificationProvider.cs" />
<Compile Include="Services\IndexingBackgroundTask.cs" />

6
src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexingTaskExecutor.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace Orchard.Indexing.Services {
public interface IIndexingTaskExecutor : IDependency {
bool DeleteIndex(string indexName);
bool UpdateIndexBatch(string indexName);
}
}

5
src/Orchard.Web/Modules/Orchard.Indexing/Services/IUpdateIndexScheduler.cs Normal file
View File

@@ -0,0 +1,5 @@
namespace Orchard.Indexing.Services {
public interface IUpdateIndexScheduler : IDependency {
void Schedule(string indexName);
}
}

26
src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs
View File

@@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using Orchard.Localization;
using Orchard.UI.Notify;
@@ -9,38 +8,41 @@ namespace Orchard.Indexing.Services
private readonly IIndexManager _indexManager;
private readonly IEnumerable<IIndexNotifierHandler> _indexNotifierHandlers;
private readonly IIndexStatisticsProvider _indexStatisticsProvider;
private readonly IIndexingTaskExecutor _indexingTaskExecutor;
public IndexingService(
IOrchardServices services,
IIndexManager indexManager,
IEnumerable<IIndexNotifierHandler> indexNotifierHandlers,
IIndexStatisticsProvider indexStatisticsProvider) {
IIndexStatisticsProvider indexStatisticsProvider,
IIndexingTaskExecutor indexingTaskExecutor) {
Services = services;
_indexManager = indexManager;
_indexNotifierHandlers = indexNotifierHandlers;
_indexStatisticsProvider = indexStatisticsProvider;
_indexingTaskExecutor = indexingTaskExecutor;
T = NullLocalizer.Instance;
}
public IOrchardServices Services { get; set; }
public Localizer T { get; set; }
void IIndexingService.RebuildIndex(string indexName) {
public void RebuildIndex(string indexName) {
if (!_indexManager.HasIndexProvider()) {
Services.Notifier.Warning(T("There is no search index to rebuild."));
return;
}
var searchProvider = _indexManager.GetSearchIndexProvider();
if (searchProvider.Exists(indexName))
searchProvider.DeleteIndex(indexName);
searchProvider.CreateIndex(indexName); // or just reset the updated date and let the background process recreate the index
Services.Notifier.Information(T("The index {0} has been rebuilt.", indexName));
if(_indexingTaskExecutor.DeleteIndex(indexName)) {
Services.Notifier.Information(T("The index {0} has been rebuilt.", indexName));
UpdateIndex(indexName);
}
else {
Services.Notifier.Warning(T("The index {0} could no ben rebuilt. It might already be in use, please try again later.", indexName));
}
}
void IIndexingService.UpdateIndex(string indexName) {
public void UpdateIndex(string indexName) {
foreach(var handler in _indexNotifierHandlers) {
handler.UpdateIndex(indexName);

57
src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs
View File

@@ -22,7 +22,8 @@ namespace Orchard.Indexing.Services {
/// and singleton locks would not be shared accross those two.
/// </remarks>
[UsedImplicitly]
public class IndexingTaskExecutor : IIndexNotifierHandler, IIndexStatisticsProvider {
public class IndexingTaskExecutor : IIndexingTaskExecutor, IIndexStatisticsProvider
{
private readonly IRepository<IndexingTaskRecord> _taskRepository;
private readonly IRepository<ContentItemVersionRecord> _contentRepository;
private IIndexProvider _indexProvider;
@@ -32,7 +33,7 @@ namespace Orchard.Indexing.Services {
private readonly ShellSettings _shellSettings;
private readonly ILockFileManager _lockFileManager;
private readonly IClock _clock;
private const int ContentItemsPerLoop = 100;
private const int ContentItemsPerLoop = 50;
private IndexingStatus _indexingStatus = IndexingStatus.Idle;
public IndexingTaskExecutor(
@@ -57,9 +58,31 @@ namespace Orchard.Indexing.Services {
public ILogger Logger { get; set; }
public void UpdateIndex(string indexName) {
// What to do here to run next batch in a separate transaction
while (UpdateIndexBatch(indexName)) {}
public bool DeleteIndex(string indexName) {
ILockFile lockFile = null;
var settingsFilename = GetSettingsFileName(indexName);
var lockFilename = settingsFilename + ".lock";
// acquire a lock file on the index
if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile)) {
Logger.Information("Could not delete the index. Already in use.");
return false;
}
using (lockFile) {
if (!_indexManager.HasIndexProvider()) {
return false;
}
var searchProvider = _indexManager.GetSearchIndexProvider();
if (searchProvider.Exists(indexName)) {
searchProvider.DeleteIndex(indexName);
}
DeleteSettings(indexName);
}
return true;
}
public bool UpdateIndexBatch(string indexName) {
@@ -83,10 +106,8 @@ namespace Orchard.Indexing.Services {
_indexProvider = _indexManager.GetSearchIndexProvider();
// should the index be rebuilt
if (!_indexProvider.Exists(indexName)) {
if (indexSettings.Mode == IndexingMode.Rebuild && indexSettings.LastContentId == 0) {
_indexProvider.CreateIndex(indexName);
indexSettings = new IndexSettings();
// mark the last available task at the moment the process is started.
// once the Rebuild is done, Update will start at this point of the table
@@ -120,7 +141,7 @@ namespace Orchard.Indexing.Services {
// load all content items
var contentItems = _contentRepository
.Fetch(
versionRecord => versionRecord.Published && versionRecord.ContentItemRecord.Id > indexSettings.LastContentId,
versionRecord => versionRecord.Published && versionRecord.Id > indexSettings.LastContentId,
order => order.Asc(versionRecord => versionRecord.Id))
.Take(ContentItemsPerLoop)
.Select(versionRecord => _contentManager.Get(versionRecord.ContentItemRecord.Id, VersionOptions.VersionRecord(versionRecord.Id)))
@@ -182,7 +203,7 @@ namespace Orchard.Indexing.Services {
// save current state of the index
indexSettings.LastIndexedUtc = _clock.UtcNow;
_appDataFolder.CreateFile(settingsFilename, indexSettings.ToString());
_appDataFolder.CreateFile(settingsFilename, indexSettings.ToXml());
if (deleteFromIndex.Count == 0 && addToIndex.Count == 0) {
// nothing more to do
@@ -218,10 +239,12 @@ namespace Orchard.Indexing.Services {
/// <summary>
/// Loads the settings file or create a new default one if it doesn't exist
/// </summary>
public IndexSettings LoadSettings(string indexName) {
public IndexSettings LoadSettings(string indexName)
{
var indexSettings = new IndexSettings();
var settingsFilename = GetSettingsFileName(indexName);
if (_appDataFolder.FileExists(settingsFilename)) {
if (_appDataFolder.FileExists(settingsFilename))
{
var content = _appDataFolder.ReadFile(settingsFilename);
indexSettings = IndexSettings.Parse(content);
}
@@ -229,6 +252,16 @@ namespace Orchard.Indexing.Services {
return indexSettings;
}
/// <summary>
/// Deletes the settings file
/// </summary>
public void DeleteSettings(string indexName) {
var settingsFilename = GetSettingsFileName(indexName);
if (_appDataFolder.FileExists(settingsFilename)) {
_appDataFolder.DeleteFile(settingsFilename);
}
}
/// <summary>
/// Creates a IDocumentIndex instance for a specific content item id. If the content
/// item is no more published, it returns null.

42
src/Orchard.Web/Modules/Orchard.Indexing/Services/UpdateIndexScheduler.cs Normal file
View File

@@ -0,0 +1,42 @@
using System;
using System.Collections.Generic;
using Orchard.Environment.Configuration;
using Orchard.Environment.Descriptor;
using Orchard.Environment.State;
namespace Orchard.Indexing.Services {
public class UpdateIndexScheduler : IUpdateIndexScheduler, IIndexNotifierHandler {
private readonly IProcessingEngine _processingEngine;
private readonly ShellSettings _shellSettings;
private readonly IShellDescriptorManager _shellDescriptorManager;
private readonly Lazy<IIndexingTaskExecutor> _indexingTaskExecutor;
public UpdateIndexScheduler(
IProcessingEngine processingEngine,
ShellSettings shellSettings,
IShellDescriptorManager shellDescriptorManager,
Lazy<IIndexingTaskExecutor> indexingTaskExecutor
) {
_processingEngine = processingEngine;
_shellSettings = shellSettings;
_shellDescriptorManager = shellDescriptorManager;
_indexingTaskExecutor = indexingTaskExecutor;
}
public void Schedule(string indexName) {
var shellDescriptor = _shellDescriptorManager.GetShellDescriptor();
_processingEngine.AddTask(
_shellSettings,
shellDescriptor,
"IIndexNotifierHandler.UpdateIndex",
new Dictionary<string, object> { { "indexName", indexName } }
);
}
public void UpdateIndex(string indexName) {
if(_indexingTaskExecutor.Value.UpdateIndexBatch(indexName)) {
Schedule(indexName);
}
}
}
}