lsm/Orchard
1
0
Fork 0
mirror of https://github.com/OrchardCMS/Orchard.git synced 2025-10-15 19:54:57 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Merge (Indexing)

Browse Source 
--HG--
branch : dev
This commit is contained in:
Sebastien Ros
2011-03-14 16:24:38 -07:00
parent 82b2ce4f7d 7e16aebada
commit a8123b4b88
27 changed files with 885 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs
View File

@@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Autofac;
using Lucene.Services;
using Moq;
@@ -19,6 +18,7 @@ using Orchard.Environment;
using Orchard.Environment.Configuration;
using Orchard.Environment.Extensions;
using Orchard.FileSystems.AppData;
using Orchard.FileSystems.LockFile;
using Orchard.Indexing;
using Orchard.Indexing.Handlers;
using Orchard.Indexing.Models;
@@ -34,14 +34,15 @@ namespace Orchard.Tests.Modules.Indexing {
private IIndexProvider _provider;
private IAppDataFolder _appDataFolder;
private ShellSettings _shellSettings;
private IIndexNotifierHandler _indexNotifier;
private IIndexingTaskExecutor _indexTaskExecutor;
private IContentManager _contentManager;
private Mock<IContentDefinitionManager> _contentDefinitionManager;
private StubLogger _logger;
private const string IndexName = "Search";
private ILockFileManager _lockFileManager;
private const string IndexName = "Search";
private readonly string _basePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
[TestFixtureTearDown]
public void Clean() {
if (Directory.Exists(_basePath)) {
@@ -59,10 +60,9 @@ namespace Orchard.Tests.Modules.Indexing {
builder.RegisterType<LuceneIndexProvider>().As<IIndexProvider>();
builder.RegisterInstance(_appDataFolder).As<IAppDataFolder>();
builder.RegisterType<IndexingTaskExecutor>().As<IIndexNotifierHandler>();
builder.RegisterType<IndexingTaskExecutor>().As<IIndexingTaskExecutor>();
builder.RegisterType<DefaultIndexManager>().As<IIndexManager>();
builder.RegisterType<IndexingTaskManager>().As<IIndexingTaskManager>();
builder.RegisterType<IndexSynLock>().As<IIndexSynLock>();
builder.RegisterType<DefaultContentManager>().As<IContentManager>();
builder.RegisterType<DefaultContentManagerSession>().As<IContentManagerSession>();
builder.RegisterInstance(_contentDefinitionManager.Object);
@@ -80,6 +80,8 @@ namespace Orchard.Tests.Modules.Indexing {
builder.RegisterType<BodyPartHandler>().As<IContentHandler>();
builder.RegisterType<StubExtensionManager>().As<IExtensionManager>();
builder.RegisterType<DefaultLockFileManager>().As<ILockFileManager>();
// setting up a ShellSettings instance
_shellSettings = new ShellSettings { Name = "My Site" };
builder.RegisterInstance(_shellSettings).As<ShellSettings>();
@@ -100,11 +102,11 @@ namespace Orchard.Tests.Modules.Indexing {
public override void Init() {
base.Init();
_lockFileManager = _container.Resolve<ILockFileManager>();
_provider = _container.Resolve<IIndexProvider>();
_indexNotifier = _container.Resolve<IIndexNotifierHandler>();
_indexTaskExecutor = _container.Resolve<IIndexingTaskExecutor>();
_contentManager = _container.Resolve<IContentManager>();
((IndexingTaskExecutor)_indexNotifier).Logger = _logger = new StubLogger();
((IndexingTaskExecutor)_indexTaskExecutor).Logger = _logger = new StubLogger();
var thingType = new ContentTypeDefinitionBuilder()
.Named(ThingDriver.ContentTypeName)
@@ -116,21 +118,14 @@ namespace Orchard.Tests.Modules.Indexing {
.Returns(thingType);
}
private string[] Indexes() {
return new DirectoryInfo(Path.Combine(_basePath, "Sites", "My Site", "Indexes")).GetDirectories().Select(d => d.Name).ToArray();
}
[Test]
public void IndexShouldBeEmptyWhenThereIsNoContent() {
_indexNotifier.UpdateIndex(IndexName);
while(_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0));
Assert.That(_logger.LogEntries.Count(), Is.EqualTo(2));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Rebuild index started"));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Index update requested, nothing to do"));
}
[Test]
public void ShouldIngoreNonIndexableContentWhenRebuildingTheIndex() {
public void ShouldIgnoreNonIndexableContentWhenRebuildingTheIndex() {
var alphaType = new ContentTypeDefinitionBuilder()
.Named("alpha")
.Build();
@@ -141,11 +136,8 @@ namespace Orchard.Tests.Modules.Indexing {
_contentManager.Create("alpha");
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0));
Assert.That(_logger.LogEntries.Count(), Is.EqualTo(2));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Rebuild index started"));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Index update requested, nothing to do"));
}
[Test]
@@ -161,11 +153,8 @@ namespace Orchard.Tests.Modules.Indexing {
_contentManager.Create("alpha");
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0));
Assert.That(_logger.LogEntries.Count(), Is.EqualTo(2));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Rebuild index started"));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Index update requested, nothing to do"));
}
[Test]
@@ -173,80 +162,68 @@ namespace Orchard.Tests.Modules.Indexing {
var content = _contentManager.Create<Thing>(ThingDriver.ContentTypeName);
content.Text = "Lorem ipsum";
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
Assert.That(_logger.LogEntries.Count(), Is.EqualTo(3));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Rebuild index started"));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Processing {0} indexing tasks"));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Added content items to index: {0}"));
}
[Test]
public void ShouldUpdateTheIndexWhenContentIsPublished() {
_contentManager.Create<Thing>(ThingDriver.ContentTypeName).Text = "Lorem ipsum";
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
// there should be nothing done
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Rebuild index started"));
_logger.Clear();
_contentManager.Create<Thing>(ThingDriver.ContentTypeName).Text = "Lorem ipsum";
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2));
Assert.That(_logger.LogEntries, Has.None.Matches<LogEntry>(entry => entry.LogFormat == "Rebuild index started"));
}
[Test]
public void IndexingTaskExecutorShouldNotBeReEntrant() {
ILockFile lockFile = null;
_lockFileManager.TryAcquireLock("Sites/My Site/Search.settings.xml.lock", ref lockFile);
using (lockFile) {
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_logger.LogEntries.Count, Is.EqualTo(1));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Index was requested but is already running"));
}
_logger.LogEntries.Clear();
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_logger.LogEntries, Has.None.Matches<LogEntry>(entry => entry.LogFormat == "Index was requested but is already running"));
}
[Test]
public void ShouldUpdateTheIndexWhenContentIsUnPublished() {
_contentManager.Create<Thing>(ThingDriver.ContentTypeName).Text = "Lorem ipsum";
_clock.Advance(TimeSpan.FromSeconds(1));
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Rebuild index started"));
_logger.Clear();
var content = _contentManager.Create<Thing>(ThingDriver.ContentTypeName);
content.Text = "Lorem ipsum";
_clock.Advance(TimeSpan.FromSeconds(1));
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2));
Assert.That(_logger.LogEntries, Has.None.Matches<LogEntry>(entry => entry.LogFormat == "Rebuild index started"));
_clock.Advance(TimeSpan.FromSeconds(1));
_contentManager.Unpublish(content.ContentItem);
_clock.Advance(TimeSpan.FromSeconds(1));
_indexNotifier.UpdateIndex(IndexName);
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
Assert.That(_logger.LogEntries, Has.None.Matches<LogEntry>(entry => entry.LogFormat == "Rebuild index started"));
}
[Test]
public void ShouldRemoveFromIndexEvenIfPublishedAndUnpublishedInTheSameSecond() {
// This test is to ensure that when a task is created, all previous tasks for the same content item
// are also removed, and thus that multiple tasks don't conflict while updating the index
_contentManager.Create<Thing>(ThingDriver.ContentTypeName).Text = "Lorem ipsum";
_clock.Advance(TimeSpan.FromSeconds(1));
_indexNotifier.UpdateIndex(IndexName);
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
Assert.That(_logger.LogEntries, Has.Some.Matches<LogEntry>(entry => entry.LogFormat == "Rebuild index started"));
_logger.Clear();
var content = _contentManager.Create<Thing>(ThingDriver.ContentTypeName);
content.Text = "Lorem ipsum";
_indexNotifier.UpdateIndex(IndexName);
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2));
Assert.That(_logger.LogEntries, Has.None.Matches<LogEntry>(entry => entry.LogFormat == "Rebuild index started"));
_contentManager.Unpublish(content.ContentItem);
_indexNotifier.UpdateIndex(IndexName);
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1));
Assert.That(_logger.LogEntries, Has.None.Matches<LogEntry>(entry => entry.LogFormat == "Rebuild index started"));
public void ShouldIndexAllContentOverTheLoopSize() {
for (int i = 0; i < 999; i++) {
var content = _contentManager.Create<Thing>(ThingDriver.ContentTypeName);
content.Text = "Lorem ipsum " + i;
}
while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {}
Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(999));
}
#region Stubs
@@ -293,7 +270,7 @@ namespace Orchard.Tests.Modules.Indexing {
}
public void Log(LogLevel level, Exception exception, string format, params object[] args) {
LogEntries.Add(new LogEntry() {
LogEntries.Add(new LogEntry {
LogArgs = args,
LogException = exception,
LogFormat = format,

19
src/Orchard.Tests.Modules/Indexing/LuceneIndexProviderTests.cs
View File

@@ -7,7 +7,6 @@ using NUnit.Framework;
using Orchard.Environment.Configuration;
using Orchard.FileSystems.AppData;
using Orchard.Indexing;
using Orchard.Indexing.Services;
using Orchard.Tests.FileSystems.AppData;
namespace Orchard.Tests.Modules.Indexing {
@@ -198,18 +197,6 @@ namespace Orchard.Tests.Modules.Indexing {
Assert.That(searchBuilder.Get(3).ContentItemId, Is.EqualTo(3));
}
[Test]
public void ProviderShouldStoreSettings() {
_provider.CreateIndex("default");
Assert.That(_provider.GetLastIndexUtc("default"), Is.Null);
_provider.SetLastIndexUtc("default", new DateTime(2010, 1, 1, 1, 1, 1, 1));
Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(new DateTime(2010, 1, 1, 1, 1, 1, 0)));
_provider.SetLastIndexUtc("default", new DateTime(1901, 1, 1, 1, 1, 1, 1));
Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(LuceneIndexProvider.DefaultMinDateTime));
}
[Test]
public void IsEmptyShouldBeTrueForNoneExistingIndexes() {
_provider.IsEmpty("dummy");
@@ -238,9 +225,7 @@ namespace Orchard.Tests.Modules.Indexing {
[Test]
public void IsDirtyShouldBeTrueWhenIndexIsModified() {
IDocumentIndex doc;
doc = _provider.New(1);
IDocumentIndex doc = _provider.New(1);
doc.Add("foo", "value");
Assert.That(doc.IsDirty, Is.True);
@@ -281,7 +266,7 @@ namespace Orchard.Tests.Modules.Indexing {
Assert.That(searchBuilder.Get(11).ContentItemId, Is.EqualTo(11));
Assert.That(searchBuilder.Get(111).ContentItemId, Is.EqualTo(111));
_provider.Delete("default", new int[] {1, 11, 111 });
_provider.Delete("default", new [] {1, 11, 111 });
Assert.That(searchBuilder.Get(1), Is.Null);
Assert.That(searchBuilder.Get(11), Is.Null);

223
src/Orchard.Tests/FileSystems/LockFile/LockFileManagerTests.cs Normal file
View File

@@ -0,0 +1,223 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using NUnit.Framework;
using Orchard.FileSystems.AppData;
using Orchard.FileSystems.LockFile;
using Orchard.Tests.Stubs;
namespace Orchard.Tests.FileSystems.LockFile {
[TestFixture]
public class LockFileManagerTests {
private string _tempFolder;
private IAppDataFolder _appDataFolder;
private ILockFileManager _lockFileManager;
private StubClock _clock;
public class StubAppDataFolderRoot : IAppDataFolderRoot {
public string RootPath { get; set; }
public string RootFolder { get; set; }
}
public static IAppDataFolder CreateAppDataFolder(string tempFolder) {
var folderRoot = new StubAppDataFolderRoot {RootPath = "~/App_Data", RootFolder = tempFolder};
var monitor = new StubVirtualPathMonitor();
return new AppDataFolder(folderRoot, monitor);
}
[SetUp]
public void Init() {
_tempFolder = Path.GetTempFileName();
File.Delete(_tempFolder);
_appDataFolder = CreateAppDataFolder(_tempFolder);
_clock = new StubClock();
_lockFileManager = new DefaultLockFileManager(_appDataFolder, _clock);
}
[TearDown]
public void Term() {
Directory.Delete(_tempFolder, true);
}
[Test]
public void LockShouldBeGrantedWhenDoesNotExist() {
ILockFile lockFile = null;
var granted = _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile);
Assert.That(granted, Is.True);
Assert.That(lockFile, Is.Not.Null);
Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True);
Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1));
}
[Test]
public void ExistingLockFileShouldPreventGrants() {
ILockFile lockFile = null;
_lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile);
Assert.That(_lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile), Is.False);
Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True);
Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1));
}
[Test]
public void ReleasingALockShouldAllowGranting() {
ILockFile lockFile = null;
_lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile);
using (lockFile) {
Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True);
Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1));
}
Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False);
Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0));
}
[Test]
public void ReleasingAReleasedLockShouldWork() {
ILockFile lockFile = null;
_lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile);
Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True);
Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1));
lockFile.Release();
Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False);
Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0));
lockFile.Release();
Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False);
Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0));
}
[Test]
public void DisposingLockShouldReleaseIt() {
ILockFile lockFile = null;
_lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile);
using (lockFile) {
Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True);
Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1));
}
Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False);
Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0));
}
[Test]
public void ExpiredLockShouldBeAvailable() {
ILockFile lockFile = null;
_lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile);
_clock.Advance(DefaultLockFileManager.Expiration);
Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False);
Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1));
}
[Test]
public void ShouldGrantExpiredLock() {
ILockFile lockFile = null;
_lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile);
_clock.Advance(DefaultLockFileManager.Expiration);
var granted = _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile);
Assert.That(granted, Is.True);
Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1));
}
private static int _lockCount;
private static readonly object _synLock = new object();
[Test]
public void AcquiringLockShouldBeThreadSafe() {
var threads = new List<Thread>();
for(var i=0; i<10; i++) {
var t = new Thread(PlayWithAcquire);
t.Start();
threads.Add(t);
}
threads.ForEach(t => t.Join());
Assert.That(_lockCount, Is.EqualTo(0));
}
[Test]
public void IsLockedShouldBeThreadSafe() {
var threads = new List<Thread>();
for (var i = 0; i < 10; i++)
{
var t = new Thread(PlayWithIsLocked);
t.Start();
threads.Add(t);
}
threads.ForEach(t => t.Join());
Assert.That(_lockCount, Is.EqualTo(0));
}
private void PlayWithAcquire() {
var r = new Random(DateTime.Now.Millisecond);
ILockFile lockFile = null;
// loop until the lock has been acquired
for (;;) {
if (!_lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile)) {
continue;
}
lock (_synLock) {
_lockCount++;
Assert.That(_lockCount, Is.EqualTo(1));
}
// keep the lock for a certain time
Thread.Sleep(r.Next(200));
lock (_synLock) {
_lockCount--;
Assert.That(_lockCount, Is.EqualTo(0));
}
lockFile.Release();
return;
}
}
private void PlayWithIsLocked() {
var r = new Random(DateTime.Now.Millisecond);
ILockFile lockFile = null;
const string path = "foo.txt.lock";
// loop until the lock has been acquired
for (;;) {
if(_lockFileManager.IsLocked(path)) {
continue;
}
if (!_lockFileManager.TryAcquireLock(path, ref lockFile)) {
continue;
}
lock (_synLock) {
_lockCount++;
Assert.That(_lockCount, Is.EqualTo(1));
}
// keep the lock for a certain time
Thread.Sleep(r.Next(200));
lock (_synLock) {
_lockCount--;
Assert.That(_lockCount, Is.EqualTo(0));
}
lockFile.Release();
return;
}
}
}
}

1
src/Orchard.Tests/Orchard.Framework.Tests.csproj
View File

@@ -247,6 +247,7 @@
<Compile Include="Environment\OrchardStarterTests.cs" />
<Compile Include="Environment\ShellBuilders\DefaultShellContainerFactoryTests.cs" />
<Compile Include="Environment\ShellBuilders\DefaultShellContextFactoryTests.cs" />
<Compile Include="FileSystems\LockFile\LockFileManagerTests.cs" />
<Compile Include="FileSystems\Dependencies\AssemblyProbingFolderTests.cs" />
<Compile Include="FileSystems\Dependencies\DependenciesFolderTests.cs" />
<Compile Include="Localization\CultureManagerTests.cs" />

42
src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs
View File

@@ -9,7 +9,6 @@ using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.Search;
using Lucene.Net.Store;
using System.Xml.Linq;
using Orchard.Environment.Configuration;
using Orchard.FileSystems.AppData;
using Orchard.Indexing;
@@ -29,8 +28,6 @@ namespace Lucene.Services {
private readonly Analyzer _analyzer ;
private readonly string _basePath;
public static readonly DateTime DefaultMinDateTime = new DateTime(1980, 1, 1);
public static readonly string Settings = "Settings";
public static readonly string LastIndexUtc = "LastIndexedUtc";
public LuceneIndexProvider(IAppDataFolder appDataFolder, ShellSettings shellSettings) {
_appDataFolder = appDataFolder;
@@ -123,11 +120,6 @@ namespace Lucene.Services {
public void DeleteIndex(string indexName) {
new DirectoryInfo(_appDataFolder.MapPath(_appDataFolder.Combine(_basePath, indexName)))
.Delete(true);
var settingsFileName = GetSettingsFileName(indexName);
if (File.Exists(settingsFileName)) {
File.Delete(settingsFileName);
}
}
public void Store(string indexName, IDocumentIndex indexDocument) {
@@ -206,40 +198,6 @@ namespace Lucene.Services {
return new LuceneSearchBuilder(GetDirectory(indexName)) { Logger = Logger };
}
private string GetSettingsFileName(string indexName) {
return _appDataFolder.MapPath(_appDataFolder.Combine(_basePath, indexName + ".settings.xml"));
}
public DateTime? GetLastIndexUtc(string indexName) {
var settingsFileName = GetSettingsFileName(indexName);
if (!File.Exists(settingsFileName))
return null;
return DateTime.Parse(XDocument.Load(settingsFileName).Descendants(LastIndexUtc).First().Value);
}
public void SetLastIndexUtc(string indexName, DateTime lastIndexUtc) {
if ( lastIndexUtc < DefaultMinDateTime ) {
lastIndexUtc = DefaultMinDateTime;
}
XDocument doc;
var settingsFileName = GetSettingsFileName(indexName);
if ( !File.Exists(settingsFileName) ) {
EnsureDirectoryExists();
doc = new XDocument(
new XElement(Settings,
new XElement(LastIndexUtc, lastIndexUtc.ToString("s"))));
}
else {
doc = XDocument.Load(settingsFileName);
doc.Element(Settings).Element(LastIndexUtc).Value = lastIndexUtc.ToString("s");
}
doc.Save(settingsFileName);
}
public IEnumerable<string> GetFields(string indexName) {
if ( !Exists(indexName) ) {
return Enumerable.Empty<string>();

6
src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs
View File

@@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Orchard.Commands;
using Orchard.ContentManagement;
@@ -35,7 +34,7 @@ namespace Orchard.Indexing.Commands {
[CommandName("index update")]
[CommandHelp("index update\r\n\t" + "Updates the search index")]
public string Update() {
_indexingService.UpdateIndex();
_indexingService.UpdateIndex(SearchIndexName);
return T("Index is now being updated...").Text;
}
@@ -43,8 +42,7 @@ namespace Orchard.Indexing.Commands {
[CommandName("index rebuild")]
[CommandHelp("index rebuild \r\n\t" + "Rebuilds the search index")]
public string Rebuild() {
_indexingService.RebuildIndex();
_indexingService.UpdateIndex();
_indexingService.RebuildIndex(SearchIndexName);
return T("Index is now being rebuilt...").Text;
}

8
src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs
View File

@@ -8,6 +8,7 @@ using Orchard.Indexing.ViewModels;
namespace Orchard.Indexing.Controllers {
public class AdminController : Controller {
private readonly IIndexingService _indexingService;
private const string DefaultIndexName = "Search";
public AdminController(IIndexingService indexingService, IOrchardServices services) {
_indexingService = indexingService;
@@ -19,7 +20,7 @@ namespace Orchard.Indexing.Controllers {
public Localizer T { get; set; }
public ActionResult Index() {
var viewModel = new IndexViewModel { IndexEntry = _indexingService.GetIndexEntry() };
var viewModel = new IndexViewModel { IndexEntry = _indexingService.GetIndexEntry(DefaultIndexName) };
if (viewModel.IndexEntry == null)
Services.Notifier.Information(T("There is no search index to manage for this site."));
@@ -32,7 +33,7 @@ namespace Orchard.Indexing.Controllers {
if (!Services.Authorizer.Authorize(StandardPermissions.SiteOwner, T("Not allowed to manage the search index.")))
return new HttpUnauthorizedResult();
_indexingService.UpdateIndex();
_indexingService.UpdateIndex(DefaultIndexName);
return RedirectToAction("Index");
}
@@ -42,8 +43,7 @@ namespace Orchard.Indexing.Controllers {
if (!Services.Authorizer.Authorize(StandardPermissions.SiteOwner, T("Not allowed to manage the search index.")))
return new HttpUnauthorizedResult();
_indexingService.RebuildIndex();
_indexingService.UpdateIndex();
_indexingService.RebuildIndex(DefaultIndexName);
return RedirectToAction("Index");
}

57
src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs Normal file
View File

@@ -0,0 +1,57 @@
using System;
using System.Linq;
using System.Xml.Linq;
namespace Orchard.Indexing.Models
{
public enum IndexingMode {
Rebuild,
Update
}
public class IndexSettings {
public IndexingMode Mode { get; set; }
public int LastIndexedId { get; set; }
public int LastContentId { get; set; }
public DateTime LastIndexedUtc { get; set; }
public static readonly string TagSettings = "Settings";
public static readonly string TagMode = "Mode";
public static readonly string TagLastIndexedId = "LastIndexedId";
public static readonly string TagLastContentId = "LastContentId";
public static readonly string TagLastIndexedUtc = "LastIndexedUtc";
public IndexSettings() {
Mode = IndexingMode.Rebuild;
LastIndexedId = 0;
LastContentId = 0;
LastIndexedUtc = DateTime.MinValue;
}
public static IndexSettings Parse(string content) {
var doc = XDocument.Parse(content);
try {
return new IndexSettings {
Mode = (IndexingMode) Enum.Parse(typeof (IndexingMode), doc.Descendants(TagMode).First().Value),
LastIndexedId = Int32.Parse(doc.Descendants(TagLastIndexedId).First().Value),
LastContentId = Int32.Parse(doc.Descendants(TagLastContentId).First().Value),
LastIndexedUtc = DateTime.Parse(doc.Descendants(TagLastIndexedUtc).First().Value).ToUniversalTime()
};
}
catch {
return new IndexSettings();
}
}
public string ToXml() {
return new XDocument(
new XElement(TagSettings,
new XElement(TagMode, Mode),
new XElement(TagLastIndexedId, LastIndexedId),
new XElement(TagLastContentId, LastContentId),
new XElement(TagLastIndexedUtc, LastIndexedUtc.ToString("u"))
)).ToString();
}
}
}

10
src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj
View File

@@ -41,6 +41,8 @@
<HintPath>..\..\..\..\lib\aspnetmvc\System.Web.Mvc.dll</HintPath>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.XML" />
<Reference Include="System.Xml.Linq" />
</ItemGroup>
<ItemGroup>
<Content Include="Module.txt" />
@@ -55,12 +57,18 @@
<Compile Include="Handlers\InfosetFieldIndexingHandler.cs" />
<Compile Include="Models\IndexingTask.cs" />
<Compile Include="Models\IndexingTaskRecord.cs" />
<Compile Include="Models\IndexSettings.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="Services\UpdateIndexScheduler.cs" />
<Compile Include="Services\IIndexingTaskExecutor.cs" />
<Compile Include="Services\IUpdateIndexScheduler.cs" />
<Compile Include="Services\IIndexStatisticsProvider.cs" />
<Compile Include="Services\IndexServiceNotificationProvider.cs" />
<Compile Include="Services\IndexingBackgroundTask.cs" />
<Compile Include="Services\IndexingTaskExecutor.cs" />
<Compile Include="Services\IndexingTaskManager.cs" />
<Compile Include="Services\IIndexService.cs" />
<Compile Include="Services\IndexSynLock.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Services\IndexService.cs" />
<Compile Include="Settings\EditorEvents.cs" />

9
src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexService.cs
View File

@@ -5,13 +5,14 @@ namespace Orchard.Indexing.Services {
public class IndexEntry {
public string IndexName { get; set; }
public int DocumentCount { get; set; }
public DateTime? LastUpdateUtc { get; set; }
public DateTime LastUpdateUtc { get; set; }
public IEnumerable<string> Fields { get; set; }
public IndexingStatus IndexingStatus { get; set; }
}
public interface IIndexingService : IDependency {
void RebuildIndex();
void UpdateIndex();
IndexEntry GetIndexEntry();
void RebuildIndex(string indexName);
void UpdateIndex(string indexName);
IndexEntry GetIndexEntry(string indexName);
}
}

13
src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexStatisticsProvider.cs Normal file
View File

@@ -0,0 +1,13 @@
using System;
namespace Orchard.Indexing.Services {
public enum IndexingStatus {
Rebuilding,
Updating,
Idle
}
public interface IIndexStatisticsProvider : IDependency {
DateTime GetLastIndexedUtc(string indexName);
IndexingStatus GetIndexingStatus(string indexName);
}
}

6
src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexingTaskExecutor.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace Orchard.Indexing.Services {
public interface IIndexingTaskExecutor : IDependency {
bool DeleteIndex(string indexName);
bool UpdateIndexBatch(string indexName);
}
}

5
src/Orchard.Web/Modules/Orchard.Indexing/Services/IUpdateIndexScheduler.cs Normal file
View File

@@ -0,0 +1,5 @@
namespace Orchard.Indexing.Services {
public interface IUpdateIndexScheduler : IDependency {
void Schedule(string indexName);
}
}

44
src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs
View File

@@ -1,59 +1,67 @@
using System.Collections.Generic;
using Orchard.Localization;
using Orchard.Localization.Services;
using Orchard.UI.Notify;
namespace Orchard.Indexing.Services
{
public class IndexingService : IIndexingService {
private const string SearchIndexName = "Search";
private readonly IIndexManager _indexManager;
private readonly IEnumerable<IIndexNotifierHandler> _indexNotifierHandlers;
private readonly IIndexStatisticsProvider _indexStatisticsProvider;
private readonly IIndexingTaskExecutor _indexingTaskExecutor;
public IndexingService(IOrchardServices services, IIndexManager indexManager, IEnumerable<IIndexNotifierHandler> indexNotifierHandlers, ICultureManager cultureManager) {
public IndexingService(
IOrchardServices services,
IIndexManager indexManager,
IEnumerable<IIndexNotifierHandler> indexNotifierHandlers,
IIndexStatisticsProvider indexStatisticsProvider,
IIndexingTaskExecutor indexingTaskExecutor) {
Services = services;
_indexManager = indexManager;
_indexNotifierHandlers = indexNotifierHandlers;
_indexStatisticsProvider = indexStatisticsProvider;
_indexingTaskExecutor = indexingTaskExecutor;
T = NullLocalizer.Instance;
}
public IOrchardServices Services { get; set; }
public Localizer T { get; set; }
void IIndexingService.RebuildIndex() {
public void RebuildIndex(string indexName) {
if (!_indexManager.HasIndexProvider()) {
Services.Notifier.Warning(T("There is no search index to rebuild."));
return;
}
var searchProvider = _indexManager.GetSearchIndexProvider();
if (searchProvider.Exists(SearchIndexName))
searchProvider.DeleteIndex(SearchIndexName);
searchProvider.CreateIndex(SearchIndexName); // or just reset the updated date and let the background process recreate the index
Services.Notifier.Information(T("The search index has been rebuilt."));
if(_indexingTaskExecutor.DeleteIndex(indexName)) {
Services.Notifier.Information(T("The index {0} has been rebuilt.", indexName));
UpdateIndex(indexName);
}
else {
Services.Notifier.Warning(T("The index {0} could no ben rebuilt. It might already be in use, please try again later.", indexName));
}
}
void IIndexingService.UpdateIndex() {
public void UpdateIndex(string indexName) {
foreach(var handler in _indexNotifierHandlers) {
handler.UpdateIndex(SearchIndexName);
handler.UpdateIndex(indexName);
}
Services.Notifier.Information(T("The search index has been updated."));
}
IndexEntry IIndexingService.GetIndexEntry() {
IndexEntry IIndexingService.GetIndexEntry(string indexName) {
var provider = _indexManager.GetSearchIndexProvider();
if (provider == null)
return null;
return new IndexEntry {
IndexName = SearchIndexName,
DocumentCount = provider.NumDocs(SearchIndexName),
Fields = provider.GetFields(SearchIndexName),
LastUpdateUtc = provider.GetLastIndexUtc(SearchIndexName)
IndexName = indexName,
DocumentCount = provider.NumDocs(indexName),
Fields = provider.GetFields(indexName),
LastUpdateUtc = _indexStatisticsProvider.GetLastIndexedUtc(indexName),
IndexingStatus = _indexStatisticsProvider.GetIndexingStatus(indexName)
};
}
}

25
src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexSynLock.cs
View File

@@ -1,25 +0,0 @@
using System.Collections.Generic;
namespace Orchard.Indexing.Services {
public interface IIndexSynLock : ISingletonDependency {
object GetSynLock(string indexName);
}
public class IndexSynLock : IIndexSynLock {
private readonly Dictionary<string, object> _synLocks;
private readonly object _synLock = new object();
public IndexSynLock() {
_synLocks =new Dictionary<string, object>();
}
public object GetSynLock(string indexName) {
lock(_synLock) {
if(!_synLocks.ContainsKey(indexName)) {
_synLocks[indexName] = new object();
}
return _synLocks[indexName];
}
}
}
}

381
src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs
View File

@@ -3,173 +3,308 @@ using System.Collections.Generic;
using System.Linq;
using JetBrains.Annotations;
using Orchard.ContentManagement;
using Orchard.ContentManagement.Records;
using Orchard.Data;
using Orchard.Environment.Configuration;
using Orchard.FileSystems.AppData;
using Orchard.FileSystems.LockFile;
using Orchard.Indexing.Models;
using Orchard.Indexing.Settings;
using Orchard.Logging;
using Orchard.Services;
using Orchard.Tasks.Indexing;
namespace Orchard.Indexing.Services {
/// <summary>
/// Contains the logic which is regularly executed to retrieve index information from multiple content handlers.
/// </summary>
/// <remarks>
/// This class is synchronized using a lock file as both command line and web workers can potentially use it,
/// and singleton locks would not be shared accross those two.
/// </remarks>
[UsedImplicitly]
public class IndexingTaskExecutor : IIndexNotifierHandler {
private readonly IClock _clock;
private readonly IRepository<IndexingTaskRecord> _repository;
public class IndexingTaskExecutor : IIndexingTaskExecutor, IIndexStatisticsProvider
{
private readonly IRepository<IndexingTaskRecord> _taskRepository;
private readonly IRepository<ContentItemVersionRecord> _contentRepository;
private IIndexProvider _indexProvider;
private readonly IIndexManager _indexManager;
private readonly IIndexingTaskManager _indexingTaskManager;
private readonly IContentManager _contentManager;
private readonly IIndexSynLock _indexSynLock;
private readonly IAppDataFolder _appDataFolder;
private readonly ShellSettings _shellSettings;
private readonly ILockFileManager _lockFileManager;
private readonly IClock _clock;
private const int ContentItemsPerLoop = 50;
private IndexingStatus _indexingStatus = IndexingStatus.Idle;
public IndexingTaskExecutor(
IClock clock,
IRepository<IndexingTaskRecord> repository,
IRepository<IndexingTaskRecord> taskRepository,
IRepository<ContentItemVersionRecord> contentRepository,
IIndexManager indexManager,
IIndexingTaskManager indexingTaskManager,
IContentManager contentManager,
IIndexSynLock indexSynLock) {
_clock = clock;
_repository = repository;
IAppDataFolder appDataFolder,
ShellSettings shellSettings,
ILockFileManager lockFileManager,
IClock clock) {
_taskRepository = taskRepository;
_contentRepository = contentRepository;
_indexManager = indexManager;
_indexingTaskManager = indexingTaskManager;
_contentManager = contentManager;
_indexSynLock = indexSynLock;
_appDataFolder = appDataFolder;
_shellSettings = shellSettings;
_lockFileManager = lockFileManager;
_clock = clock;
Logger = NullLogger.Instance;
}
public ILogger Logger { get; set; }
public void UpdateIndex(string indexName) {
var synLock = _indexSynLock.GetSynLock(indexName);
public bool DeleteIndex(string indexName) {
ILockFile lockFile = null;
var settingsFilename = GetSettingsFileName(indexName);
var lockFilename = settingsFilename + ".lock";
if (!System.Threading.Monitor.TryEnter(synLock)) {
Logger.Information("Index was requested but was already running");
return;
// acquire a lock file on the index
if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile)) {
Logger.Information("Could not delete the index. Already in use.");
return false;
}
try {
using (lockFile) {
if (!_indexManager.HasIndexProvider()) {
return;
return false;
}
var searchProvider = _indexManager.GetSearchIndexProvider();
if (searchProvider.Exists(indexName)) {
searchProvider.DeleteIndex(indexName);
}
DeleteSettings(indexName);
}
return true;
}
public bool UpdateIndexBatch(string indexName) {
ILockFile lockFile = null;
var settingsFilename = GetSettingsFileName(indexName);
var lockFilename = settingsFilename + ".lock";
// acquire a lock file on the index
if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile)) {
Logger.Information("Index was requested but is already running");
return false;
}
using (lockFile) {
if (!_indexManager.HasIndexProvider()) {
return false;
}
// load index settings to know what is the current state of indexing
var indexSettings = LoadSettings(indexName);
_indexProvider = _indexManager.GetSearchIndexProvider();
var updateIndexDocuments = new List<IDocumentIndex>();
var addedContentItemIds = new List<string>();
DateTime? lastIndexUtc;
// Do we need to rebuild the full index (first time module is used, or rebuild index requested) ?
if (_indexProvider.IsEmpty(indexName)) {
Logger.Information("Rebuild index started");
// mark current last task, as we should process older ones (in case of rebuild index only)
lastIndexUtc = _indexingTaskManager.GetLastTaskDateTime();
// get every existing content item to index it
foreach (var contentItem in _contentManager.Query(VersionOptions.Published).List()) {
try {
// skip items which are not indexed
var settings = GetTypeIndexingSettings(contentItem);
if (!settings.Included)
continue;
var documentIndex = _indexProvider.New(contentItem.Id);
_contentManager.Index(contentItem, documentIndex);
if (documentIndex.IsDirty) {
updateIndexDocuments.Add(documentIndex);
addedContentItemIds.Add(contentItem.Id.ToString());
}
}
catch (Exception ex) {
Logger.Warning(ex, "Unable to index content item #{0} during rebuild", contentItem.Id);
}
}
}
else {
// retrieve last processed index time
lastIndexUtc = _indexProvider.GetLastIndexUtc(indexName);
}
_indexProvider.SetLastIndexUtc(indexName, _clock.UtcNow);
// retrieve not yet processed tasks
var taskRecords = lastIndexUtc == null
? _repository.Fetch(x => true).ToArray()
: _repository.Fetch(x => x.CreatedUtc >= lastIndexUtc).ToArray(); // CreatedUtc and lastIndexUtc might be equal if a content item is created in a background task
// nothing to do ?)))
if (taskRecords.Length + updateIndexDocuments.Count == 0) {
Logger.Information("Index update requested, nothing to do");
return;
}
Logger.Information("Processing {0} indexing tasks", taskRecords.Length);
if (!_indexProvider.Exists(indexName)) {
if (indexSettings.Mode == IndexingMode.Rebuild && indexSettings.LastContentId == 0) {
_indexProvider.CreateIndex(indexName);
// mark the last available task at the moment the process is started.
// once the Rebuild is done, Update will start at this point of the table
indexSettings.LastIndexedId = _taskRepository
.Table
.OrderByDescending(x => x.Id)
.Select(x => x.Id)
.FirstOrDefault();
}
// process Delete tasks
try {
var deleteIds = taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete).Select(t => t.ContentItemRecord.Id).ToArray();
if (deleteIds.Length > 0) {
_indexProvider.Delete(indexName, deleteIds);
Logger.Information("Deleted content items from index: {0}", String.Join(", ", deleteIds));
}
}
catch (Exception ex) {
Logger.Warning(ex, "An error occured while removing a document from the index");
}
// process Update tasks
foreach (var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update)) {
var task = new IndexingTask(_contentManager, taskRecord);
// skip items which are not indexed
var settings = GetTypeIndexingSettings(task.ContentItem);
if (!settings.Included)
continue;
try {
var documentIndex = _indexProvider.New(task.ContentItem.Id);
_contentManager.Index(task.ContentItem, documentIndex);
if (!addedContentItemIds.Contains(task.ContentItem.Id.ToString()) && documentIndex.IsDirty) {
updateIndexDocuments.Add(documentIndex);
}
}
catch (Exception ex) {
Logger.Warning(ex, "Unable to process indexing task #{0}", taskRecord.Id);
}
}
if (updateIndexDocuments.Count > 0) {
try {
_indexProvider.Store(indexName, updateIndexDocuments);
Logger.Information("Added content items to index: {0}", String.Join(", ", addedContentItemIds));
}
catch (Exception ex) {
Logger.Warning(ex, "An error occured while adding a document to the index");
}
}
}
finally {
System.Threading.Monitor.Exit(synLock);
// execute indexing commands by batch of [ContentItemsPerLoop] content items
return BatchIndex(indexName, settingsFilename, indexSettings);
}
}
static TypeIndexing GetTypeIndexingSettings(ContentItem contentItem) {
/// <summary>
/// Indexes a batch of content items
/// </summary>
/// <returns>
/// <c>true</c> if there are more items to process; otherwise, <c>false</c>.
/// </returns>
private bool BatchIndex(string indexName, string settingsFilename, IndexSettings indexSettings) {
var addToIndex = new List<IDocumentIndex>();
var deleteFromIndex = new List<int>();
// Rebuilding the index ?
if (indexSettings.Mode == IndexingMode.Rebuild) {
Logger.Information("Rebuilding index");
_indexingStatus = IndexingStatus.Rebuilding;
// load all content items
var contentItems = _contentRepository
.Fetch(
versionRecord => versionRecord.Published && versionRecord.Id > indexSettings.LastContentId,
order => order.Asc(versionRecord => versionRecord.Id))
.Take(ContentItemsPerLoop)
.Select(versionRecord => _contentManager.Get(versionRecord.ContentItemRecord.Id, VersionOptions.VersionRecord(versionRecord.Id)))
.Distinct()
.ToList();
// if no more elements to index, switch to update mode
if (contentItems.Count == 0) {
indexSettings.Mode = IndexingMode.Update;
}
foreach (var item in contentItems) {
try {
IDocumentIndex documentIndex = ExtractDocumentIndex(item);
if (documentIndex != null && documentIndex.IsDirty) {
addToIndex.Add(documentIndex);
}
indexSettings.LastContentId = item.VersionRecord.Id;
}
catch (Exception ex) {
Logger.Warning(ex, "Unable to index content item #{0} during rebuild", item.Id);
}
}
}
if (indexSettings.Mode == IndexingMode.Update) {
Logger.Information("Updating index");
_indexingStatus = IndexingStatus.Updating;
var contentItems = _taskRepository
.Fetch(x => x.Id > indexSettings.LastIndexedId)
.OrderBy(x => x.Id)
.Take(ContentItemsPerLoop)
.GroupBy(x => x.ContentItemRecord.Id)
.Select(group => new {TaskId = group.Max(task => task.Id), ContentItem = _contentManager.Get(group.Key, VersionOptions.Published)})
.OrderBy(x => x.TaskId)
.ToArray();
foreach (var item in contentItems) {
try {
IDocumentIndex documentIndex = ExtractDocumentIndex(item.ContentItem);
if (documentIndex == null) {
deleteFromIndex.Add(item.ContentItem.Id);
}
else if (documentIndex.IsDirty) {
addToIndex.Add(documentIndex);
}
indexSettings.LastIndexedId = item.TaskId;
}
catch (Exception ex) {
Logger.Warning(ex, "Unable to index content item #{0} during update", item.ContentItem.Id);
}
}
}
// save current state of the index
indexSettings.LastIndexedUtc = _clock.UtcNow;
_appDataFolder.CreateFile(settingsFilename, indexSettings.ToXml());
if (deleteFromIndex.Count == 0 && addToIndex.Count == 0) {
// nothing more to do
_indexingStatus = IndexingStatus.Idle;
return false;
}
// save new and updated documents to the index
try {
if (addToIndex.Count > 0) {
_indexProvider.Store(indexName, addToIndex);
Logger.Information("Added content items to index: {0}", addToIndex.Count);
}
}
catch (Exception ex) {
Logger.Warning(ex, "An error occured while adding a document to the index");
}
// removing documents from the index
try {
if (deleteFromIndex.Count > 0) {
_indexProvider.Delete(indexName, deleteFromIndex);
Logger.Information("Added content items to index: {0}", addToIndex.Count);
}
}
catch (Exception ex) {
Logger.Warning(ex, "An error occured while removing a document from the index");
}
return true;
}
/// <summary>
/// Loads the settings file or create a new default one if it doesn't exist
/// </summary>
public IndexSettings LoadSettings(string indexName)
{
var indexSettings = new IndexSettings();
var settingsFilename = GetSettingsFileName(indexName);
if (_appDataFolder.FileExists(settingsFilename))
{
var content = _appDataFolder.ReadFile(settingsFilename);
indexSettings = IndexSettings.Parse(content);
}
return indexSettings;
}
/// <summary>
/// Deletes the settings file
/// </summary>
public void DeleteSettings(string indexName) {
var settingsFilename = GetSettingsFileName(indexName);
if (_appDataFolder.FileExists(settingsFilename)) {
_appDataFolder.DeleteFile(settingsFilename);
}
}
/// <summary>
/// Creates a IDocumentIndex instance for a specific content item id. If the content
/// item is no more published, it returns null.
/// </summary>
private IDocumentIndex ExtractDocumentIndex(ContentItem contentItem) {
// ignore deleted or unpublished items
if (contentItem == null || !contentItem.IsPublished()) {
return null;
}
// skip items from types which are not indexed
var settings = GetTypeIndexingSettings(contentItem);
if (!settings.Included) {
return null;
}
var documentIndex = _indexProvider.New(contentItem.Id);
// call all handlers to add content to index
_contentManager.Index(contentItem, documentIndex);
return documentIndex;
}
private static TypeIndexing GetTypeIndexingSettings(ContentItem contentItem) {
if (contentItem == null ||
contentItem.TypeDefinition == null ||
contentItem.TypeDefinition.Settings == null) {
return new TypeIndexing { Included = false };
return new TypeIndexing {Included = false};
}
return contentItem.TypeDefinition.Settings.GetModel<TypeIndexing>();
}
private string GetSettingsFileName(string indexName) {
return _appDataFolder.Combine("Sites", _shellSettings.Name, indexName + ".settings.xml");
}
public DateTime GetLastIndexedUtc(string indexName) {
var indexSettings = LoadSettings(indexName);
return indexSettings.LastIndexedUtc;
}
public IndexingStatus GetIndexingStatus(string indexName) {
return _indexingStatus;
}
}
}

20
src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs
View File

@@ -30,8 +30,6 @@ namespace Orchard.Indexing.Services {
throw new ArgumentNullException("contentItem");
}
DeleteTasks(contentItem);
var taskRecord = new IndexingTaskRecord {
CreatedUtc = _clock.UtcNow,
ContentItemRecord = contentItem.Record,
@@ -53,23 +51,5 @@ namespace Orchard.Indexing.Services {
CreateTask(contentItem, IndexingTaskRecord.Delete);
Logger.Information("Deleting index task created for [{0}:{1}]", contentItem.ContentType, contentItem.Id);
}
public DateTime GetLastTaskDateTime() {
return _repository.Table.Max(t => t.CreatedUtc) ?? new DateTime(1980, 1, 1);
}
/// <summary>
/// Removes existing tasks for the specified content item
/// </summary>
public void DeleteTasks(ContentItem contentItem) {
var tasks = _repository
.Fetch(x => x.ContentItemRecord.Id == contentItem.Id)
.ToArray();
foreach (var task in tasks) {
_repository.Delete(task);
}
_repository.Flush();
}
}
}

42
src/Orchard.Web/Modules/Orchard.Indexing/Services/UpdateIndexScheduler.cs Normal file
View File

@@ -0,0 +1,42 @@
using System;
using System.Collections.Generic;
using Orchard.Environment.Configuration;
using Orchard.Environment.Descriptor;
using Orchard.Environment.State;
namespace Orchard.Indexing.Services {
public class UpdateIndexScheduler : IUpdateIndexScheduler, IIndexNotifierHandler {
private readonly IProcessingEngine _processingEngine;
private readonly ShellSettings _shellSettings;
private readonly IShellDescriptorManager _shellDescriptorManager;
private readonly Lazy<IIndexingTaskExecutor> _indexingTaskExecutor;
public UpdateIndexScheduler(
IProcessingEngine processingEngine,
ShellSettings shellSettings,
IShellDescriptorManager shellDescriptorManager,
Lazy<IIndexingTaskExecutor> indexingTaskExecutor
) {
_processingEngine = processingEngine;
_shellSettings = shellSettings;
_shellDescriptorManager = shellDescriptorManager;
_indexingTaskExecutor = indexingTaskExecutor;
}
public void Schedule(string indexName) {
var shellDescriptor = _shellDescriptorManager.GetShellDescriptor();
_processingEngine.AddTask(
_shellSettings,
shellDescriptor,
"IIndexNotifierHandler.UpdateIndex",
new Dictionary<string, object> { { "indexName", indexName } }
);
}
public void UpdateIndex(string indexName) {
if(_indexingTaskExecutor.Value.UpdateIndexBatch(indexName)) {
Schedule(indexName);
}
}
}
}

16
src/Orchard.Web/Modules/Orchard.Indexing/Views/Admin/Index.cshtml
View File

@@ -1,4 +1,5 @@
@model Orchard.Indexing.ViewModels.IndexViewModel
@using Orchard.Indexing.Services;
@{ Layout.Title = T("Search Index Management").ToString(); }
@@ -6,7 +7,7 @@
<fieldset>
@if (Model.IndexEntry == null) {
<p>@T("There is currently no search index")</p>
} else if (Model.IndexEntry.LastUpdateUtc == null) {
} else if (Model.IndexEntry.LastUpdateUtc == DateTime.MinValue) {
<p>@T("The search index has not been built yet.")</p>
} else {
if (Model.IndexEntry.DocumentCount == 0) {
@@ -21,8 +22,17 @@
<p>@T("The search index contains the following fields: {0}.", string.Join(T(", ").Text, Model.IndexEntry.Fields))</p>
}
<p>@T("The search index was last updated {0}.", Display.DateTimeRelative(dateTimeUtc: Model.IndexEntry.LastUpdateUtc.Value))</p>
}
<p>@T("The search index was last updated {0}.", Display.DateTimeRelative(dateTimeUtc: Model.IndexEntry.LastUpdateUtc))</p>
switch(Model.IndexEntry.IndexingStatus) {
case IndexingStatus.Rebuilding:
@T("The indexing process is currently being rebuilt.");
break;
case IndexingStatus.Updating:
@T("The indexing process is currently being updated.");
break;
}
}
<p>@T("Update the search index now: ")<button type="submit" title="@T("Update the search index.")" class="primaryAction">@T("Update")</button></p>
@Html.AntiForgeryTokenOrchard()
</fieldset>

2
src/Orchard/Environment/OrchardStarter.cs
View File

@@ -18,6 +18,7 @@ using Orchard.Environment.Descriptor;
using Orchard.Events;
using Orchard.FileSystems.AppData;
using Orchard.FileSystems.Dependencies;
using Orchard.FileSystems.LockFile;
using Orchard.FileSystems.VirtualPath;
using Orchard.FileSystems.WebSite;
using Orchard.Logging;
@@ -55,6 +56,7 @@ namespace Orchard.Environment {
RegisterVolatileProvider<WebSiteFolder, IWebSiteFolder>(builder);
RegisterVolatileProvider<AppDataFolder, IAppDataFolder>(builder);
RegisterVolatileProvider<DefaultLockFileManager, ILockFileManager>(builder);
RegisterVolatileProvider<Clock, IClock>(builder);
RegisterVolatileProvider<DefaultDependenciesFolder, IDependenciesFolder>(builder);
RegisterVolatileProvider<DefaultAssemblyProbingFolder, IAssemblyProbingFolder>(builder);

73
src/Orchard/FileSystems/LockFile/DefaultLockFileManager.cs Normal file
View File

@@ -0,0 +1,73 @@
using System;
using System.Threading;
using Orchard.FileSystems.AppData;
using Orchard.Services;
namespace Orchard.FileSystems.LockFile {
public class DefaultLockFileManager : ILockFileManager {
private readonly IAppDataFolder _appDataFolder;
private readonly IClock _clock;
private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
public static TimeSpan Expiration { get; private set; }
public DefaultLockFileManager(IAppDataFolder appDataFolder, IClock clock) {
_appDataFolder = appDataFolder;
_clock = clock;
Expiration = TimeSpan.FromMinutes(10);
}
public bool TryAcquireLock(string path, ref ILockFile lockFile) {
if (!_rwLock.TryEnterWriteLock(0)) {
return false;
}
try {
if (IsLockedImpl(path)) {
return false;
}
lockFile = new LockFile(_appDataFolder, path, _clock.UtcNow.ToString(), _rwLock);
return true;
}
catch {
// an error occured while reading/creating the lock file
return false;
}
finally {
_rwLock.ExitWriteLock();
}
}
public bool IsLocked(string path) {
_rwLock.EnterWriteLock();
try {
return IsLockedImpl(path);
}
catch {
// an error occured while reading the file
return true;
}
finally {
_rwLock.ExitWriteLock();
}
}
private bool IsLockedImpl(string path) {
if (_appDataFolder.FileExists(path)) {
var content = _appDataFolder.ReadFile(path);
DateTime creationUtc;
if (DateTime.TryParse(content, out creationUtc))
{
// if expired the file is not removed
// it should be automatically as there is a finalizer in LockFile
// or the next taker can do it, unless it also fails, again
return creationUtc.Add(Expiration) > _clock.UtcNow;
}
}
return false;
}
}
}

8
src/Orchard/FileSystems/LockFile/ILockFile.cs Normal file
View File

@@ -0,0 +1,8 @@
using System;
namespace Orchard.FileSystems.LockFile
{
public interface ILockFile : IDisposable {
void Release();
}
}

26
src/Orchard/FileSystems/LockFile/ILockFileManager.cs Normal file
View File

@@ -0,0 +1,26 @@
using Orchard.Caching;
namespace Orchard.FileSystems.LockFile {
/// <summary>
/// Abstraction for lock files creation.
/// </summary>
/// <remarks>
/// All virtual paths passed in or returned are relative to "~/App_Data".
/// </remarks>
public interface ILockFileManager : IVolatileProvider {
/// <summary>
/// Attempts to acquire an exclusive lock file.
/// </summary>
/// <param name="path">The filename of the lock file to create.</param>
/// <param name="lockFile">A reference to the lock file object if the lock is granted.</param>
/// <returns><c>true</c> if the lock is granted; otherwise, <c>false</c>.</returns>
bool TryAcquireLock(string path, ref ILockFile lockFile);
/// <summary>
/// Wether a lock file is already existing.
/// </summary>
/// <param name="path">The filename of the lock file to test.</param>
/// <returns><c>true</c> if the lock file exists; otherwise, <c>false</c>.</returns>
bool IsLocked(string path);
}
}

54
src/Orchard/FileSystems/LockFile/LockFile.cs Normal file
View File

@@ -0,0 +1,54 @@
using System.Threading;
using Orchard.FileSystems.AppData;
namespace Orchard.FileSystems.LockFile {
/// <summary>
/// Represents a Lock File acquired on the file system
/// </summary>
/// <remarks>
/// The instance needs to be disposed in order to release the lock explicitly
/// </remarks>
public class LockFile : ILockFile {
private readonly IAppDataFolder _appDataFolder;
private readonly string _path;
private readonly string _content;
private readonly ReaderWriterLockSlim _rwLock;
private bool _released;
public LockFile(IAppDataFolder appDataFolder, string path, string content, ReaderWriterLockSlim rwLock) {
_appDataFolder = appDataFolder;
_path = path;
_content = content;
_rwLock = rwLock;
// create the physical lock file
_appDataFolder.CreateFile(path, content);
}
public void Dispose() {
Release();
}
public void Release() {
_rwLock.EnterWriteLock();
try{
if (_released || !_appDataFolder.FileExists(_path)) {
// nothing to do, might happen if re-granted, and already released
return;
}
_released = true;
// check it has not been granted in the meantime
var current = _appDataFolder.ReadFile(_path);
if (current == _content) {
_appDataFolder.DeleteFile(_path);
}
}
finally {
_rwLock.ExitWriteLock();
}
}
}
}

13
src/Orchard/Indexing/IIndexProvider.cs
View File

@@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
namespace Orchard.Indexing {
public interface IIndexProvider : ISingletonDependency {
@@ -60,16 +59,6 @@ namespace Orchard.Indexing {
/// <returns>A search builder instance</returns>
ISearchBuilder CreateSearchBuilder(string indexName);
/// <summary>
/// Returns the date and time when the index was last processed, or null if the index doesn't exist
/// </summary>
DateTime? GetLastIndexUtc(string indexName);
/// <summary>
/// Sets the date and time when the index was last processed
/// </summary>
void SetLastIndexUtc(string indexName, DateTime lastIndexUtc);
/// <summary>
/// Returns every field available in the specified index
/// </summary>

4
src/Orchard/Orchard.Framework.csproj
View File

@@ -186,6 +186,10 @@
<Compile Include="Environment\WorkContextImplementation.cs" />
<Compile Include="Environment\WorkContextModule.cs" />
<Compile Include="Environment\WorkContextProperty.cs" />
<Compile Include="FileSystems\LockFile\ILockFile.cs" />
<Compile Include="FileSystems\LockFile\ILockFileManager.cs" />
<Compile Include="FileSystems\LockFile\LockFile.cs" />
<Compile Include="FileSystems\LockFile\DefaultLockFileManager.cs" />
<Compile Include="FileSystems\Media\FileSystemStorageProvider.cs" />
<Compile Include="Localization\Services\CurrentCultureWorkContext.cs" />
<Compile Include="Localization\Services\DefaultLocalizedStringManager.cs" />

11
src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs
View File

@@ -12,16 +12,5 @@ namespace Orchard.Tasks.Indexing {
/// Adds a new entry in the index task table in order to delete an existing index for the specified content item.
/// </summary>
void CreateDeleteIndexTask(ContentItem contentItem);
/// <summary>
/// Returns the Date Time of the last task created
/// </summary>
DateTime GetLastTaskDateTime();
/// <summary>
/// Deletes all indexing tasks assigned to a specific content item
/// </summary>
/// <param name="contentItem"></param>
void DeleteTasks(ContentItem contentItem);
}
}