diff --git a/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs b/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs index 664b10f06..68e0b46fa 100644 --- a/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs +++ b/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using Autofac; using Lucene.Services; using Moq; @@ -19,6 +18,7 @@ using Orchard.Environment; using Orchard.Environment.Configuration; using Orchard.Environment.Extensions; using Orchard.FileSystems.AppData; +using Orchard.FileSystems.LockFile; using Orchard.Indexing; using Orchard.Indexing.Handlers; using Orchard.Indexing.Models; @@ -34,14 +34,15 @@ namespace Orchard.Tests.Modules.Indexing { private IIndexProvider _provider; private IAppDataFolder _appDataFolder; private ShellSettings _shellSettings; - private IIndexNotifierHandler _indexNotifier; + private IIndexingTaskExecutor _indexTaskExecutor; private IContentManager _contentManager; private Mock _contentDefinitionManager; private StubLogger _logger; - private const string IndexName = "Search"; + private ILockFileManager _lockFileManager; + private const string IndexName = "Search"; private readonly string _basePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); - + [TestFixtureTearDown] public void Clean() { if (Directory.Exists(_basePath)) { @@ -59,10 +60,9 @@ namespace Orchard.Tests.Modules.Indexing { builder.RegisterType().As(); builder.RegisterInstance(_appDataFolder).As(); - builder.RegisterType().As(); + builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); - builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterInstance(_contentDefinitionManager.Object); @@ -80,6 +80,8 @@ namespace Orchard.Tests.Modules.Indexing { builder.RegisterType().As(); builder.RegisterType().As(); + builder.RegisterType().As(); + // setting up a ShellSettings instance _shellSettings = new ShellSettings { Name = "My Site" }; builder.RegisterInstance(_shellSettings).As(); @@ -100,11 +102,11 @@ namespace Orchard.Tests.Modules.Indexing { public override void Init() { base.Init(); - + _lockFileManager = _container.Resolve(); _provider = _container.Resolve(); - _indexNotifier = _container.Resolve(); + _indexTaskExecutor = _container.Resolve(); _contentManager = _container.Resolve(); - ((IndexingTaskExecutor)_indexNotifier).Logger = _logger = new StubLogger(); + ((IndexingTaskExecutor)_indexTaskExecutor).Logger = _logger = new StubLogger(); var thingType = new ContentTypeDefinitionBuilder() .Named(ThingDriver.ContentTypeName) @@ -116,21 +118,14 @@ namespace Orchard.Tests.Modules.Indexing { .Returns(thingType); } - private string[] Indexes() { - return new DirectoryInfo(Path.Combine(_basePath, "Sites", "My Site", "Indexes")).GetDirectories().Select(d => d.Name).ToArray(); - } - [Test] public void IndexShouldBeEmptyWhenThereIsNoContent() { - _indexNotifier.UpdateIndex(IndexName); + while(_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0)); - Assert.That(_logger.LogEntries.Count(), Is.EqualTo(2)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Index update requested, nothing to do")); } [Test] - public void ShouldIngoreNonIndexableContentWhenRebuildingTheIndex() { + public void ShouldIgnoreNonIndexableContentWhenRebuildingTheIndex() { var alphaType = new ContentTypeDefinitionBuilder() .Named("alpha") .Build(); @@ -141,11 +136,8 @@ namespace Orchard.Tests.Modules.Indexing { _contentManager.Create("alpha"); - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0)); - Assert.That(_logger.LogEntries.Count(), Is.EqualTo(2)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Index update requested, nothing to do")); } [Test] @@ -161,11 +153,8 @@ namespace Orchard.Tests.Modules.Indexing { _contentManager.Create("alpha"); - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0)); - Assert.That(_logger.LogEntries.Count(), Is.EqualTo(2)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Index update requested, nothing to do")); } [Test] @@ -173,80 +162,68 @@ namespace Orchard.Tests.Modules.Indexing { var content = _contentManager.Create(ThingDriver.ContentTypeName); content.Text = "Lorem ipsum"; - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); - Assert.That(_logger.LogEntries.Count(), Is.EqualTo(3)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Processing {0} indexing tasks")); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Added content items to index: {0}")); } [Test] public void ShouldUpdateTheIndexWhenContentIsPublished() { _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} + Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); + + // there should be nothing done + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - _logger.Clear(); _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2)); - Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Rebuild index started")); + } + + [Test] + public void IndexingTaskExecutorShouldNotBeReEntrant() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("Sites/My Site/Search.settings.xml.lock", ref lockFile); + using (lockFile) { + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} + Assert.That(_logger.LogEntries.Count, Is.EqualTo(1)); + Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Index was requested but is already running")); + } + + _logger.LogEntries.Clear(); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} + Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Index was requested but is already running")); } [Test] public void ShouldUpdateTheIndexWhenContentIsUnPublished() { _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; - _clock.Advance(TimeSpan.FromSeconds(1)); - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - _logger.Clear(); var content = _contentManager.Create(ThingDriver.ContentTypeName); content.Text = "Lorem ipsum"; - _clock.Advance(TimeSpan.FromSeconds(1)); - - _indexNotifier.UpdateIndex(IndexName); + + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2)); - Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Rebuild index started")); - _clock.Advance(TimeSpan.FromSeconds(1)); _contentManager.Unpublish(content.ContentItem); - _clock.Advance(TimeSpan.FromSeconds(1)); - - _indexNotifier.UpdateIndex(IndexName); + + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); - Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Rebuild index started")); } + [Test] - public void ShouldRemoveFromIndexEvenIfPublishedAndUnpublishedInTheSameSecond() { - // This test is to ensure that when a task is created, all previous tasks for the same content item - // are also removed, and thus that multiple tasks don't conflict while updating the index - - _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; - _clock.Advance(TimeSpan.FromSeconds(1)); - - _indexNotifier.UpdateIndex(IndexName); - Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - _logger.Clear(); - - var content = _contentManager.Create(ThingDriver.ContentTypeName); - content.Text = "Lorem ipsum"; - - _indexNotifier.UpdateIndex(IndexName); - Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2)); - Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Rebuild index started")); - - _contentManager.Unpublish(content.ContentItem); - - _indexNotifier.UpdateIndex(IndexName); - Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); - Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Rebuild index started")); + public void ShouldIndexAllContentOverTheLoopSize() { + for (int i = 0; i < 999; i++) { + var content = _contentManager.Create(ThingDriver.ContentTypeName); + content.Text = "Lorem ipsum " + i; + } + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} + Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(999)); } #region Stubs @@ -293,7 +270,7 @@ namespace Orchard.Tests.Modules.Indexing { } public void Log(LogLevel level, Exception exception, string format, params object[] args) { - LogEntries.Add(new LogEntry() { + LogEntries.Add(new LogEntry { LogArgs = args, LogException = exception, LogFormat = format, diff --git a/src/Orchard.Tests.Modules/Indexing/LuceneIndexProviderTests.cs b/src/Orchard.Tests.Modules/Indexing/LuceneIndexProviderTests.cs index 2d69c727e..2f170757e 100644 --- a/src/Orchard.Tests.Modules/Indexing/LuceneIndexProviderTests.cs +++ b/src/Orchard.Tests.Modules/Indexing/LuceneIndexProviderTests.cs @@ -7,7 +7,6 @@ using NUnit.Framework; using Orchard.Environment.Configuration; using Orchard.FileSystems.AppData; using Orchard.Indexing; -using Orchard.Indexing.Services; using Orchard.Tests.FileSystems.AppData; namespace Orchard.Tests.Modules.Indexing { @@ -198,18 +197,6 @@ namespace Orchard.Tests.Modules.Indexing { Assert.That(searchBuilder.Get(3).ContentItemId, Is.EqualTo(3)); } - [Test] - public void ProviderShouldStoreSettings() { - _provider.CreateIndex("default"); - Assert.That(_provider.GetLastIndexUtc("default"), Is.Null); - - _provider.SetLastIndexUtc("default", new DateTime(2010, 1, 1, 1, 1, 1, 1)); - Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(new DateTime(2010, 1, 1, 1, 1, 1, 0))); - - _provider.SetLastIndexUtc("default", new DateTime(1901, 1, 1, 1, 1, 1, 1)); - Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(LuceneIndexProvider.DefaultMinDateTime)); - } - [Test] public void IsEmptyShouldBeTrueForNoneExistingIndexes() { _provider.IsEmpty("dummy"); @@ -238,9 +225,7 @@ namespace Orchard.Tests.Modules.Indexing { [Test] public void IsDirtyShouldBeTrueWhenIndexIsModified() { - IDocumentIndex doc; - - doc = _provider.New(1); + IDocumentIndex doc = _provider.New(1); doc.Add("foo", "value"); Assert.That(doc.IsDirty, Is.True); @@ -281,7 +266,7 @@ namespace Orchard.Tests.Modules.Indexing { Assert.That(searchBuilder.Get(11).ContentItemId, Is.EqualTo(11)); Assert.That(searchBuilder.Get(111).ContentItemId, Is.EqualTo(111)); - _provider.Delete("default", new int[] {1, 11, 111 }); + _provider.Delete("default", new [] {1, 11, 111 }); Assert.That(searchBuilder.Get(1), Is.Null); Assert.That(searchBuilder.Get(11), Is.Null); diff --git a/src/Orchard.Tests/FileSystems/LockFile/LockFileManagerTests.cs b/src/Orchard.Tests/FileSystems/LockFile/LockFileManagerTests.cs new file mode 100644 index 000000000..1c6a41894 --- /dev/null +++ b/src/Orchard.Tests/FileSystems/LockFile/LockFileManagerTests.cs @@ -0,0 +1,223 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using NUnit.Framework; +using Orchard.FileSystems.AppData; +using Orchard.FileSystems.LockFile; +using Orchard.Tests.Stubs; + +namespace Orchard.Tests.FileSystems.LockFile { + [TestFixture] + public class LockFileManagerTests { + private string _tempFolder; + private IAppDataFolder _appDataFolder; + private ILockFileManager _lockFileManager; + private StubClock _clock; + + public class StubAppDataFolderRoot : IAppDataFolderRoot { + public string RootPath { get; set; } + public string RootFolder { get; set; } + } + + public static IAppDataFolder CreateAppDataFolder(string tempFolder) { + var folderRoot = new StubAppDataFolderRoot {RootPath = "~/App_Data", RootFolder = tempFolder}; + var monitor = new StubVirtualPathMonitor(); + return new AppDataFolder(folderRoot, monitor); + } + + [SetUp] + public void Init() { + _tempFolder = Path.GetTempFileName(); + File.Delete(_tempFolder); + _appDataFolder = CreateAppDataFolder(_tempFolder); + + _clock = new StubClock(); + _lockFileManager = new DefaultLockFileManager(_appDataFolder, _clock); + } + + [TearDown] + public void Term() { + Directory.Delete(_tempFolder, true); + } + + [Test] + public void LockShouldBeGrantedWhenDoesNotExist() { + ILockFile lockFile = null; + var granted = _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + Assert.That(granted, Is.True); + Assert.That(lockFile, Is.Not.Null); + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + } + + [Test] + public void ExistingLockFileShouldPreventGrants() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + Assert.That(_lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile), Is.False); + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + } + + [Test] + public void ReleasingALockShouldAllowGranting() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + using (lockFile) { + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + } + + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0)); + } + + [Test] + public void ReleasingAReleasedLockShouldWork() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + + lockFile.Release(); + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0)); + + lockFile.Release(); + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0)); + } + + [Test] + public void DisposingLockShouldReleaseIt() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + using (lockFile) { + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + } + + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0)); + } + + [Test] + public void ExpiredLockShouldBeAvailable() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + _clock.Advance(DefaultLockFileManager.Expiration); + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + } + + [Test] + public void ShouldGrantExpiredLock() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + _clock.Advance(DefaultLockFileManager.Expiration); + var granted = _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + Assert.That(granted, Is.True); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + } + + private static int _lockCount; + private static readonly object _synLock = new object(); + + [Test] + public void AcquiringLockShouldBeThreadSafe() { + var threads = new List(); + for(var i=0; i<10; i++) { + var t = new Thread(PlayWithAcquire); + t.Start(); + threads.Add(t); + } + + threads.ForEach(t => t.Join()); + Assert.That(_lockCount, Is.EqualTo(0)); + } + + [Test] + public void IsLockedShouldBeThreadSafe() { + var threads = new List(); + for (var i = 0; i < 10; i++) + { + var t = new Thread(PlayWithIsLocked); + t.Start(); + threads.Add(t); + } + + threads.ForEach(t => t.Join()); + Assert.That(_lockCount, Is.EqualTo(0)); + } + + private void PlayWithAcquire() { + var r = new Random(DateTime.Now.Millisecond); + ILockFile lockFile = null; + + // loop until the lock has been acquired + for (;;) { + if (!_lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile)) { + continue; + } + + lock (_synLock) { + _lockCount++; + Assert.That(_lockCount, Is.EqualTo(1)); + } + + // keep the lock for a certain time + Thread.Sleep(r.Next(200)); + lock (_synLock) { + _lockCount--; + Assert.That(_lockCount, Is.EqualTo(0)); + } + + lockFile.Release(); + return; + } + } + + private void PlayWithIsLocked() { + var r = new Random(DateTime.Now.Millisecond); + ILockFile lockFile = null; + const string path = "foo.txt.lock"; + + // loop until the lock has been acquired + for (;;) { + if(_lockFileManager.IsLocked(path)) { + continue; + } + + if (!_lockFileManager.TryAcquireLock(path, ref lockFile)) { + continue; + } + + lock (_synLock) { + _lockCount++; + Assert.That(_lockCount, Is.EqualTo(1)); + } + + // keep the lock for a certain time + Thread.Sleep(r.Next(200)); + lock (_synLock) { + _lockCount--; + Assert.That(_lockCount, Is.EqualTo(0)); + } + + lockFile.Release(); + return; + } + } + + } +} diff --git a/src/Orchard.Tests/Orchard.Framework.Tests.csproj b/src/Orchard.Tests/Orchard.Framework.Tests.csproj index 2234734a0..9d1c806f2 100644 --- a/src/Orchard.Tests/Orchard.Framework.Tests.csproj +++ b/src/Orchard.Tests/Orchard.Framework.Tests.csproj @@ -247,6 +247,7 @@ + diff --git a/src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs b/src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs index d89e8e32f..0d6e3719f 100644 --- a/src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs +++ b/src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs @@ -9,7 +9,6 @@ using Lucene.Net.Documents; using Lucene.Net.Index; using Lucene.Net.Search; using Lucene.Net.Store; -using System.Xml.Linq; using Orchard.Environment.Configuration; using Orchard.FileSystems.AppData; using Orchard.Indexing; @@ -29,8 +28,6 @@ namespace Lucene.Services { private readonly Analyzer _analyzer ; private readonly string _basePath; public static readonly DateTime DefaultMinDateTime = new DateTime(1980, 1, 1); - public static readonly string Settings = "Settings"; - public static readonly string LastIndexUtc = "LastIndexedUtc"; public LuceneIndexProvider(IAppDataFolder appDataFolder, ShellSettings shellSettings) { _appDataFolder = appDataFolder; @@ -123,11 +120,6 @@ namespace Lucene.Services { public void DeleteIndex(string indexName) { new DirectoryInfo(_appDataFolder.MapPath(_appDataFolder.Combine(_basePath, indexName))) .Delete(true); - - var settingsFileName = GetSettingsFileName(indexName); - if (File.Exists(settingsFileName)) { - File.Delete(settingsFileName); - } } public void Store(string indexName, IDocumentIndex indexDocument) { @@ -206,40 +198,6 @@ namespace Lucene.Services { return new LuceneSearchBuilder(GetDirectory(indexName)) { Logger = Logger }; } - private string GetSettingsFileName(string indexName) { - return _appDataFolder.MapPath(_appDataFolder.Combine(_basePath, indexName + ".settings.xml")); - } - - public DateTime? GetLastIndexUtc(string indexName) { - var settingsFileName = GetSettingsFileName(indexName); - - if (!File.Exists(settingsFileName)) - return null; - - return DateTime.Parse(XDocument.Load(settingsFileName).Descendants(LastIndexUtc).First().Value); - } - - public void SetLastIndexUtc(string indexName, DateTime lastIndexUtc) { - if ( lastIndexUtc < DefaultMinDateTime ) { - lastIndexUtc = DefaultMinDateTime; - } - - XDocument doc; - var settingsFileName = GetSettingsFileName(indexName); - if ( !File.Exists(settingsFileName) ) { - EnsureDirectoryExists(); - doc = new XDocument( - new XElement(Settings, - new XElement(LastIndexUtc, lastIndexUtc.ToString("s")))); - } - else { - doc = XDocument.Load(settingsFileName); - doc.Element(Settings).Element(LastIndexUtc).Value = lastIndexUtc.ToString("s"); - } - - doc.Save(settingsFileName); - } - public IEnumerable GetFields(string indexName) { if ( !Exists(indexName) ) { return Enumerable.Empty(); diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs index b8b702044..f09dc8aa5 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Linq; using Orchard.Commands; using Orchard.ContentManagement; @@ -35,7 +34,7 @@ namespace Orchard.Indexing.Commands { [CommandName("index update")] [CommandHelp("index update\r\n\t" + "Updates the search index")] public string Update() { - _indexingService.UpdateIndex(); + _indexingService.UpdateIndex(SearchIndexName); return T("Index is now being updated...").Text; } @@ -43,8 +42,7 @@ namespace Orchard.Indexing.Commands { [CommandName("index rebuild")] [CommandHelp("index rebuild \r\n\t" + "Rebuilds the search index")] public string Rebuild() { - _indexingService.RebuildIndex(); - _indexingService.UpdateIndex(); + _indexingService.RebuildIndex(SearchIndexName); return T("Index is now being rebuilt...").Text; } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs index f593f0206..3057b3a4b 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs @@ -8,6 +8,7 @@ using Orchard.Indexing.ViewModels; namespace Orchard.Indexing.Controllers { public class AdminController : Controller { private readonly IIndexingService _indexingService; + private const string DefaultIndexName = "Search"; public AdminController(IIndexingService indexingService, IOrchardServices services) { _indexingService = indexingService; @@ -19,7 +20,7 @@ namespace Orchard.Indexing.Controllers { public Localizer T { get; set; } public ActionResult Index() { - var viewModel = new IndexViewModel { IndexEntry = _indexingService.GetIndexEntry() }; + var viewModel = new IndexViewModel { IndexEntry = _indexingService.GetIndexEntry(DefaultIndexName) }; if (viewModel.IndexEntry == null) Services.Notifier.Information(T("There is no search index to manage for this site.")); @@ -32,7 +33,7 @@ namespace Orchard.Indexing.Controllers { if (!Services.Authorizer.Authorize(StandardPermissions.SiteOwner, T("Not allowed to manage the search index."))) return new HttpUnauthorizedResult(); - _indexingService.UpdateIndex(); + _indexingService.UpdateIndex(DefaultIndexName); return RedirectToAction("Index"); } @@ -42,8 +43,7 @@ namespace Orchard.Indexing.Controllers { if (!Services.Authorizer.Authorize(StandardPermissions.SiteOwner, T("Not allowed to manage the search index."))) return new HttpUnauthorizedResult(); - _indexingService.RebuildIndex(); - _indexingService.UpdateIndex(); + _indexingService.RebuildIndex(DefaultIndexName); return RedirectToAction("Index"); } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs new file mode 100644 index 000000000..93276adc9 --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs @@ -0,0 +1,57 @@ +using System; +using System.Linq; +using System.Xml.Linq; + +namespace Orchard.Indexing.Models +{ + public enum IndexingMode { + Rebuild, + Update + } + + public class IndexSettings { + public IndexingMode Mode { get; set; } + public int LastIndexedId { get; set; } + public int LastContentId { get; set; } + public DateTime LastIndexedUtc { get; set; } + + public static readonly string TagSettings = "Settings"; + public static readonly string TagMode = "Mode"; + public static readonly string TagLastIndexedId = "LastIndexedId"; + public static readonly string TagLastContentId = "LastContentId"; + public static readonly string TagLastIndexedUtc = "LastIndexedUtc"; + + public IndexSettings() { + Mode = IndexingMode.Rebuild; + LastIndexedId = 0; + LastContentId = 0; + LastIndexedUtc = DateTime.MinValue; + } + + public static IndexSettings Parse(string content) { + var doc = XDocument.Parse(content); + + try { + return new IndexSettings { + Mode = (IndexingMode) Enum.Parse(typeof (IndexingMode), doc.Descendants(TagMode).First().Value), + LastIndexedId = Int32.Parse(doc.Descendants(TagLastIndexedId).First().Value), + LastContentId = Int32.Parse(doc.Descendants(TagLastContentId).First().Value), + LastIndexedUtc = DateTime.Parse(doc.Descendants(TagLastIndexedUtc).First().Value).ToUniversalTime() + }; + } + catch { + return new IndexSettings(); + } + } + + public string ToXml() { + return new XDocument( + new XElement(TagSettings, + new XElement(TagMode, Mode), + new XElement(TagLastIndexedId, LastIndexedId), + new XElement(TagLastContentId, LastContentId), + new XElement(TagLastIndexedUtc, LastIndexedUtc.ToString("u")) + )).ToString(); + } + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj b/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj index be7ea6005..651d5d389 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj @@ -41,6 +41,8 @@ ..\..\..\..\lib\aspnetmvc\System.Web.Mvc.dll + + @@ -55,12 +57,18 @@ + + Code + + + + + - diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexService.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexService.cs index 5214bc6bb..c0fa721fb 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexService.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexService.cs @@ -5,13 +5,14 @@ namespace Orchard.Indexing.Services { public class IndexEntry { public string IndexName { get; set; } public int DocumentCount { get; set; } - public DateTime? LastUpdateUtc { get; set; } + public DateTime LastUpdateUtc { get; set; } public IEnumerable Fields { get; set; } + public IndexingStatus IndexingStatus { get; set; } } public interface IIndexingService : IDependency { - void RebuildIndex(); - void UpdateIndex(); - IndexEntry GetIndexEntry(); + void RebuildIndex(string indexName); + void UpdateIndex(string indexName); + IndexEntry GetIndexEntry(string indexName); } } \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexStatisticsProvider.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexStatisticsProvider.cs new file mode 100644 index 000000000..6b2dcb48c --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexStatisticsProvider.cs @@ -0,0 +1,13 @@ +using System; + +namespace Orchard.Indexing.Services { + public enum IndexingStatus { + Rebuilding, + Updating, + Idle + } + public interface IIndexStatisticsProvider : IDependency { + DateTime GetLastIndexedUtc(string indexName); + IndexingStatus GetIndexingStatus(string indexName); + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexingTaskExecutor.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexingTaskExecutor.cs new file mode 100644 index 000000000..89e385e6f --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexingTaskExecutor.cs @@ -0,0 +1,6 @@ +namespace Orchard.Indexing.Services { + public interface IIndexingTaskExecutor : IDependency { + bool DeleteIndex(string indexName); + bool UpdateIndexBatch(string indexName); + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IUpdateIndexScheduler.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IUpdateIndexScheduler.cs new file mode 100644 index 000000000..eae037600 --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IUpdateIndexScheduler.cs @@ -0,0 +1,5 @@ +namespace Orchard.Indexing.Services { + public interface IUpdateIndexScheduler : IDependency { + void Schedule(string indexName); + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs index 90014a203..aad07cd8d 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs @@ -1,59 +1,67 @@ using System.Collections.Generic; using Orchard.Localization; -using Orchard.Localization.Services; using Orchard.UI.Notify; namespace Orchard.Indexing.Services { public class IndexingService : IIndexingService { - private const string SearchIndexName = "Search"; private readonly IIndexManager _indexManager; private readonly IEnumerable _indexNotifierHandlers; + private readonly IIndexStatisticsProvider _indexStatisticsProvider; + private readonly IIndexingTaskExecutor _indexingTaskExecutor; - public IndexingService(IOrchardServices services, IIndexManager indexManager, IEnumerable indexNotifierHandlers, ICultureManager cultureManager) { + public IndexingService( + IOrchardServices services, + IIndexManager indexManager, + IEnumerable indexNotifierHandlers, + IIndexStatisticsProvider indexStatisticsProvider, + IIndexingTaskExecutor indexingTaskExecutor) { Services = services; _indexManager = indexManager; _indexNotifierHandlers = indexNotifierHandlers; + _indexStatisticsProvider = indexStatisticsProvider; + _indexingTaskExecutor = indexingTaskExecutor; T = NullLocalizer.Instance; } public IOrchardServices Services { get; set; } public Localizer T { get; set; } - void IIndexingService.RebuildIndex() { + public void RebuildIndex(string indexName) { if (!_indexManager.HasIndexProvider()) { Services.Notifier.Warning(T("There is no search index to rebuild.")); return; } - var searchProvider = _indexManager.GetSearchIndexProvider(); - if (searchProvider.Exists(SearchIndexName)) - searchProvider.DeleteIndex(SearchIndexName); - - searchProvider.CreateIndex(SearchIndexName); // or just reset the updated date and let the background process recreate the index - - Services.Notifier.Information(T("The search index has been rebuilt.")); + if(_indexingTaskExecutor.DeleteIndex(indexName)) { + Services.Notifier.Information(T("The index {0} has been rebuilt.", indexName)); + UpdateIndex(indexName); + } + else { + Services.Notifier.Warning(T("The index {0} could no ben rebuilt. It might already be in use, please try again later.", indexName)); + } } - void IIndexingService.UpdateIndex() { + public void UpdateIndex(string indexName) { foreach(var handler in _indexNotifierHandlers) { - handler.UpdateIndex(SearchIndexName); + handler.UpdateIndex(indexName); } Services.Notifier.Information(T("The search index has been updated.")); } - IndexEntry IIndexingService.GetIndexEntry() { + IndexEntry IIndexingService.GetIndexEntry(string indexName) { var provider = _indexManager.GetSearchIndexProvider(); if (provider == null) return null; return new IndexEntry { - IndexName = SearchIndexName, - DocumentCount = provider.NumDocs(SearchIndexName), - Fields = provider.GetFields(SearchIndexName), - LastUpdateUtc = provider.GetLastIndexUtc(SearchIndexName) + IndexName = indexName, + DocumentCount = provider.NumDocs(indexName), + Fields = provider.GetFields(indexName), + LastUpdateUtc = _indexStatisticsProvider.GetLastIndexedUtc(indexName), + IndexingStatus = _indexStatisticsProvider.GetIndexingStatus(indexName) }; } } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexSynLock.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexSynLock.cs deleted file mode 100644 index 7b545ccaf..000000000 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexSynLock.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Collections.Generic; - -namespace Orchard.Indexing.Services { - public interface IIndexSynLock : ISingletonDependency { - object GetSynLock(string indexName); - } - - public class IndexSynLock : IIndexSynLock { - private readonly Dictionary _synLocks; - private readonly object _synLock = new object(); - - public IndexSynLock() { - _synLocks =new Dictionary(); - } - - public object GetSynLock(string indexName) { - lock(_synLock) { - if(!_synLocks.ContainsKey(indexName)) { - _synLocks[indexName] = new object(); - } - return _synLocks[indexName]; - } - } - } -} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs index 3747fe997..9526e1f69 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs @@ -3,173 +3,308 @@ using System.Collections.Generic; using System.Linq; using JetBrains.Annotations; using Orchard.ContentManagement; +using Orchard.ContentManagement.Records; using Orchard.Data; +using Orchard.Environment.Configuration; +using Orchard.FileSystems.AppData; +using Orchard.FileSystems.LockFile; using Orchard.Indexing.Models; using Orchard.Indexing.Settings; using Orchard.Logging; using Orchard.Services; -using Orchard.Tasks.Indexing; namespace Orchard.Indexing.Services { /// /// Contains the logic which is regularly executed to retrieve index information from multiple content handlers. /// + /// + /// This class is synchronized using a lock file as both command line and web workers can potentially use it, + /// and singleton locks would not be shared accross those two. + /// [UsedImplicitly] - public class IndexingTaskExecutor : IIndexNotifierHandler { - private readonly IClock _clock; - private readonly IRepository _repository; + public class IndexingTaskExecutor : IIndexingTaskExecutor, IIndexStatisticsProvider + { + private readonly IRepository _taskRepository; + private readonly IRepository _contentRepository; private IIndexProvider _indexProvider; private readonly IIndexManager _indexManager; - private readonly IIndexingTaskManager _indexingTaskManager; private readonly IContentManager _contentManager; - private readonly IIndexSynLock _indexSynLock; + private readonly IAppDataFolder _appDataFolder; + private readonly ShellSettings _shellSettings; + private readonly ILockFileManager _lockFileManager; + private readonly IClock _clock; + private const int ContentItemsPerLoop = 50; + private IndexingStatus _indexingStatus = IndexingStatus.Idle; public IndexingTaskExecutor( - IClock clock, - IRepository repository, + IRepository taskRepository, + IRepository contentRepository, IIndexManager indexManager, - IIndexingTaskManager indexingTaskManager, IContentManager contentManager, - IIndexSynLock indexSynLock) { - _clock = clock; - _repository = repository; + IAppDataFolder appDataFolder, + ShellSettings shellSettings, + ILockFileManager lockFileManager, + IClock clock) { + _taskRepository = taskRepository; + _contentRepository = contentRepository; _indexManager = indexManager; - _indexingTaskManager = indexingTaskManager; _contentManager = contentManager; - _indexSynLock = indexSynLock; + _appDataFolder = appDataFolder; + _shellSettings = shellSettings; + _lockFileManager = lockFileManager; + _clock = clock; Logger = NullLogger.Instance; } public ILogger Logger { get; set; } - public void UpdateIndex(string indexName) { - var synLock = _indexSynLock.GetSynLock(indexName); + public bool DeleteIndex(string indexName) { + ILockFile lockFile = null; + var settingsFilename = GetSettingsFileName(indexName); + var lockFilename = settingsFilename + ".lock"; - if (!System.Threading.Monitor.TryEnter(synLock)) { - Logger.Information("Index was requested but was already running"); - return; + // acquire a lock file on the index + if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile)) { + Logger.Information("Could not delete the index. Already in use."); + return false; } - try { - + using (lockFile) { if (!_indexManager.HasIndexProvider()) { - return; + return false; } + var searchProvider = _indexManager.GetSearchIndexProvider(); + if (searchProvider.Exists(indexName)) { + searchProvider.DeleteIndex(indexName); + } + + DeleteSettings(indexName); + } + + return true; + } + + public bool UpdateIndexBatch(string indexName) { + ILockFile lockFile = null; + var settingsFilename = GetSettingsFileName(indexName); + var lockFilename = settingsFilename + ".lock"; + + // acquire a lock file on the index + if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile)) { + Logger.Information("Index was requested but is already running"); + return false; + } + + using (lockFile) { + if (!_indexManager.HasIndexProvider()) { + return false; + } + + // load index settings to know what is the current state of indexing + var indexSettings = LoadSettings(indexName); + _indexProvider = _indexManager.GetSearchIndexProvider(); - var updateIndexDocuments = new List(); - var addedContentItemIds = new List(); - DateTime? lastIndexUtc; - // Do we need to rebuild the full index (first time module is used, or rebuild index requested) ? - if (_indexProvider.IsEmpty(indexName)) { - Logger.Information("Rebuild index started"); - - // mark current last task, as we should process older ones (in case of rebuild index only) - lastIndexUtc = _indexingTaskManager.GetLastTaskDateTime(); - - // get every existing content item to index it - foreach (var contentItem in _contentManager.Query(VersionOptions.Published).List()) { - try { - // skip items which are not indexed - var settings = GetTypeIndexingSettings(contentItem); - if (!settings.Included) - continue; - - var documentIndex = _indexProvider.New(contentItem.Id); - - _contentManager.Index(contentItem, documentIndex); - if (documentIndex.IsDirty) { - updateIndexDocuments.Add(documentIndex); - addedContentItemIds.Add(contentItem.Id.ToString()); - } - } - catch (Exception ex) { - Logger.Warning(ex, "Unable to index content item #{0} during rebuild", contentItem.Id); - } - } - - } - else { - // retrieve last processed index time - lastIndexUtc = _indexProvider.GetLastIndexUtc(indexName); - } - - _indexProvider.SetLastIndexUtc(indexName, _clock.UtcNow); - - // retrieve not yet processed tasks - var taskRecords = lastIndexUtc == null - ? _repository.Fetch(x => true).ToArray() - : _repository.Fetch(x => x.CreatedUtc >= lastIndexUtc).ToArray(); // CreatedUtc and lastIndexUtc might be equal if a content item is created in a background task - - // nothing to do ?))) - if (taskRecords.Length + updateIndexDocuments.Count == 0) { - Logger.Information("Index update requested, nothing to do"); - return; - } - - Logger.Information("Processing {0} indexing tasks", taskRecords.Length); - - if (!_indexProvider.Exists(indexName)) { + if (indexSettings.Mode == IndexingMode.Rebuild && indexSettings.LastContentId == 0) { _indexProvider.CreateIndex(indexName); + + // mark the last available task at the moment the process is started. + // once the Rebuild is done, Update will start at this point of the table + indexSettings.LastIndexedId = _taskRepository + .Table + .OrderByDescending(x => x.Id) + .Select(x => x.Id) + .FirstOrDefault(); } - // process Delete tasks - try { - var deleteIds = taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete).Select(t => t.ContentItemRecord.Id).ToArray(); - if (deleteIds.Length > 0) { - _indexProvider.Delete(indexName, deleteIds); - Logger.Information("Deleted content items from index: {0}", String.Join(", ", deleteIds)); - } - } - catch (Exception ex) { - Logger.Warning(ex, "An error occured while removing a document from the index"); - } - - // process Update tasks - foreach (var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update)) { - var task = new IndexingTask(_contentManager, taskRecord); - - // skip items which are not indexed - var settings = GetTypeIndexingSettings(task.ContentItem); - if (!settings.Included) - continue; - - try { - var documentIndex = _indexProvider.New(task.ContentItem.Id); - _contentManager.Index(task.ContentItem, documentIndex); - if (!addedContentItemIds.Contains(task.ContentItem.Id.ToString()) && documentIndex.IsDirty) { - updateIndexDocuments.Add(documentIndex); - } - - } - catch (Exception ex) { - Logger.Warning(ex, "Unable to process indexing task #{0}", taskRecord.Id); - } - } - - if (updateIndexDocuments.Count > 0) { - try { - _indexProvider.Store(indexName, updateIndexDocuments); - Logger.Information("Added content items to index: {0}", String.Join(", ", addedContentItemIds)); - } - catch (Exception ex) { - Logger.Warning(ex, "An error occured while adding a document to the index"); - } - } - } - finally { - System.Threading.Monitor.Exit(synLock); + // execute indexing commands by batch of [ContentItemsPerLoop] content items + return BatchIndex(indexName, settingsFilename, indexSettings); } } - static TypeIndexing GetTypeIndexingSettings(ContentItem contentItem) { + /// + /// Indexes a batch of content items + /// + /// + /// true if there are more items to process; otherwise, false. + /// + private bool BatchIndex(string indexName, string settingsFilename, IndexSettings indexSettings) { + var addToIndex = new List(); + var deleteFromIndex = new List(); + + // Rebuilding the index ? + if (indexSettings.Mode == IndexingMode.Rebuild) { + Logger.Information("Rebuilding index"); + _indexingStatus = IndexingStatus.Rebuilding; + + // load all content items + var contentItems = _contentRepository + .Fetch( + versionRecord => versionRecord.Published && versionRecord.Id > indexSettings.LastContentId, + order => order.Asc(versionRecord => versionRecord.Id)) + .Take(ContentItemsPerLoop) + .Select(versionRecord => _contentManager.Get(versionRecord.ContentItemRecord.Id, VersionOptions.VersionRecord(versionRecord.Id))) + .Distinct() + .ToList(); + + // if no more elements to index, switch to update mode + if (contentItems.Count == 0) { + indexSettings.Mode = IndexingMode.Update; + } + + foreach (var item in contentItems) { + try { + IDocumentIndex documentIndex = ExtractDocumentIndex(item); + + if (documentIndex != null && documentIndex.IsDirty) { + addToIndex.Add(documentIndex); + } + + indexSettings.LastContentId = item.VersionRecord.Id; + } + catch (Exception ex) { + Logger.Warning(ex, "Unable to index content item #{0} during rebuild", item.Id); + } + } + } + + if (indexSettings.Mode == IndexingMode.Update) { + Logger.Information("Updating index"); + _indexingStatus = IndexingStatus.Updating; + + var contentItems = _taskRepository + .Fetch(x => x.Id > indexSettings.LastIndexedId) + .OrderBy(x => x.Id) + .Take(ContentItemsPerLoop) + .GroupBy(x => x.ContentItemRecord.Id) + .Select(group => new {TaskId = group.Max(task => task.Id), ContentItem = _contentManager.Get(group.Key, VersionOptions.Published)}) + .OrderBy(x => x.TaskId) + .ToArray(); + + foreach (var item in contentItems) { + try { + IDocumentIndex documentIndex = ExtractDocumentIndex(item.ContentItem); + + if (documentIndex == null) { + deleteFromIndex.Add(item.ContentItem.Id); + } + else if (documentIndex.IsDirty) { + addToIndex.Add(documentIndex); + } + + indexSettings.LastIndexedId = item.TaskId; + } + catch (Exception ex) { + Logger.Warning(ex, "Unable to index content item #{0} during update", item.ContentItem.Id); + } + } + } + + // save current state of the index + indexSettings.LastIndexedUtc = _clock.UtcNow; + _appDataFolder.CreateFile(settingsFilename, indexSettings.ToXml()); + + if (deleteFromIndex.Count == 0 && addToIndex.Count == 0) { + // nothing more to do + _indexingStatus = IndexingStatus.Idle; + return false; + } + + // save new and updated documents to the index + try { + if (addToIndex.Count > 0) { + _indexProvider.Store(indexName, addToIndex); + Logger.Information("Added content items to index: {0}", addToIndex.Count); + } + } + catch (Exception ex) { + Logger.Warning(ex, "An error occured while adding a document to the index"); + } + + // removing documents from the index + try { + if (deleteFromIndex.Count > 0) { + _indexProvider.Delete(indexName, deleteFromIndex); + Logger.Information("Added content items to index: {0}", addToIndex.Count); + } + } + catch (Exception ex) { + Logger.Warning(ex, "An error occured while removing a document from the index"); + } + + return true; + } + + /// + /// Loads the settings file or create a new default one if it doesn't exist + /// + public IndexSettings LoadSettings(string indexName) + { + var indexSettings = new IndexSettings(); + var settingsFilename = GetSettingsFileName(indexName); + if (_appDataFolder.FileExists(settingsFilename)) + { + var content = _appDataFolder.ReadFile(settingsFilename); + indexSettings = IndexSettings.Parse(content); + } + + return indexSettings; + } + + /// + /// Deletes the settings file + /// + public void DeleteSettings(string indexName) { + var settingsFilename = GetSettingsFileName(indexName); + if (_appDataFolder.FileExists(settingsFilename)) { + _appDataFolder.DeleteFile(settingsFilename); + } + } + + /// + /// Creates a IDocumentIndex instance for a specific content item id. If the content + /// item is no more published, it returns null. + /// + private IDocumentIndex ExtractDocumentIndex(ContentItem contentItem) { + // ignore deleted or unpublished items + if (contentItem == null || !contentItem.IsPublished()) { + return null; + } + + // skip items from types which are not indexed + var settings = GetTypeIndexingSettings(contentItem); + if (!settings.Included) { + return null; + } + + var documentIndex = _indexProvider.New(contentItem.Id); + + // call all handlers to add content to index + _contentManager.Index(contentItem, documentIndex); + return documentIndex; + } + + private static TypeIndexing GetTypeIndexingSettings(ContentItem contentItem) { if (contentItem == null || contentItem.TypeDefinition == null || contentItem.TypeDefinition.Settings == null) { - return new TypeIndexing { Included = false }; + return new TypeIndexing {Included = false}; } return contentItem.TypeDefinition.Settings.GetModel(); } + + private string GetSettingsFileName(string indexName) { + return _appDataFolder.Combine("Sites", _shellSettings.Name, indexName + ".settings.xml"); + } + + public DateTime GetLastIndexedUtc(string indexName) { + var indexSettings = LoadSettings(indexName); + return indexSettings.LastIndexedUtc; + } + + public IndexingStatus GetIndexingStatus(string indexName) { + return _indexingStatus; + } } } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs index ccb4ff385..2726b0715 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs @@ -30,8 +30,6 @@ namespace Orchard.Indexing.Services { throw new ArgumentNullException("contentItem"); } - DeleteTasks(contentItem); - var taskRecord = new IndexingTaskRecord { CreatedUtc = _clock.UtcNow, ContentItemRecord = contentItem.Record, @@ -53,23 +51,5 @@ namespace Orchard.Indexing.Services { CreateTask(contentItem, IndexingTaskRecord.Delete); Logger.Information("Deleting index task created for [{0}:{1}]", contentItem.ContentType, contentItem.Id); } - - public DateTime GetLastTaskDateTime() { - return _repository.Table.Max(t => t.CreatedUtc) ?? new DateTime(1980, 1, 1); - } - - /// - /// Removes existing tasks for the specified content item - /// - public void DeleteTasks(ContentItem contentItem) { - var tasks = _repository - .Fetch(x => x.ContentItemRecord.Id == contentItem.Id) - .ToArray(); - foreach (var task in tasks) { - _repository.Delete(task); - } - - _repository.Flush(); - } } } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/UpdateIndexScheduler.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/UpdateIndexScheduler.cs new file mode 100644 index 000000000..878f5a65d --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/UpdateIndexScheduler.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using Orchard.Environment.Configuration; +using Orchard.Environment.Descriptor; +using Orchard.Environment.State; + +namespace Orchard.Indexing.Services { + public class UpdateIndexScheduler : IUpdateIndexScheduler, IIndexNotifierHandler { + private readonly IProcessingEngine _processingEngine; + private readonly ShellSettings _shellSettings; + private readonly IShellDescriptorManager _shellDescriptorManager; + private readonly Lazy _indexingTaskExecutor; + + public UpdateIndexScheduler( + IProcessingEngine processingEngine, + ShellSettings shellSettings, + IShellDescriptorManager shellDescriptorManager, + Lazy indexingTaskExecutor + ) { + _processingEngine = processingEngine; + _shellSettings = shellSettings; + _shellDescriptorManager = shellDescriptorManager; + _indexingTaskExecutor = indexingTaskExecutor; + } + + public void Schedule(string indexName) { + var shellDescriptor = _shellDescriptorManager.GetShellDescriptor(); + _processingEngine.AddTask( + _shellSettings, + shellDescriptor, + "IIndexNotifierHandler.UpdateIndex", + new Dictionary { { "indexName", indexName } } + ); + } + + public void UpdateIndex(string indexName) { + if(_indexingTaskExecutor.Value.UpdateIndexBatch(indexName)) { + Schedule(indexName); + } + } + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Views/Admin/Index.cshtml b/src/Orchard.Web/Modules/Orchard.Indexing/Views/Admin/Index.cshtml index c0b8ece0b..7f6b210de 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Views/Admin/Index.cshtml +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Views/Admin/Index.cshtml @@ -1,4 +1,5 @@ @model Orchard.Indexing.ViewModels.IndexViewModel +@using Orchard.Indexing.Services; @{ Layout.Title = T("Search Index Management").ToString(); } @@ -6,7 +7,7 @@
@if (Model.IndexEntry == null) {

@T("There is currently no search index")

- } else if (Model.IndexEntry.LastUpdateUtc == null) { + } else if (Model.IndexEntry.LastUpdateUtc == DateTime.MinValue) {

@T("The search index has not been built yet.")

} else { if (Model.IndexEntry.DocumentCount == 0) { @@ -21,8 +22,17 @@

@T("The search index contains the following fields: {0}.", string.Join(T(", ").Text, Model.IndexEntry.Fields))

} -

@T("The search index was last updated {0}.", Display.DateTimeRelative(dateTimeUtc: Model.IndexEntry.LastUpdateUtc.Value))

- } +

@T("The search index was last updated {0}.", Display.DateTimeRelative(dateTimeUtc: Model.IndexEntry.LastUpdateUtc))

+ + switch(Model.IndexEntry.IndexingStatus) { + case IndexingStatus.Rebuilding: + @T("The indexing process is currently being rebuilt."); + break; + case IndexingStatus.Updating: + @T("The indexing process is currently being updated."); + break; + } + }

@T("Update the search index now: ")

@Html.AntiForgeryTokenOrchard()
diff --git a/src/Orchard.Web/Modules/Orchard.jQuery/ResourceManifest.cs b/src/Orchard.Web/Modules/Orchard.jQuery/ResourceManifest.cs index 30aae3a81..5af198ffc 100644 --- a/src/Orchard.Web/Modules/Orchard.jQuery/ResourceManifest.cs +++ b/src/Orchard.Web/Modules/Orchard.jQuery/ResourceManifest.cs @@ -49,6 +49,9 @@ namespace Orchard.UI.Resources { manifest.DefineScript("jQueryUtils_TimePicker").SetUrl("ui.timepickr.js").SetVersion("0.7.0a").SetDependencies("jQueryUtils", "jQueryUI_Core", "jQueryUI_Widget"); manifest.DefineStyle("jQueryUI_Orchard").SetUrl("jquery-ui-1.7.2.custom.css").SetVersion("1.7.2"); + manifest.DefineStyle("jQueryUI_DatePicker").SetUrl("ui.datepicker.css").SetDependencies("jQueryUI_Orchard").SetVersion("1.7.2"); + manifest.DefineStyle("jQueryUtils_TimePicker").SetUrl("ui.timepickr.css"); + } } } diff --git a/src/Orchard/Environment/OrchardStarter.cs b/src/Orchard/Environment/OrchardStarter.cs index 30988def7..5f4b1e1c9 100644 --- a/src/Orchard/Environment/OrchardStarter.cs +++ b/src/Orchard/Environment/OrchardStarter.cs @@ -18,6 +18,7 @@ using Orchard.Environment.Descriptor; using Orchard.Events; using Orchard.FileSystems.AppData; using Orchard.FileSystems.Dependencies; +using Orchard.FileSystems.LockFile; using Orchard.FileSystems.VirtualPath; using Orchard.FileSystems.WebSite; using Orchard.Logging; @@ -55,6 +56,7 @@ namespace Orchard.Environment { RegisterVolatileProvider(builder); RegisterVolatileProvider(builder); + RegisterVolatileProvider(builder); RegisterVolatileProvider(builder); RegisterVolatileProvider(builder); RegisterVolatileProvider(builder); diff --git a/src/Orchard/FileSystems/LockFile/DefaultLockFileManager.cs b/src/Orchard/FileSystems/LockFile/DefaultLockFileManager.cs new file mode 100644 index 000000000..117f6f03a --- /dev/null +++ b/src/Orchard/FileSystems/LockFile/DefaultLockFileManager.cs @@ -0,0 +1,73 @@ +using System; +using System.Threading; +using Orchard.FileSystems.AppData; +using Orchard.Services; + +namespace Orchard.FileSystems.LockFile { + public class DefaultLockFileManager : ILockFileManager { + private readonly IAppDataFolder _appDataFolder; + private readonly IClock _clock; + private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim(); + public static TimeSpan Expiration { get; private set; } + + public DefaultLockFileManager(IAppDataFolder appDataFolder, IClock clock) { + _appDataFolder = appDataFolder; + _clock = clock; + Expiration = TimeSpan.FromMinutes(10); + } + + public bool TryAcquireLock(string path, ref ILockFile lockFile) { + if (!_rwLock.TryEnterWriteLock(0)) { + return false; + } + + try { + if (IsLockedImpl(path)) { + return false; + } + + lockFile = new LockFile(_appDataFolder, path, _clock.UtcNow.ToString(), _rwLock); + return true; + } + catch { + // an error occured while reading/creating the lock file + return false; + } + finally { + _rwLock.ExitWriteLock(); + } + } + + public bool IsLocked(string path) { + _rwLock.EnterWriteLock(); + + try { + return IsLockedImpl(path); + } + catch { + // an error occured while reading the file + return true; + } + finally { + _rwLock.ExitWriteLock(); + } + } + + private bool IsLockedImpl(string path) { + if (_appDataFolder.FileExists(path)) { + var content = _appDataFolder.ReadFile(path); + + DateTime creationUtc; + if (DateTime.TryParse(content, out creationUtc)) + { + // if expired the file is not removed + // it should be automatically as there is a finalizer in LockFile + // or the next taker can do it, unless it also fails, again + return creationUtc.Add(Expiration) > _clock.UtcNow; + } + } + + return false; + } + } +} diff --git a/src/Orchard/FileSystems/LockFile/ILockFile.cs b/src/Orchard/FileSystems/LockFile/ILockFile.cs new file mode 100644 index 000000000..69e5ee5cc --- /dev/null +++ b/src/Orchard/FileSystems/LockFile/ILockFile.cs @@ -0,0 +1,8 @@ +using System; + +namespace Orchard.FileSystems.LockFile +{ + public interface ILockFile : IDisposable { + void Release(); + } +} diff --git a/src/Orchard/FileSystems/LockFile/ILockFileManager.cs b/src/Orchard/FileSystems/LockFile/ILockFileManager.cs new file mode 100644 index 000000000..b1f242861 --- /dev/null +++ b/src/Orchard/FileSystems/LockFile/ILockFileManager.cs @@ -0,0 +1,26 @@ +using Orchard.Caching; + +namespace Orchard.FileSystems.LockFile { + /// + /// Abstraction for lock files creation. + /// + /// + /// All virtual paths passed in or returned are relative to "~/App_Data". + /// + public interface ILockFileManager : IVolatileProvider { + /// + /// Attempts to acquire an exclusive lock file. + /// + /// The filename of the lock file to create. + /// A reference to the lock file object if the lock is granted. + /// true if the lock is granted; otherwise, false. + bool TryAcquireLock(string path, ref ILockFile lockFile); + + /// + /// Wether a lock file is already existing. + /// + /// The filename of the lock file to test. + /// true if the lock file exists; otherwise, false. + bool IsLocked(string path); + } +} diff --git a/src/Orchard/FileSystems/LockFile/LockFile.cs b/src/Orchard/FileSystems/LockFile/LockFile.cs new file mode 100644 index 000000000..6925446a5 --- /dev/null +++ b/src/Orchard/FileSystems/LockFile/LockFile.cs @@ -0,0 +1,54 @@ +using System.Threading; +using Orchard.FileSystems.AppData; + +namespace Orchard.FileSystems.LockFile { + /// + /// Represents a Lock File acquired on the file system + /// + /// + /// The instance needs to be disposed in order to release the lock explicitly + /// + public class LockFile : ILockFile { + private readonly IAppDataFolder _appDataFolder; + private readonly string _path; + private readonly string _content; + private readonly ReaderWriterLockSlim _rwLock; + private bool _released; + + public LockFile(IAppDataFolder appDataFolder, string path, string content, ReaderWriterLockSlim rwLock) { + _appDataFolder = appDataFolder; + _path = path; + _content = content; + _rwLock = rwLock; + + // create the physical lock file + _appDataFolder.CreateFile(path, content); + } + + public void Dispose() { + Release(); + } + + public void Release() { + _rwLock.EnterWriteLock(); + + try{ + if (_released || !_appDataFolder.FileExists(_path)) { + // nothing to do, might happen if re-granted, and already released + return; + } + + _released = true; + + // check it has not been granted in the meantime + var current = _appDataFolder.ReadFile(_path); + if (current == _content) { + _appDataFolder.DeleteFile(_path); + } + } + finally { + _rwLock.ExitWriteLock(); + } + } + } +} diff --git a/src/Orchard/Indexing/IIndexProvider.cs b/src/Orchard/Indexing/IIndexProvider.cs index 2acda505f..e5413faec 100644 --- a/src/Orchard/Indexing/IIndexProvider.cs +++ b/src/Orchard/Indexing/IIndexProvider.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; namespace Orchard.Indexing { public interface IIndexProvider : ISingletonDependency { @@ -60,16 +59,6 @@ namespace Orchard.Indexing { /// A search builder instance ISearchBuilder CreateSearchBuilder(string indexName); - /// - /// Returns the date and time when the index was last processed, or null if the index doesn't exist - /// - DateTime? GetLastIndexUtc(string indexName); - - /// - /// Sets the date and time when the index was last processed - /// - void SetLastIndexUtc(string indexName, DateTime lastIndexUtc); - /// /// Returns every field available in the specified index /// diff --git a/src/Orchard/Orchard.Framework.csproj b/src/Orchard/Orchard.Framework.csproj index 755ff0377..4979e9f29 100644 --- a/src/Orchard/Orchard.Framework.csproj +++ b/src/Orchard/Orchard.Framework.csproj @@ -186,6 +186,10 @@ + + + + diff --git a/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs b/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs index 7f908ced1..aab78c650 100644 --- a/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs +++ b/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs @@ -12,16 +12,5 @@ namespace Orchard.Tasks.Indexing { /// Adds a new entry in the index task table in order to delete an existing index for the specified content item. /// void CreateDeleteIndexTask(ContentItem contentItem); - - /// - /// Returns the Date Time of the last task created - /// - DateTime GetLastTaskDateTime(); - - /// - /// Deletes all indexing tasks assigned to a specific content item - /// - /// - void DeleteTasks(ContentItem contentItem); } } \ No newline at end of file