From 919585be1acc5878708eafbdea1d2e244755d860 Mon Sep 17 00:00:00 2001 From: Sebastien Ros Date: Tue, 1 Mar 2011 22:51:09 -0800 Subject: [PATCH 1/7] Adding a new file locking service - ILockFileManager to create and check lock files - ILockFile represent an acquired lock file - Lock files expire after 10 minutes by default - Lock files are created in ~/App_Data - Default implementation provided (as it is in Orchard.Framework) --HG-- branch : indexing --- .../LockFile/LockFileManagerTests.cs | 116 ++++++++++++++++++ .../Orchard.Framework.Tests.csproj | 1 + src/Orchard/Environment/OrchardStarter.cs | 2 + .../LockFile/DefaultLockFileManager.cs | 55 +++++++++ src/Orchard/FileSystems/LockFile/ILockFile.cs | 8 ++ .../FileSystems/LockFile/ILockFileManager.cs | 26 ++++ src/Orchard/FileSystems/LockFile/LockFile.cs | 60 +++++++++ src/Orchard/Orchard.Framework.csproj | 4 + 8 files changed, 272 insertions(+) create mode 100644 src/Orchard.Tests/FileSystems/LockFile/LockFileManagerTests.cs create mode 100644 src/Orchard/FileSystems/LockFile/DefaultLockFileManager.cs create mode 100644 src/Orchard/FileSystems/LockFile/ILockFile.cs create mode 100644 src/Orchard/FileSystems/LockFile/ILockFileManager.cs create mode 100644 src/Orchard/FileSystems/LockFile/LockFile.cs diff --git a/src/Orchard.Tests/FileSystems/LockFile/LockFileManagerTests.cs b/src/Orchard.Tests/FileSystems/LockFile/LockFileManagerTests.cs new file mode 100644 index 000000000..e1493bdd4 --- /dev/null +++ b/src/Orchard.Tests/FileSystems/LockFile/LockFileManagerTests.cs @@ -0,0 +1,116 @@ +using System.IO; +using System.Linq; +using NUnit.Framework; +using Orchard.FileSystems.AppData; +using Orchard.FileSystems.LockFile; +using Orchard.Tests.Stubs; + +namespace Orchard.Tests.FileSystems.LockFile { + [TestFixture] + public class LockFileManagerTests { + private string _tempFolder; + private IAppDataFolder _appDataFolder; + private ILockFileManager _lockFileManager; + private StubClock _clock; + + public class StubAppDataFolderRoot : IAppDataFolderRoot { + public string RootPath { get; set; } + public string RootFolder { get; set; } + } + + public static IAppDataFolder CreateAppDataFolder(string tempFolder) { + var folderRoot = new StubAppDataFolderRoot {RootPath = "~/App_Data", RootFolder = tempFolder}; + var monitor = new StubVirtualPathMonitor(); + return new AppDataFolder(folderRoot, monitor); + } + + [SetUp] + public void Init() { + _tempFolder = Path.GetTempFileName(); + File.Delete(_tempFolder); + _appDataFolder = CreateAppDataFolder(_tempFolder); + + _clock = new StubClock(); + _lockFileManager = new DefaultLockFileManager(_appDataFolder, _clock); + } + + [TearDown] + public void Term() { + Directory.Delete(_tempFolder, true); + } + + [Test] + public void LockShouldBeGrantedWhenDoesNotExist() { + ILockFile lockFile = null; + var granted = _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + Assert.That(granted, Is.True); + Assert.That(lockFile, Is.Not.Null); + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + } + + [Test] + public void ExistingLockFileShouldPreventGrants() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + Assert.That(_lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile), Is.False); + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + } + + [Test] + public void ReleasingALockShouldAllowGranting() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + using (lockFile) { + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + } + + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0)); + } + + [Test] + public void ReleasingAReleasedLockShouldWork() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + + lockFile.Release(); + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0)); + + lockFile.Release(); + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0)); + } + + [Test] + public void ExpiredLockShouldBeAvailable() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + _clock.Advance(DefaultLockFileManager.Expiration); + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + } + + [Test] + public void ShouldGrantExpiredLock() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + _clock.Advance(DefaultLockFileManager.Expiration); + var granted = _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + Assert.That(granted, Is.True); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + } + } +} diff --git a/src/Orchard.Tests/Orchard.Framework.Tests.csproj b/src/Orchard.Tests/Orchard.Framework.Tests.csproj index 7c7b052db..4064da875 100644 --- a/src/Orchard.Tests/Orchard.Framework.Tests.csproj +++ b/src/Orchard.Tests/Orchard.Framework.Tests.csproj @@ -245,6 +245,7 @@ + diff --git a/src/Orchard/Environment/OrchardStarter.cs b/src/Orchard/Environment/OrchardStarter.cs index 30988def7..5f4b1e1c9 100644 --- a/src/Orchard/Environment/OrchardStarter.cs +++ b/src/Orchard/Environment/OrchardStarter.cs @@ -18,6 +18,7 @@ using Orchard.Environment.Descriptor; using Orchard.Events; using Orchard.FileSystems.AppData; using Orchard.FileSystems.Dependencies; +using Orchard.FileSystems.LockFile; using Orchard.FileSystems.VirtualPath; using Orchard.FileSystems.WebSite; using Orchard.Logging; @@ -55,6 +56,7 @@ namespace Orchard.Environment { RegisterVolatileProvider(builder); RegisterVolatileProvider(builder); + RegisterVolatileProvider(builder); RegisterVolatileProvider(builder); RegisterVolatileProvider(builder); RegisterVolatileProvider(builder); diff --git a/src/Orchard/FileSystems/LockFile/DefaultLockFileManager.cs b/src/Orchard/FileSystems/LockFile/DefaultLockFileManager.cs new file mode 100644 index 000000000..28ab27b71 --- /dev/null +++ b/src/Orchard/FileSystems/LockFile/DefaultLockFileManager.cs @@ -0,0 +1,55 @@ +using System; +using Orchard.FileSystems.AppData; +using Orchard.Services; + +namespace Orchard.FileSystems.LockFile { + public class DefaultLockFileManager : ILockFileManager { + private readonly IAppDataFolder _appDataFolder; + private readonly IClock _clock; + + public static TimeSpan Expiration { get; private set; } + + public DefaultLockFileManager(IAppDataFolder appDataFolder, IClock clock) { + _appDataFolder = appDataFolder; + _clock = clock; + Expiration = TimeSpan.FromMinutes(10); + } + + public bool TryAcquireLock(string path, ref ILockFile lockFile) { + try { + if(IsLocked(path)) { + return false; + } + + lockFile = new LockFile(_appDataFolder, path, _clock.UtcNow.ToString()); + return true; + } + catch { + // an error occured while reading/creating the lock file + return false; + } + } + + public bool IsLocked(string path) { + try { + if (_appDataFolder.FileExists(path)) { + var content = _appDataFolder.ReadFile(path); + + DateTime creationUtc; + if (DateTime.TryParse(content, out creationUtc)) { + // if expired the file is not removed + // it should be automatically as there is a finalizer in LockFile + // or the next taker can do it, unless it also fails, again + return creationUtc.Add(Expiration) > _clock.UtcNow; + } + } + } + catch { + // an error occured while reading the file + return true; + } + + return false; + } + } +} diff --git a/src/Orchard/FileSystems/LockFile/ILockFile.cs b/src/Orchard/FileSystems/LockFile/ILockFile.cs new file mode 100644 index 000000000..69e5ee5cc --- /dev/null +++ b/src/Orchard/FileSystems/LockFile/ILockFile.cs @@ -0,0 +1,8 @@ +using System; + +namespace Orchard.FileSystems.LockFile +{ + public interface ILockFile : IDisposable { + void Release(); + } +} diff --git a/src/Orchard/FileSystems/LockFile/ILockFileManager.cs b/src/Orchard/FileSystems/LockFile/ILockFileManager.cs new file mode 100644 index 000000000..b1f242861 --- /dev/null +++ b/src/Orchard/FileSystems/LockFile/ILockFileManager.cs @@ -0,0 +1,26 @@ +using Orchard.Caching; + +namespace Orchard.FileSystems.LockFile { + /// + /// Abstraction for lock files creation. + /// + /// + /// All virtual paths passed in or returned are relative to "~/App_Data". + /// + public interface ILockFileManager : IVolatileProvider { + /// + /// Attempts to acquire an exclusive lock file. + /// + /// The filename of the lock file to create. + /// A reference to the lock file object if the lock is granted. + /// true if the lock is granted; otherwise, false. + bool TryAcquireLock(string path, ref ILockFile lockFile); + + /// + /// Wether a lock file is already existing. + /// + /// The filename of the lock file to test. + /// true if the lock file exists; otherwise, false. + bool IsLocked(string path); + } +} diff --git a/src/Orchard/FileSystems/LockFile/LockFile.cs b/src/Orchard/FileSystems/LockFile/LockFile.cs new file mode 100644 index 000000000..1ecb594e3 --- /dev/null +++ b/src/Orchard/FileSystems/LockFile/LockFile.cs @@ -0,0 +1,60 @@ +using System; +using Orchard.FileSystems.AppData; + +namespace Orchard.FileSystems.LockFile { + /// + /// Represents a Lock File acquire on the file system + /// + public class LockFile : ILockFile { + private readonly IAppDataFolder _appDataFolder; + private readonly string _path; + private readonly string _content; + private bool _released; + + public LockFile(IAppDataFolder appDataFolder, string path, string content) { + _appDataFolder = appDataFolder; + _path = path; + _content = content; + + // create the physical lock file + _appDataFolder.CreateFile(path, content); + } + + public void Dispose() { + // dispose both managed and unmanaged resources + Dispose(true); + + // don't call the finalizer if dispose is called + GC.SuppressFinalize(this); + } + + public void Release() { + Dispose(true); + } + + protected virtual void Dispose(bool disposing) { + if(disposing) { + // release managed code here + // nothing right now, just a placeholder to preserve the pattern + } + + if (_released || !_appDataFolder.FileExists(_path)) { + // nothing to do, night happen if re-granted, and already released + return; + } + + _released = true; + + // check it has not been granted in the meantime + var current = _appDataFolder.ReadFile(_path); + if (current == _content) { + _appDataFolder.DeleteFile(_path); + } + } + + ~LockFile() { + // dispose unmanaged resources (file) + Dispose(false); + } + } +} diff --git a/src/Orchard/Orchard.Framework.csproj b/src/Orchard/Orchard.Framework.csproj index 32067969b..92bb5bf98 100644 --- a/src/Orchard/Orchard.Framework.csproj +++ b/src/Orchard/Orchard.Framework.csproj @@ -183,6 +183,10 @@ + + + + From f50e47bfddc16cb45513181231660004097292a8 Mon Sep 17 00:00:00 2001 From: Sebastien Ros Date: Wed, 2 Mar 2011 12:33:50 -0800 Subject: [PATCH 2/7] Removing the finalizer and implementing thread safety --HG-- branch : indexing --- .../LockFile/LockFileManagerTests.cs | 109 +++++++++++++++++- .../LockFile/DefaultLockFileManager.cs | 46 +++++--- src/Orchard/FileSystems/LockFile/LockFile.cs | 56 ++++----- 3 files changed, 165 insertions(+), 46 deletions(-) diff --git a/src/Orchard.Tests/FileSystems/LockFile/LockFileManagerTests.cs b/src/Orchard.Tests/FileSystems/LockFile/LockFileManagerTests.cs index e1493bdd4..1c6a41894 100644 --- a/src/Orchard.Tests/FileSystems/LockFile/LockFileManagerTests.cs +++ b/src/Orchard.Tests/FileSystems/LockFile/LockFileManagerTests.cs @@ -1,5 +1,8 @@ -using System.IO; +using System; +using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Threading; using NUnit.Framework; using Orchard.FileSystems.AppData; using Orchard.FileSystems.LockFile; @@ -91,6 +94,20 @@ namespace Orchard.Tests.FileSystems.LockFile { Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0)); } + [Test] + public void DisposingLockShouldReleaseIt() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile); + + using (lockFile) { + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); + } + + Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.False); + Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(0)); + } + [Test] public void ExpiredLockShouldBeAvailable() { ILockFile lockFile = null; @@ -112,5 +129,95 @@ namespace Orchard.Tests.FileSystems.LockFile { Assert.That(granted, Is.True); Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1)); } + + private static int _lockCount; + private static readonly object _synLock = new object(); + + [Test] + public void AcquiringLockShouldBeThreadSafe() { + var threads = new List(); + for(var i=0; i<10; i++) { + var t = new Thread(PlayWithAcquire); + t.Start(); + threads.Add(t); + } + + threads.ForEach(t => t.Join()); + Assert.That(_lockCount, Is.EqualTo(0)); + } + + [Test] + public void IsLockedShouldBeThreadSafe() { + var threads = new List(); + for (var i = 0; i < 10; i++) + { + var t = new Thread(PlayWithIsLocked); + t.Start(); + threads.Add(t); + } + + threads.ForEach(t => t.Join()); + Assert.That(_lockCount, Is.EqualTo(0)); + } + + private void PlayWithAcquire() { + var r = new Random(DateTime.Now.Millisecond); + ILockFile lockFile = null; + + // loop until the lock has been acquired + for (;;) { + if (!_lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile)) { + continue; + } + + lock (_synLock) { + _lockCount++; + Assert.That(_lockCount, Is.EqualTo(1)); + } + + // keep the lock for a certain time + Thread.Sleep(r.Next(200)); + lock (_synLock) { + _lockCount--; + Assert.That(_lockCount, Is.EqualTo(0)); + } + + lockFile.Release(); + return; + } + } + + private void PlayWithIsLocked() { + var r = new Random(DateTime.Now.Millisecond); + ILockFile lockFile = null; + const string path = "foo.txt.lock"; + + // loop until the lock has been acquired + for (;;) { + if(_lockFileManager.IsLocked(path)) { + continue; + } + + if (!_lockFileManager.TryAcquireLock(path, ref lockFile)) { + continue; + } + + lock (_synLock) { + _lockCount++; + Assert.That(_lockCount, Is.EqualTo(1)); + } + + // keep the lock for a certain time + Thread.Sleep(r.Next(200)); + lock (_synLock) { + _lockCount--; + Assert.That(_lockCount, Is.EqualTo(0)); + } + + lockFile.Release(); + return; + } + } + } } diff --git a/src/Orchard/FileSystems/LockFile/DefaultLockFileManager.cs b/src/Orchard/FileSystems/LockFile/DefaultLockFileManager.cs index 28ab27b71..117f6f03a 100644 --- a/src/Orchard/FileSystems/LockFile/DefaultLockFileManager.cs +++ b/src/Orchard/FileSystems/LockFile/DefaultLockFileManager.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using Orchard.FileSystems.AppData; using Orchard.Services; @@ -6,7 +7,7 @@ namespace Orchard.FileSystems.LockFile { public class DefaultLockFileManager : ILockFileManager { private readonly IAppDataFolder _appDataFolder; private readonly IClock _clock; - + private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim(); public static TimeSpan Expiration { get; private set; } public DefaultLockFileManager(IAppDataFolder appDataFolder, IClock clock) { @@ -16,38 +17,55 @@ namespace Orchard.FileSystems.LockFile { } public bool TryAcquireLock(string path, ref ILockFile lockFile) { + if (!_rwLock.TryEnterWriteLock(0)) { + return false; + } + try { - if(IsLocked(path)) { + if (IsLockedImpl(path)) { return false; } - lockFile = new LockFile(_appDataFolder, path, _clock.UtcNow.ToString()); + lockFile = new LockFile(_appDataFolder, path, _clock.UtcNow.ToString(), _rwLock); return true; } catch { // an error occured while reading/creating the lock file return false; } + finally { + _rwLock.ExitWriteLock(); + } } public bool IsLocked(string path) { - try { - if (_appDataFolder.FileExists(path)) { - var content = _appDataFolder.ReadFile(path); + _rwLock.EnterWriteLock(); - DateTime creationUtc; - if (DateTime.TryParse(content, out creationUtc)) { - // if expired the file is not removed - // it should be automatically as there is a finalizer in LockFile - // or the next taker can do it, unless it also fails, again - return creationUtc.Add(Expiration) > _clock.UtcNow; - } - } + try { + return IsLockedImpl(path); } catch { // an error occured while reading the file return true; } + finally { + _rwLock.ExitWriteLock(); + } + } + + private bool IsLockedImpl(string path) { + if (_appDataFolder.FileExists(path)) { + var content = _appDataFolder.ReadFile(path); + + DateTime creationUtc; + if (DateTime.TryParse(content, out creationUtc)) + { + // if expired the file is not removed + // it should be automatically as there is a finalizer in LockFile + // or the next taker can do it, unless it also fails, again + return creationUtc.Add(Expiration) > _clock.UtcNow; + } + } return false; } diff --git a/src/Orchard/FileSystems/LockFile/LockFile.cs b/src/Orchard/FileSystems/LockFile/LockFile.cs index 1ecb594e3..6925446a5 100644 --- a/src/Orchard/FileSystems/LockFile/LockFile.cs +++ b/src/Orchard/FileSystems/LockFile/LockFile.cs @@ -1,60 +1,54 @@ -using System; +using System.Threading; using Orchard.FileSystems.AppData; namespace Orchard.FileSystems.LockFile { /// - /// Represents a Lock File acquire on the file system + /// Represents a Lock File acquired on the file system /// + /// + /// The instance needs to be disposed in order to release the lock explicitly + /// public class LockFile : ILockFile { private readonly IAppDataFolder _appDataFolder; private readonly string _path; private readonly string _content; + private readonly ReaderWriterLockSlim _rwLock; private bool _released; - public LockFile(IAppDataFolder appDataFolder, string path, string content) { + public LockFile(IAppDataFolder appDataFolder, string path, string content, ReaderWriterLockSlim rwLock) { _appDataFolder = appDataFolder; _path = path; _content = content; + _rwLock = rwLock; // create the physical lock file _appDataFolder.CreateFile(path, content); } public void Dispose() { - // dispose both managed and unmanaged resources - Dispose(true); - - // don't call the finalizer if dispose is called - GC.SuppressFinalize(this); + Release(); } public void Release() { - Dispose(true); - } + _rwLock.EnterWriteLock(); - protected virtual void Dispose(bool disposing) { - if(disposing) { - // release managed code here - // nothing right now, just a placeholder to preserve the pattern + try{ + if (_released || !_appDataFolder.FileExists(_path)) { + // nothing to do, might happen if re-granted, and already released + return; + } + + _released = true; + + // check it has not been granted in the meantime + var current = _appDataFolder.ReadFile(_path); + if (current == _content) { + _appDataFolder.DeleteFile(_path); + } } - - if (_released || !_appDataFolder.FileExists(_path)) { - // nothing to do, night happen if re-granted, and already released - return; + finally { + _rwLock.ExitWriteLock(); } - - _released = true; - - // check it has not been granted in the meantime - var current = _appDataFolder.ReadFile(_path); - if (current == _content) { - _appDataFolder.DeleteFile(_path); - } - } - - ~LockFile() { - // dispose unmanaged resources (file) - Dispose(false); } } } From 32efbc19ccbd6c308ea95dad997bc3e147cc92f4 Mon Sep 17 00:00:00 2001 From: Sebastien Ros Date: Thu, 3 Mar 2011 14:19:02 -0800 Subject: [PATCH 3/7] Using lock files instead of IndexSynLock --HG-- branch : indexing --- .../Indexing/IndexingTaskExecutorTests.cs | 37 ++++++++++++----- .../Lucene/Services/LuceneIndexProvider.cs | 8 ++-- .../Orchard.Indexing/Orchard.Indexing.csproj | 1 - .../Orchard.Indexing/Services/IndexSynLock.cs | 25 ------------ .../Services/IndexingTaskExecutor.cs | 40 +++++++++++++------ 5 files changed, 59 insertions(+), 52 deletions(-) delete mode 100644 src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexSynLock.cs diff --git a/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs b/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs index 664b10f06..2c7b5bbc7 100644 --- a/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs +++ b/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs @@ -19,12 +19,14 @@ using Orchard.Environment; using Orchard.Environment.Configuration; using Orchard.Environment.Extensions; using Orchard.FileSystems.AppData; +using Orchard.FileSystems.LockFile; using Orchard.Indexing; using Orchard.Indexing.Handlers; using Orchard.Indexing.Models; using Orchard.Indexing.Services; using Orchard.Logging; using Orchard.Security; +using Orchard.Services; using Orchard.Tasks.Indexing; using Orchard.Tests.FileSystems.AppData; using Orchard.Tests.Stubs; @@ -38,10 +40,12 @@ namespace Orchard.Tests.Modules.Indexing { private IContentManager _contentManager; private Mock _contentDefinitionManager; private StubLogger _logger; - private const string IndexName = "Search"; + private ILockFileManager _lockFileManager; + private const string IndexName = "Search"; private readonly string _basePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); - + + [TestFixtureTearDown] public void Clean() { if (Directory.Exists(_basePath)) { @@ -62,7 +66,6 @@ namespace Orchard.Tests.Modules.Indexing { builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); - builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterInstance(_contentDefinitionManager.Object); @@ -80,6 +83,9 @@ namespace Orchard.Tests.Modules.Indexing { builder.RegisterType().As(); builder.RegisterType().As(); + builder.RegisterType().As(); + builder.RegisterInstance(_clock = new StubClock()); + // setting up a ShellSettings instance _shellSettings = new ShellSettings { Name = "My Site" }; builder.RegisterInstance(_shellSettings).As(); @@ -100,7 +106,7 @@ namespace Orchard.Tests.Modules.Indexing { public override void Init() { base.Init(); - + _lockFileManager = _container.Resolve(); _provider = _container.Resolve(); _indexNotifier = _container.Resolve(); _contentManager = _container.Resolve(); @@ -116,10 +122,6 @@ namespace Orchard.Tests.Modules.Indexing { .Returns(thingType); } - private string[] Indexes() { - return new DirectoryInfo(Path.Combine(_basePath, "Sites", "My Site", "Indexes")).GetDirectories().Select(d => d.Name).ToArray(); - } - [Test] public void IndexShouldBeEmptyWhenThereIsNoContent() { _indexNotifier.UpdateIndex(IndexName); @@ -130,7 +132,7 @@ namespace Orchard.Tests.Modules.Indexing { } [Test] - public void ShouldIngoreNonIndexableContentWhenRebuildingTheIndex() { + public void ShouldIgnoreNonIndexableContentWhenRebuildingTheIndex() { var alphaType = new ContentTypeDefinitionBuilder() .Named("alpha") .Build(); @@ -195,6 +197,21 @@ namespace Orchard.Tests.Modules.Indexing { Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Rebuild index started")); } + [Test] + public void IndexingTaskExecutorShouldBeReEntrant() { + ILockFile lockFile = null; + _lockFileManager.TryAcquireLock("Sites/My Site/Search.settings.xml.lock", ref lockFile); + using (lockFile) { + _indexNotifier.UpdateIndex(IndexName); + Assert.That(_logger.LogEntries.Count, Is.EqualTo(1)); + Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Index was requested but was already running")); + } + + _logger.LogEntries.Clear(); + _indexNotifier.UpdateIndex(IndexName); + Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Index was requested but was already running")); + } + [Test] public void ShouldUpdateTheIndexWhenContentIsUnPublished() { _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; @@ -293,7 +310,7 @@ namespace Orchard.Tests.Modules.Indexing { } public void Log(LogLevel level, Exception exception, string format, params object[] args) { - LogEntries.Add(new LogEntry() { + LogEntries.Add(new LogEntry { LogArgs = args, LogException = exception, LogFormat = format, diff --git a/src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs b/src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs index d89e8e32f..2ace88b61 100644 --- a/src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs +++ b/src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs @@ -206,10 +206,6 @@ namespace Lucene.Services { return new LuceneSearchBuilder(GetDirectory(indexName)) { Logger = Logger }; } - private string GetSettingsFileName(string indexName) { - return _appDataFolder.MapPath(_appDataFolder.Combine(_basePath, indexName + ".settings.xml")); - } - public DateTime? GetLastIndexUtc(string indexName) { var settingsFileName = GetSettingsFileName(indexName); @@ -254,5 +250,9 @@ namespace Lucene.Services { reader.Close(); } } + + private string GetSettingsFileName(string indexName) { + return _appDataFolder.MapPath(_appDataFolder.Combine(_basePath, indexName + ".settings.xml")); + } } } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj b/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj index be7ea6005..aff4ca59d 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj @@ -60,7 +60,6 @@ - diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexSynLock.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexSynLock.cs deleted file mode 100644 index 7b545ccaf..000000000 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexSynLock.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Collections.Generic; - -namespace Orchard.Indexing.Services { - public interface IIndexSynLock : ISingletonDependency { - object GetSynLock(string indexName); - } - - public class IndexSynLock : IIndexSynLock { - private readonly Dictionary _synLocks; - private readonly object _synLock = new object(); - - public IndexSynLock() { - _synLocks =new Dictionary(); - } - - public object GetSynLock(string indexName) { - lock(_synLock) { - if(!_synLocks.ContainsKey(indexName)) { - _synLocks[indexName] = new object(); - } - return _synLocks[indexName]; - } - } - } -} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs index 3747fe997..e9a969d17 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs @@ -4,6 +4,9 @@ using System.Linq; using JetBrains.Annotations; using Orchard.ContentManagement; using Orchard.Data; +using Orchard.Environment.Configuration; +using Orchard.FileSystems.AppData; +using Orchard.FileSystems.LockFile; using Orchard.Indexing.Models; using Orchard.Indexing.Settings; using Orchard.Logging; @@ -14,6 +17,10 @@ namespace Orchard.Indexing.Services { /// /// Contains the logic which is regularly executed to retrieve index information from multiple content handlers. /// + /// + /// This class is synchronized using a lock file as both command line and web workers can potentially use it, + /// and singleton locks would not be shared accross those two. + /// [UsedImplicitly] public class IndexingTaskExecutor : IIndexNotifierHandler { private readonly IClock _clock; @@ -22,7 +29,9 @@ namespace Orchard.Indexing.Services { private readonly IIndexManager _indexManager; private readonly IIndexingTaskManager _indexingTaskManager; private readonly IContentManager _contentManager; - private readonly IIndexSynLock _indexSynLock; + private readonly IAppDataFolder _appDataFolder; + private readonly ShellSettings _shellSettings; + private readonly ILockFileManager _lockFileManager; public IndexingTaskExecutor( IClock clock, @@ -30,28 +39,34 @@ namespace Orchard.Indexing.Services { IIndexManager indexManager, IIndexingTaskManager indexingTaskManager, IContentManager contentManager, - IIndexSynLock indexSynLock) { + IAppDataFolder appDataFolder, + ShellSettings shellSettings, + ILockFileManager lockFileManager) { _clock = clock; _repository = repository; _indexManager = indexManager; _indexingTaskManager = indexingTaskManager; _contentManager = contentManager; - _indexSynLock = indexSynLock; + _appDataFolder = appDataFolder; + _shellSettings = shellSettings; + _lockFileManager = lockFileManager; + Logger = NullLogger.Instance; } public ILogger Logger { get; set; } public void UpdateIndex(string indexName) { - var synLock = _indexSynLock.GetSynLock(indexName); + ILockFile lockFile = null; + var settingsFilename = GetSettingsFileName(indexName); + var lockFilename = settingsFilename + ".lock"; - if (!System.Threading.Monitor.TryEnter(synLock)) { + if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile)) { Logger.Information("Index was requested but was already running"); return; } - try { - + using (lockFile) { if (!_indexManager.HasIndexProvider()) { return; } @@ -99,8 +114,8 @@ namespace Orchard.Indexing.Services { // retrieve not yet processed tasks var taskRecords = lastIndexUtc == null - ? _repository.Fetch(x => true).ToArray() - : _repository.Fetch(x => x.CreatedUtc >= lastIndexUtc).ToArray(); // CreatedUtc and lastIndexUtc might be equal if a content item is created in a background task + ? _repository.Fetch(x => true).ToArray() + : _repository.Fetch(x => x.CreatedUtc >= lastIndexUtc).ToArray(); // CreatedUtc and lastIndexUtc might be equal if a content item is created in a background task // nothing to do ?))) if (taskRecords.Length + updateIndexDocuments.Count == 0) { @@ -158,9 +173,6 @@ namespace Orchard.Indexing.Services { } } } - finally { - System.Threading.Monitor.Exit(synLock); - } } static TypeIndexing GetTypeIndexingSettings(ContentItem contentItem) { @@ -171,5 +183,9 @@ namespace Orchard.Indexing.Services { } return contentItem.TypeDefinition.Settings.GetModel(); } + + private string GetSettingsFileName(string indexName) { + return _appDataFolder.Combine("Sites", _shellSettings.Name, indexName + ".settings.xml"); + } } } From 6d3dffd77ec980ee214a212acab6bca0483fe4f1 Mon Sep 17 00:00:00 2001 From: Sebastien Ros Date: Fri, 4 Mar 2011 11:05:19 -0800 Subject: [PATCH 4/7] Implementing revised indexing loop design --HG-- branch : indexing --- .../Indexing/IndexingTaskExecutorTests.cs | 62 +--- .../Indexing/LuceneIndexProviderTests.cs | 19 +- .../Lucene/Services/LuceneIndexProvider.cs | 42 --- .../Commands/IndexingCommands.cs | 7 +- .../Controllers/AdminController.cs | 9 +- .../Orchard.Indexing/Models/IndexSettings.cs | 57 ++++ .../Orchard.Indexing/Orchard.Indexing.csproj | 6 + .../Services/IIndexService.cs | 9 +- .../Services/IIndexStatisticsProvider.cs | 13 + .../Orchard.Indexing/Services/IndexService.cs | 38 +-- .../Services/IndexingTaskExecutor.cs | 268 +++++++++++------- .../Services/IndexingTaskManager.cs | 16 -- .../Orchard.Indexing/Views/Admin/Index.cshtml | 16 +- src/Orchard/Indexing/IIndexProvider.cs | 13 +- .../Tasks/Indexing/IIndexingTaskManager.cs | 6 - 15 files changed, 306 insertions(+), 275 deletions(-) create mode 100644 src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs create mode 100644 src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexStatisticsProvider.cs diff --git a/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs b/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs index 2c7b5bbc7..7e6549a4a 100644 --- a/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs +++ b/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs @@ -26,7 +26,6 @@ using Orchard.Indexing.Models; using Orchard.Indexing.Services; using Orchard.Logging; using Orchard.Security; -using Orchard.Services; using Orchard.Tasks.Indexing; using Orchard.Tests.FileSystems.AppData; using Orchard.Tests.Stubs; @@ -84,7 +83,6 @@ namespace Orchard.Tests.Modules.Indexing { builder.RegisterType().As(); builder.RegisterType().As(); - builder.RegisterInstance(_clock = new StubClock()); // setting up a ShellSettings instance _shellSettings = new ShellSettings { Name = "My Site" }; @@ -126,9 +124,6 @@ namespace Orchard.Tests.Modules.Indexing { public void IndexShouldBeEmptyWhenThereIsNoContent() { _indexNotifier.UpdateIndex(IndexName); Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0)); - Assert.That(_logger.LogEntries.Count(), Is.EqualTo(2)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Index update requested, nothing to do")); } [Test] @@ -145,9 +140,6 @@ namespace Orchard.Tests.Modules.Indexing { _indexNotifier.UpdateIndex(IndexName); Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0)); - Assert.That(_logger.LogEntries.Count(), Is.EqualTo(2)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Index update requested, nothing to do")); } [Test] @@ -165,9 +157,6 @@ namespace Orchard.Tests.Modules.Indexing { _indexNotifier.UpdateIndex(IndexName); Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0)); - Assert.That(_logger.LogEntries.Count(), Is.EqualTo(2)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Index update requested, nothing to do")); } [Test] @@ -177,10 +166,6 @@ namespace Orchard.Tests.Modules.Indexing { _indexNotifier.UpdateIndex(IndexName); Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); - Assert.That(_logger.LogEntries.Count(), Is.EqualTo(3)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Processing {0} indexing tasks")); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Added content items to index: {0}")); } [Test] @@ -188,82 +173,47 @@ namespace Orchard.Tests.Modules.Indexing { _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; _indexNotifier.UpdateIndex(IndexName); Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - _logger.Clear(); + + // there should be nothing done + _indexNotifier.UpdateIndex(IndexName); _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; _indexNotifier.UpdateIndex(IndexName); Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2)); - Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Rebuild index started")); } [Test] - public void IndexingTaskExecutorShouldBeReEntrant() { + public void IndexingTaskExecutorShouldNotBeReEntrant() { ILockFile lockFile = null; _lockFileManager.TryAcquireLock("Sites/My Site/Search.settings.xml.lock", ref lockFile); using (lockFile) { _indexNotifier.UpdateIndex(IndexName); Assert.That(_logger.LogEntries.Count, Is.EqualTo(1)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Index was requested but was already running")); + Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Index was requested but is already running")); } _logger.LogEntries.Clear(); _indexNotifier.UpdateIndex(IndexName); - Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Index was requested but was already running")); + Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Index was requested but is already running")); } [Test] public void ShouldUpdateTheIndexWhenContentIsUnPublished() { _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; - _clock.Advance(TimeSpan.FromSeconds(1)); _indexNotifier.UpdateIndex(IndexName); Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - _logger.Clear(); var content = _contentManager.Create(ThingDriver.ContentTypeName); content.Text = "Lorem ipsum"; - _clock.Advance(TimeSpan.FromSeconds(1)); _indexNotifier.UpdateIndex(IndexName); Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2)); - Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Rebuild index started")); - _clock.Advance(TimeSpan.FromSeconds(1)); _contentManager.Unpublish(content.ContentItem); - _clock.Advance(TimeSpan.FromSeconds(1)); _indexNotifier.UpdateIndex(IndexName); Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); - Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Rebuild index started")); - } - - [Test] - public void ShouldRemoveFromIndexEvenIfPublishedAndUnpublishedInTheSameSecond() { - // This test is to ensure that when a task is created, all previous tasks for the same content item - // are also removed, and thus that multiple tasks don't conflict while updating the index - - _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; - _clock.Advance(TimeSpan.FromSeconds(1)); - - _indexNotifier.UpdateIndex(IndexName); - Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); - Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Rebuild index started")); - _logger.Clear(); - - var content = _contentManager.Create(ThingDriver.ContentTypeName); - content.Text = "Lorem ipsum"; - - _indexNotifier.UpdateIndex(IndexName); - Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2)); - Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Rebuild index started")); - - _contentManager.Unpublish(content.ContentItem); - - _indexNotifier.UpdateIndex(IndexName); - Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); - Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Rebuild index started")); } #region Stubs diff --git a/src/Orchard.Tests.Modules/Indexing/LuceneIndexProviderTests.cs b/src/Orchard.Tests.Modules/Indexing/LuceneIndexProviderTests.cs index 2d69c727e..2f170757e 100644 --- a/src/Orchard.Tests.Modules/Indexing/LuceneIndexProviderTests.cs +++ b/src/Orchard.Tests.Modules/Indexing/LuceneIndexProviderTests.cs @@ -7,7 +7,6 @@ using NUnit.Framework; using Orchard.Environment.Configuration; using Orchard.FileSystems.AppData; using Orchard.Indexing; -using Orchard.Indexing.Services; using Orchard.Tests.FileSystems.AppData; namespace Orchard.Tests.Modules.Indexing { @@ -198,18 +197,6 @@ namespace Orchard.Tests.Modules.Indexing { Assert.That(searchBuilder.Get(3).ContentItemId, Is.EqualTo(3)); } - [Test] - public void ProviderShouldStoreSettings() { - _provider.CreateIndex("default"); - Assert.That(_provider.GetLastIndexUtc("default"), Is.Null); - - _provider.SetLastIndexUtc("default", new DateTime(2010, 1, 1, 1, 1, 1, 1)); - Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(new DateTime(2010, 1, 1, 1, 1, 1, 0))); - - _provider.SetLastIndexUtc("default", new DateTime(1901, 1, 1, 1, 1, 1, 1)); - Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(LuceneIndexProvider.DefaultMinDateTime)); - } - [Test] public void IsEmptyShouldBeTrueForNoneExistingIndexes() { _provider.IsEmpty("dummy"); @@ -238,9 +225,7 @@ namespace Orchard.Tests.Modules.Indexing { [Test] public void IsDirtyShouldBeTrueWhenIndexIsModified() { - IDocumentIndex doc; - - doc = _provider.New(1); + IDocumentIndex doc = _provider.New(1); doc.Add("foo", "value"); Assert.That(doc.IsDirty, Is.True); @@ -281,7 +266,7 @@ namespace Orchard.Tests.Modules.Indexing { Assert.That(searchBuilder.Get(11).ContentItemId, Is.EqualTo(11)); Assert.That(searchBuilder.Get(111).ContentItemId, Is.EqualTo(111)); - _provider.Delete("default", new int[] {1, 11, 111 }); + _provider.Delete("default", new [] {1, 11, 111 }); Assert.That(searchBuilder.Get(1), Is.Null); Assert.That(searchBuilder.Get(11), Is.Null); diff --git a/src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs b/src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs index 2ace88b61..0d6e3719f 100644 --- a/src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs +++ b/src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs @@ -9,7 +9,6 @@ using Lucene.Net.Documents; using Lucene.Net.Index; using Lucene.Net.Search; using Lucene.Net.Store; -using System.Xml.Linq; using Orchard.Environment.Configuration; using Orchard.FileSystems.AppData; using Orchard.Indexing; @@ -29,8 +28,6 @@ namespace Lucene.Services { private readonly Analyzer _analyzer ; private readonly string _basePath; public static readonly DateTime DefaultMinDateTime = new DateTime(1980, 1, 1); - public static readonly string Settings = "Settings"; - public static readonly string LastIndexUtc = "LastIndexedUtc"; public LuceneIndexProvider(IAppDataFolder appDataFolder, ShellSettings shellSettings) { _appDataFolder = appDataFolder; @@ -123,11 +120,6 @@ namespace Lucene.Services { public void DeleteIndex(string indexName) { new DirectoryInfo(_appDataFolder.MapPath(_appDataFolder.Combine(_basePath, indexName))) .Delete(true); - - var settingsFileName = GetSettingsFileName(indexName); - if (File.Exists(settingsFileName)) { - File.Delete(settingsFileName); - } } public void Store(string indexName, IDocumentIndex indexDocument) { @@ -206,36 +198,6 @@ namespace Lucene.Services { return new LuceneSearchBuilder(GetDirectory(indexName)) { Logger = Logger }; } - public DateTime? GetLastIndexUtc(string indexName) { - var settingsFileName = GetSettingsFileName(indexName); - - if (!File.Exists(settingsFileName)) - return null; - - return DateTime.Parse(XDocument.Load(settingsFileName).Descendants(LastIndexUtc).First().Value); - } - - public void SetLastIndexUtc(string indexName, DateTime lastIndexUtc) { - if ( lastIndexUtc < DefaultMinDateTime ) { - lastIndexUtc = DefaultMinDateTime; - } - - XDocument doc; - var settingsFileName = GetSettingsFileName(indexName); - if ( !File.Exists(settingsFileName) ) { - EnsureDirectoryExists(); - doc = new XDocument( - new XElement(Settings, - new XElement(LastIndexUtc, lastIndexUtc.ToString("s")))); - } - else { - doc = XDocument.Load(settingsFileName); - doc.Element(Settings).Element(LastIndexUtc).Value = lastIndexUtc.ToString("s"); - } - - doc.Save(settingsFileName); - } - public IEnumerable GetFields(string indexName) { if ( !Exists(indexName) ) { return Enumerable.Empty(); @@ -250,9 +212,5 @@ namespace Lucene.Services { reader.Close(); } } - - private string GetSettingsFileName(string indexName) { - return _appDataFolder.MapPath(_appDataFolder.Combine(_basePath, indexName + ".settings.xml")); - } } } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs index b8b702044..18c95a234 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Linq; using Orchard.Commands; using Orchard.ContentManagement; @@ -35,7 +34,7 @@ namespace Orchard.Indexing.Commands { [CommandName("index update")] [CommandHelp("index update\r\n\t" + "Updates the search index")] public string Update() { - _indexingService.UpdateIndex(); + _indexingService.UpdateIndex(SearchIndexName); return T("Index is now being updated...").Text; } @@ -43,8 +42,8 @@ namespace Orchard.Indexing.Commands { [CommandName("index rebuild")] [CommandHelp("index rebuild \r\n\t" + "Rebuilds the search index")] public string Rebuild() { - _indexingService.RebuildIndex(); - _indexingService.UpdateIndex(); + _indexingService.RebuildIndex(SearchIndexName); + _indexingService.UpdateIndex(SearchIndexName); return T("Index is now being rebuilt...").Text; } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs index f593f0206..2a7bccc7c 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs @@ -8,6 +8,7 @@ using Orchard.Indexing.ViewModels; namespace Orchard.Indexing.Controllers { public class AdminController : Controller { private readonly IIndexingService _indexingService; + private const string DefaultIndexName = "Search"; public AdminController(IIndexingService indexingService, IOrchardServices services) { _indexingService = indexingService; @@ -19,7 +20,7 @@ namespace Orchard.Indexing.Controllers { public Localizer T { get; set; } public ActionResult Index() { - var viewModel = new IndexViewModel { IndexEntry = _indexingService.GetIndexEntry() }; + var viewModel = new IndexViewModel { IndexEntry = _indexingService.GetIndexEntry(DefaultIndexName) }; if (viewModel.IndexEntry == null) Services.Notifier.Information(T("There is no search index to manage for this site.")); @@ -32,7 +33,7 @@ namespace Orchard.Indexing.Controllers { if (!Services.Authorizer.Authorize(StandardPermissions.SiteOwner, T("Not allowed to manage the search index."))) return new HttpUnauthorizedResult(); - _indexingService.UpdateIndex(); + _indexingService.UpdateIndex(DefaultIndexName); return RedirectToAction("Index"); } @@ -42,8 +43,8 @@ namespace Orchard.Indexing.Controllers { if (!Services.Authorizer.Authorize(StandardPermissions.SiteOwner, T("Not allowed to manage the search index."))) return new HttpUnauthorizedResult(); - _indexingService.RebuildIndex(); - _indexingService.UpdateIndex(); + _indexingService.RebuildIndex(DefaultIndexName); + _indexingService.UpdateIndex(DefaultIndexName); return RedirectToAction("Index"); } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs new file mode 100644 index 000000000..c0b3e0e67 --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs @@ -0,0 +1,57 @@ +using System; +using System.Linq; +using System.Xml.Linq; + +namespace Orchard.Indexing.Models +{ + public enum IndexingMode { + Rebuild, + Update + } + + public class IndexSettings { + public IndexingMode Mode { get; set; } + public int LastIndexedId { get; set; } + public int LastContentId { get; set; } + public DateTime LastIndexedUtc { get; set; } + + public static readonly string TagSettings = "Settings"; + public static readonly string TagMode = "Mode"; + public static readonly string TagLastIndexedId = "LastIndexedId"; + public static readonly string TagLastContentId = "LastContentId"; + public static readonly string TagLastIndexedUtc = "LastIndexedUtc"; + + public IndexSettings() { + Mode = IndexingMode.Rebuild; + LastIndexedId = 0; + LastContentId = 0; + LastIndexedUtc = DateTime.MinValue; + } + + public static IndexSettings Parse(string content) { + var doc = XDocument.Parse(content); + + try { + return new IndexSettings { + Mode = (IndexingMode) Enum.Parse(typeof (IndexingMode), doc.Descendants(TagMode).First().Value), + LastIndexedId = Int32.Parse(doc.Descendants(TagLastIndexedId).First().Value), + LastContentId = Int32.Parse(doc.Descendants(TagLastContentId).First().Value), + LastIndexedUtc = DateTime.Parse(doc.Descendants(TagLastIndexedUtc).First().Value) + }; + } + catch { + return new IndexSettings(); + } + } + + public override string ToString() { + return new XDocument( + new XElement(TagSettings, + new XElement(TagMode, Mode), + new XElement(TagLastIndexedId, LastIndexedId), + new XElement(TagLastContentId, LastContentId), + new XElement(TagLastIndexedUtc, LastIndexedUtc.ToString("s")) + )).ToString(); + } + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj b/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj index aff4ca59d..d33b84836 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj @@ -41,6 +41,8 @@ ..\..\..\..\lib\aspnetmvc\System.Web.Mvc.dll + + @@ -55,6 +57,10 @@ + + Code + + diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexService.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexService.cs index 5214bc6bb..c0fa721fb 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexService.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexService.cs @@ -5,13 +5,14 @@ namespace Orchard.Indexing.Services { public class IndexEntry { public string IndexName { get; set; } public int DocumentCount { get; set; } - public DateTime? LastUpdateUtc { get; set; } + public DateTime LastUpdateUtc { get; set; } public IEnumerable Fields { get; set; } + public IndexingStatus IndexingStatus { get; set; } } public interface IIndexingService : IDependency { - void RebuildIndex(); - void UpdateIndex(); - IndexEntry GetIndexEntry(); + void RebuildIndex(string indexName); + void UpdateIndex(string indexName); + IndexEntry GetIndexEntry(string indexName); } } \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexStatisticsProvider.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexStatisticsProvider.cs new file mode 100644 index 000000000..6b2dcb48c --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexStatisticsProvider.cs @@ -0,0 +1,13 @@ +using System; + +namespace Orchard.Indexing.Services { + public enum IndexingStatus { + Rebuilding, + Updating, + Idle + } + public interface IIndexStatisticsProvider : IDependency { + DateTime GetLastIndexedUtc(string indexName); + IndexingStatus GetIndexingStatus(string indexName); + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs index 90014a203..30705b750 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs @@ -1,59 +1,65 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using Orchard.Localization; -using Orchard.Localization.Services; using Orchard.UI.Notify; namespace Orchard.Indexing.Services { public class IndexingService : IIndexingService { - private const string SearchIndexName = "Search"; private readonly IIndexManager _indexManager; private readonly IEnumerable _indexNotifierHandlers; + private readonly IIndexStatisticsProvider _indexStatisticsProvider; - public IndexingService(IOrchardServices services, IIndexManager indexManager, IEnumerable indexNotifierHandlers, ICultureManager cultureManager) { + public IndexingService( + IOrchardServices services, + IIndexManager indexManager, + IEnumerable indexNotifierHandlers, + IIndexStatisticsProvider indexStatisticsProvider) { Services = services; _indexManager = indexManager; _indexNotifierHandlers = indexNotifierHandlers; + _indexStatisticsProvider = indexStatisticsProvider; T = NullLocalizer.Instance; } public IOrchardServices Services { get; set; } public Localizer T { get; set; } - void IIndexingService.RebuildIndex() { + void IIndexingService.RebuildIndex(string indexName) { if (!_indexManager.HasIndexProvider()) { Services.Notifier.Warning(T("There is no search index to rebuild.")); return; } var searchProvider = _indexManager.GetSearchIndexProvider(); - if (searchProvider.Exists(SearchIndexName)) - searchProvider.DeleteIndex(SearchIndexName); + if (searchProvider.Exists(indexName)) + searchProvider.DeleteIndex(indexName); - searchProvider.CreateIndex(SearchIndexName); // or just reset the updated date and let the background process recreate the index + searchProvider.CreateIndex(indexName); // or just reset the updated date and let the background process recreate the index - Services.Notifier.Information(T("The search index has been rebuilt.")); + Services.Notifier.Information(T("The index {0} has been rebuilt.", indexName)); } - void IIndexingService.UpdateIndex() { + void IIndexingService.UpdateIndex(string indexName) { foreach(var handler in _indexNotifierHandlers) { - handler.UpdateIndex(SearchIndexName); + handler.UpdateIndex(indexName); } Services.Notifier.Information(T("The search index has been updated.")); } - IndexEntry IIndexingService.GetIndexEntry() { + IndexEntry IIndexingService.GetIndexEntry(string indexName) { var provider = _indexManager.GetSearchIndexProvider(); if (provider == null) return null; return new IndexEntry { - IndexName = SearchIndexName, - DocumentCount = provider.NumDocs(SearchIndexName), - Fields = provider.GetFields(SearchIndexName), - LastUpdateUtc = provider.GetLastIndexUtc(SearchIndexName) + IndexName = indexName, + DocumentCount = provider.NumDocs(indexName), + Fields = provider.GetFields(indexName), + LastUpdateUtc = _indexStatisticsProvider.GetLastIndexedUtc(indexName), + IndexingStatus = _indexStatisticsProvider.GetIndexingStatus(indexName) }; } } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs index e9a969d17..974392f54 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs @@ -11,7 +11,6 @@ using Orchard.Indexing.Models; using Orchard.Indexing.Settings; using Orchard.Logging; using Orchard.Services; -using Orchard.Tasks.Indexing; namespace Orchard.Indexing.Services { /// @@ -22,35 +21,33 @@ namespace Orchard.Indexing.Services { /// and singleton locks would not be shared accross those two. /// [UsedImplicitly] - public class IndexingTaskExecutor : IIndexNotifierHandler { - private readonly IClock _clock; + public class IndexingTaskExecutor : IIndexNotifierHandler, IIndexStatisticsProvider { private readonly IRepository _repository; private IIndexProvider _indexProvider; private readonly IIndexManager _indexManager; - private readonly IIndexingTaskManager _indexingTaskManager; private readonly IContentManager _contentManager; private readonly IAppDataFolder _appDataFolder; private readonly ShellSettings _shellSettings; private readonly ILockFileManager _lockFileManager; + private readonly IClock _clock; + private const int ContentItemsPerLoop = 100; + private IndexingStatus _indexingStatus = IndexingStatus.Idle; public IndexingTaskExecutor( - IClock clock, IRepository repository, IIndexManager indexManager, - IIndexingTaskManager indexingTaskManager, IContentManager contentManager, IAppDataFolder appDataFolder, ShellSettings shellSettings, - ILockFileManager lockFileManager) { - _clock = clock; + ILockFileManager lockFileManager, + IClock clock) { _repository = repository; _indexManager = indexManager; - _indexingTaskManager = indexingTaskManager; _contentManager = contentManager; _appDataFolder = appDataFolder; _shellSettings = shellSettings; _lockFileManager = lockFileManager; - + _clock = clock; Logger = NullLogger.Instance; } @@ -61,120 +58,192 @@ namespace Orchard.Indexing.Services { var settingsFilename = GetSettingsFileName(indexName); var lockFilename = settingsFilename + ".lock"; + // acquire a lock file on the index if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile)) { - Logger.Information("Index was requested but was already running"); + Logger.Information("Index was requested but is already running"); return; } - using (lockFile) { + using (lockFile) + { if (!_indexManager.HasIndexProvider()) { return; } + // load index settings to know what is the current state of indexing + var indexSettings = LoadSettings(indexName); + _indexProvider = _indexManager.GetSearchIndexProvider(); - var updateIndexDocuments = new List(); - var addedContentItemIds = new List(); - DateTime? lastIndexUtc; - - // Do we need to rebuild the full index (first time module is used, or rebuild index requested) ? - if (_indexProvider.IsEmpty(indexName)) { - Logger.Information("Rebuild index started"); - - // mark current last task, as we should process older ones (in case of rebuild index only) - lastIndexUtc = _indexingTaskManager.GetLastTaskDateTime(); - - // get every existing content item to index it - foreach (var contentItem in _contentManager.Query(VersionOptions.Published).List()) { - try { - // skip items which are not indexed - var settings = GetTypeIndexingSettings(contentItem); - if (!settings.Included) - continue; - - var documentIndex = _indexProvider.New(contentItem.Id); - - _contentManager.Index(contentItem, documentIndex); - if (documentIndex.IsDirty) { - updateIndexDocuments.Add(documentIndex); - addedContentItemIds.Add(contentItem.Id.ToString()); - } - } - catch (Exception ex) { - Logger.Warning(ex, "Unable to index content item #{0} during rebuild", contentItem.Id); - } - } - - } - else { - // retrieve last processed index time - lastIndexUtc = _indexProvider.GetLastIndexUtc(indexName); - } - - _indexProvider.SetLastIndexUtc(indexName, _clock.UtcNow); - - // retrieve not yet processed tasks - var taskRecords = lastIndexUtc == null - ? _repository.Fetch(x => true).ToArray() - : _repository.Fetch(x => x.CreatedUtc >= lastIndexUtc).ToArray(); // CreatedUtc and lastIndexUtc might be equal if a content item is created in a background task - - // nothing to do ?))) - if (taskRecords.Length + updateIndexDocuments.Count == 0) { - Logger.Information("Index update requested, nothing to do"); - return; - } - - Logger.Information("Processing {0} indexing tasks", taskRecords.Length); + // should the index be rebuilt if (!_indexProvider.Exists(indexName)) { _indexProvider.CreateIndex(indexName); + indexSettings = new IndexSettings(); } - // process Delete tasks - try { - var deleteIds = taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete).Select(t => t.ContentItemRecord.Id).ToArray(); - if (deleteIds.Length > 0) { - _indexProvider.Delete(indexName, deleteIds); - Logger.Information("Deleted content items from index: {0}", String.Join(", ", deleteIds)); - } - } - catch (Exception ex) { - Logger.Warning(ex, "An error occured while removing a document from the index"); - } + // execute indexing commands by batch of [ContentItemsPerLoop] content items + for (; ; ){ + var addToIndex = new List(); + var deleteFromIndex = new List(); - // process Update tasks - foreach (var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update)) { - var task = new IndexingTask(_contentManager, taskRecord); + // Rebuilding the index ? + if (indexSettings.Mode == IndexingMode.Rebuild) { + Logger.Information("Rebuilding index"); + _indexingStatus = IndexingStatus.Rebuilding; - // skip items which are not indexed - var settings = GetTypeIndexingSettings(task.ContentItem); - if (!settings.Included) - continue; + // store the last inserted task + var lastIndexId = _repository + .Fetch(x => true) + .OrderByDescending(x => x.Id) + .Select(x => x.Id) + .FirstOrDefault(); - try { - var documentIndex = _indexProvider.New(task.ContentItem.Id); - _contentManager.Index(task.ContentItem, documentIndex); - if (!addedContentItemIds.Contains(task.ContentItem.Id.ToString()) && documentIndex.IsDirty) { - updateIndexDocuments.Add(documentIndex); + // load all content items + var contentItemIds = _contentManager + .Query(VersionOptions.Published) + .List() + .Where(x => x.Id > indexSettings.LastContentId) + .OrderBy(x => x.Id) + .Select(x => x.Id) + .Distinct() + .Take(ContentItemsPerLoop) + .ToArray(); + + indexSettings.LastIndexedId = lastIndexId; + + // if no more elements to index, switch to update mode + if (contentItemIds.Length == 0) { + indexSettings.Mode = IndexingMode.Update; } - } - catch (Exception ex) { - Logger.Warning(ex, "Unable to process indexing task #{0}", taskRecord.Id); - } - } + foreach (var id in contentItemIds) { + try { + IDocumentIndex documentIndex = ExtractDocumentIndex(id); - if (updateIndexDocuments.Count > 0) { + if (documentIndex != null && documentIndex.IsDirty) { + addToIndex.Add(documentIndex); + } + + // store the last processed element + indexSettings.LastContentId = contentItemIds.LastOrDefault(); + } + catch (Exception ex) { + Logger.Warning(ex, "Unable to index content item #{0} during rebuild", id); + } + } + } + + if (indexSettings.Mode == IndexingMode.Update) { + Logger.Information("Updating index"); + _indexingStatus = IndexingStatus.Updating; + + // load next content items to index, by filtering and ordering on the task id + var lastIndexId = _repository + .Fetch(x => x.Id > indexSettings.LastIndexedId) + .OrderByDescending(x => x.Id) + .Select(x => x.Id) + .FirstOrDefault(); + + var contentItemIds = _repository + .Fetch(x => x.Id > indexSettings.LastIndexedId) + .OrderBy(x => x.Id) + .Take(ContentItemsPerLoop) + .Select(x => x.ContentItemRecord.Id) + .Distinct() // don't process the same content item twice + .ToArray(); + + indexSettings.LastIndexedId = lastIndexId; + + foreach (var id in contentItemIds) { + try { + IDocumentIndex documentIndex = ExtractDocumentIndex(id); + + if (documentIndex == null) { + deleteFromIndex.Add(id); + } + else if (documentIndex.IsDirty) { + addToIndex.Add(documentIndex); + } + } + catch (Exception ex) { + Logger.Warning(ex, "Unable to index content item #{0} during rebuild", id); + } + } + } + + // save current state of the index + indexSettings.LastIndexedUtc = _clock.UtcNow; + _appDataFolder.CreateFile(settingsFilename, indexSettings.ToString()); + + if (deleteFromIndex.Count == 0 && addToIndex.Count == 0) { + // nothing more to do + _indexingStatus = IndexingStatus.Idle; + return; + } + + // save new and updated documents to the index try { - _indexProvider.Store(indexName, updateIndexDocuments); - Logger.Information("Added content items to index: {0}", String.Join(", ", addedContentItemIds)); + if (addToIndex.Count > 0) { + _indexProvider.Store(indexName, addToIndex); + Logger.Information("Added content items to index: {0}", addToIndex.Count); + } } catch (Exception ex) { Logger.Warning(ex, "An error occured while adding a document to the index"); } + + // removing documents from the index + try { + if (deleteFromIndex.Count > 0) { + _indexProvider.Delete(indexName, deleteFromIndex); + Logger.Information("Added content items to index: {0}", addToIndex.Count); + } + } + catch (Exception ex) { + Logger.Warning(ex, "An error occured while removing a document from the index"); + } } } } + /// + /// Loads the settings file or create a new default one if it doesn't exist + /// + public IndexSettings LoadSettings(string indexName) { + var indexSettings = new IndexSettings(); + var settingsFilename = GetSettingsFileName(indexName); + if (_appDataFolder.FileExists(settingsFilename)) { + var content = _appDataFolder.ReadFile(settingsFilename); + indexSettings = IndexSettings.Parse(content); + } + + return indexSettings; + } + + /// + /// Creates a IDocumentIndex instance for a specific content item id. If the content + /// item is no more published, it returns null. + /// + private IDocumentIndex ExtractDocumentIndex(int id) { + var contentItem = _contentManager.Get(id, VersionOptions.Published); + + // ignore deleted or unpublished items + if(contentItem == null || !contentItem.IsPublished()) { + return null; + } + + // skip items from types which are not indexed + var settings = GetTypeIndexingSettings(contentItem); + if (!settings.Included) + return null; + + var documentIndex = _indexProvider.New(contentItem.Id); + + // call all handlers to add content to index + _contentManager.Index(contentItem, documentIndex); + return documentIndex; + } + static TypeIndexing GetTypeIndexingSettings(ContentItem contentItem) { if (contentItem == null || contentItem.TypeDefinition == null || @@ -187,5 +256,14 @@ namespace Orchard.Indexing.Services { private string GetSettingsFileName(string indexName) { return _appDataFolder.Combine("Sites", _shellSettings.Name, indexName + ".settings.xml"); } + + public DateTime GetLastIndexedUtc(string indexName) { + var indexSettings = LoadSettings(indexName); + return indexSettings.LastIndexedUtc; + } + + public IndexingStatus GetIndexingStatus(string indexName) { + return _indexingStatus; + } } } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs index ccb4ff385..993f962e9 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs @@ -30,8 +30,6 @@ namespace Orchard.Indexing.Services { throw new ArgumentNullException("contentItem"); } - DeleteTasks(contentItem); - var taskRecord = new IndexingTaskRecord { CreatedUtc = _clock.UtcNow, ContentItemRecord = contentItem.Record, @@ -57,19 +55,5 @@ namespace Orchard.Indexing.Services { public DateTime GetLastTaskDateTime() { return _repository.Table.Max(t => t.CreatedUtc) ?? new DateTime(1980, 1, 1); } - - /// - /// Removes existing tasks for the specified content item - /// - public void DeleteTasks(ContentItem contentItem) { - var tasks = _repository - .Fetch(x => x.ContentItemRecord.Id == contentItem.Id) - .ToArray(); - foreach (var task in tasks) { - _repository.Delete(task); - } - - _repository.Flush(); - } } } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Views/Admin/Index.cshtml b/src/Orchard.Web/Modules/Orchard.Indexing/Views/Admin/Index.cshtml index c0b8ece0b..7f6b210de 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Views/Admin/Index.cshtml +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Views/Admin/Index.cshtml @@ -1,4 +1,5 @@ @model Orchard.Indexing.ViewModels.IndexViewModel +@using Orchard.Indexing.Services; @{ Layout.Title = T("Search Index Management").ToString(); } @@ -6,7 +7,7 @@
@if (Model.IndexEntry == null) {

@T("There is currently no search index")

- } else if (Model.IndexEntry.LastUpdateUtc == null) { + } else if (Model.IndexEntry.LastUpdateUtc == DateTime.MinValue) {

@T("The search index has not been built yet.")

} else { if (Model.IndexEntry.DocumentCount == 0) { @@ -21,8 +22,17 @@

@T("The search index contains the following fields: {0}.", string.Join(T(", ").Text, Model.IndexEntry.Fields))

} -

@T("The search index was last updated {0}.", Display.DateTimeRelative(dateTimeUtc: Model.IndexEntry.LastUpdateUtc.Value))

- } +

@T("The search index was last updated {0}.", Display.DateTimeRelative(dateTimeUtc: Model.IndexEntry.LastUpdateUtc))

+ + switch(Model.IndexEntry.IndexingStatus) { + case IndexingStatus.Rebuilding: + @T("The indexing process is currently being rebuilt."); + break; + case IndexingStatus.Updating: + @T("The indexing process is currently being updated."); + break; + } + }

@T("Update the search index now: ")

@Html.AntiForgeryTokenOrchard()
diff --git a/src/Orchard/Indexing/IIndexProvider.cs b/src/Orchard/Indexing/IIndexProvider.cs index 2acda505f..e5413faec 100644 --- a/src/Orchard/Indexing/IIndexProvider.cs +++ b/src/Orchard/Indexing/IIndexProvider.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; namespace Orchard.Indexing { public interface IIndexProvider : ISingletonDependency { @@ -60,16 +59,6 @@ namespace Orchard.Indexing { /// A search builder instance ISearchBuilder CreateSearchBuilder(string indexName); - /// - /// Returns the date and time when the index was last processed, or null if the index doesn't exist - /// - DateTime? GetLastIndexUtc(string indexName); - - /// - /// Sets the date and time when the index was last processed - /// - void SetLastIndexUtc(string indexName, DateTime lastIndexUtc); - /// /// Returns every field available in the specified index /// diff --git a/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs b/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs index 7f908ced1..85b6b02f2 100644 --- a/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs +++ b/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs @@ -17,11 +17,5 @@ namespace Orchard.Tasks.Indexing { /// Returns the Date Time of the last task created /// DateTime GetLastTaskDateTime(); - - /// - /// Deletes all indexing tasks assigned to a specific content item - /// - /// - void DeleteTasks(ContentItem contentItem); } } \ No newline at end of file From 599d892f4836a87ccb5fe69c84d0ed5630616de5 Mon Sep 17 00:00:00 2001 From: Sebastien Ros Date: Fri, 4 Mar 2011 16:21:19 -0800 Subject: [PATCH 5/7] Removing unused methods and refactoring the batch loop --HG-- branch : indexing --- .../Indexing/IndexingTaskExecutorTests.cs | 13 +- .../Orchard.Indexing/Models/IndexSettings.cs | 2 +- .../Services/IndexingTaskExecutor.cs | 256 +++++++++--------- .../Services/IndexingTaskManager.cs | 4 - .../Tasks/Indexing/IIndexingTaskManager.cs | 5 - 5 files changed, 145 insertions(+), 135 deletions(-) diff --git a/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs b/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs index 7e6549a4a..01b77337a 100644 --- a/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs +++ b/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using Autofac; using Lucene.Services; using Moq; @@ -176,6 +175,7 @@ namespace Orchard.Tests.Modules.Indexing { // there should be nothing done _indexNotifier.UpdateIndex(IndexName); + Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; _indexNotifier.UpdateIndex(IndexName); @@ -216,6 +216,17 @@ namespace Orchard.Tests.Modules.Indexing { Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); } + + [Test] + public void ShouldIndexAllContentOverTheLoopSize() { + for (int i = 0; i < 999; i++) { + var content = _contentManager.Create(ThingDriver.ContentTypeName); + content.Text = "Lorem ipsum " + i; + } + _indexNotifier.UpdateIndex(IndexName); + Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(999)); + } + #region Stubs public class ThingHandler : ContentHandler { public ThingHandler() { diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs index c0b3e0e67..c895bc474 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs @@ -50,7 +50,7 @@ namespace Orchard.Indexing.Models new XElement(TagMode, Mode), new XElement(TagLastIndexedId, LastIndexedId), new XElement(TagLastContentId, LastContentId), - new XElement(TagLastIndexedUtc, LastIndexedUtc.ToString("s")) + new XElement(TagLastIndexedUtc, LastIndexedUtc.ToString("u")) )).ToString(); } } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs index 974392f54..14ae41bc4 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using JetBrains.Annotations; using Orchard.ContentManagement; +using Orchard.ContentManagement.Records; using Orchard.Data; using Orchard.Environment.Configuration; using Orchard.FileSystems.AppData; @@ -22,7 +23,8 @@ namespace Orchard.Indexing.Services { /// [UsedImplicitly] public class IndexingTaskExecutor : IIndexNotifierHandler, IIndexStatisticsProvider { - private readonly IRepository _repository; + private readonly IRepository _taskRepository; + private readonly IRepository _contentRepository; private IIndexProvider _indexProvider; private readonly IIndexManager _indexManager; private readonly IContentManager _contentManager; @@ -34,14 +36,16 @@ namespace Orchard.Indexing.Services { private IndexingStatus _indexingStatus = IndexingStatus.Idle; public IndexingTaskExecutor( - IRepository repository, + IRepository taskRepository, + IRepository contentRepository, IIndexManager indexManager, IContentManager contentManager, IAppDataFolder appDataFolder, ShellSettings shellSettings, ILockFileManager lockFileManager, IClock clock) { - _repository = repository; + _taskRepository = taskRepository; + _contentRepository = contentRepository; _indexManager = indexManager; _contentManager = contentManager; _appDataFolder = appDataFolder; @@ -54,6 +58,11 @@ namespace Orchard.Indexing.Services { public ILogger Logger { get; set; } public void UpdateIndex(string indexName) { + // What to do here to run next batch in a separate transaction + while (UpdateIndexBatch(indexName)) {} + } + + public bool UpdateIndexBatch(string indexName) { ILockFile lockFile = null; var settingsFilename = GetSettingsFileName(indexName); var lockFilename = settingsFilename + ".lock"; @@ -61,13 +70,12 @@ namespace Orchard.Indexing.Services { // acquire a lock file on the index if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile)) { Logger.Information("Index was requested but is already running"); - return; + return false; } - using (lockFile) - { + using (lockFile) { if (!_indexManager.HasIndexProvider()) { - return; + return false; } // load index settings to know what is the current state of indexing @@ -79,131 +87,132 @@ namespace Orchard.Indexing.Services { if (!_indexProvider.Exists(indexName)) { _indexProvider.CreateIndex(indexName); indexSettings = new IndexSettings(); + + // mark the last available task at the moment the process is started. + // once the Rebuild is done, Update will start at this point of the table + indexSettings.LastIndexedId = _taskRepository + .Table + .OrderByDescending(x => x.Id) + .Select(x => x.Id) + .FirstOrDefault(); } // execute indexing commands by batch of [ContentItemsPerLoop] content items - for (; ; ){ - var addToIndex = new List(); - var deleteFromIndex = new List(); + return BatchIndex(indexName, settingsFilename, indexSettings); + } + } - // Rebuilding the index ? - if (indexSettings.Mode == IndexingMode.Rebuild) { - Logger.Information("Rebuilding index"); - _indexingStatus = IndexingStatus.Rebuilding; + /// + /// Indexes a batch of content items + /// + /// + /// true if there are more items to process; otherwise, false. + /// + private bool BatchIndex(string indexName, string settingsFilename, IndexSettings indexSettings) { + var addToIndex = new List(); + var deleteFromIndex = new List(); - // store the last inserted task - var lastIndexId = _repository - .Fetch(x => true) - .OrderByDescending(x => x.Id) - .Select(x => x.Id) - .FirstOrDefault(); + // Rebuilding the index ? + if (indexSettings.Mode == IndexingMode.Rebuild) { + Logger.Information("Rebuilding index"); + _indexingStatus = IndexingStatus.Rebuilding; - // load all content items - var contentItemIds = _contentManager - .Query(VersionOptions.Published) - .List() - .Where(x => x.Id > indexSettings.LastContentId) - .OrderBy(x => x.Id) - .Select(x => x.Id) - .Distinct() - .Take(ContentItemsPerLoop) - .ToArray(); + // load all content items + var contentItems = _contentRepository + .Fetch( + versionRecord => versionRecord.Published && versionRecord.ContentItemRecord.Id > indexSettings.LastContentId, + order => order.Asc(versionRecord => versionRecord.Id)) + .Take(ContentItemsPerLoop) + .Select(versionRecord => _contentManager.Get(versionRecord.ContentItemRecord.Id, VersionOptions.VersionRecord(versionRecord.Id))) + .Distinct() + .ToList(); - indexSettings.LastIndexedId = lastIndexId; + // if no more elements to index, switch to update mode + if (contentItems.Count == 0) { + indexSettings.Mode = IndexingMode.Update; + } - // if no more elements to index, switch to update mode - if (contentItemIds.Length == 0) { - indexSettings.Mode = IndexingMode.Update; - } - - foreach (var id in contentItemIds) { - try { - IDocumentIndex documentIndex = ExtractDocumentIndex(id); - - if (documentIndex != null && documentIndex.IsDirty) { - addToIndex.Add(documentIndex); - } - - // store the last processed element - indexSettings.LastContentId = contentItemIds.LastOrDefault(); - } - catch (Exception ex) { - Logger.Warning(ex, "Unable to index content item #{0} during rebuild", id); - } - } - } - - if (indexSettings.Mode == IndexingMode.Update) { - Logger.Information("Updating index"); - _indexingStatus = IndexingStatus.Updating; - - // load next content items to index, by filtering and ordering on the task id - var lastIndexId = _repository - .Fetch(x => x.Id > indexSettings.LastIndexedId) - .OrderByDescending(x => x.Id) - .Select(x => x.Id) - .FirstOrDefault(); - - var contentItemIds = _repository - .Fetch(x => x.Id > indexSettings.LastIndexedId) - .OrderBy(x => x.Id) - .Take(ContentItemsPerLoop) - .Select(x => x.ContentItemRecord.Id) - .Distinct() // don't process the same content item twice - .ToArray(); - - indexSettings.LastIndexedId = lastIndexId; - - foreach (var id in contentItemIds) { - try { - IDocumentIndex documentIndex = ExtractDocumentIndex(id); - - if (documentIndex == null) { - deleteFromIndex.Add(id); - } - else if (documentIndex.IsDirty) { - addToIndex.Add(documentIndex); - } - } - catch (Exception ex) { - Logger.Warning(ex, "Unable to index content item #{0} during rebuild", id); - } - } - } - - // save current state of the index - indexSettings.LastIndexedUtc = _clock.UtcNow; - _appDataFolder.CreateFile(settingsFilename, indexSettings.ToString()); - - if (deleteFromIndex.Count == 0 && addToIndex.Count == 0) { - // nothing more to do - _indexingStatus = IndexingStatus.Idle; - return; - } - - // save new and updated documents to the index + foreach (var item in contentItems) { try { - if (addToIndex.Count > 0) { - _indexProvider.Store(indexName, addToIndex); - Logger.Information("Added content items to index: {0}", addToIndex.Count); + IDocumentIndex documentIndex = ExtractDocumentIndex(item); + + if (documentIndex != null && documentIndex.IsDirty) { + addToIndex.Add(documentIndex); } + + indexSettings.LastContentId = item.VersionRecord.Id; } catch (Exception ex) { - Logger.Warning(ex, "An error occured while adding a document to the index"); - } - - // removing documents from the index - try { - if (deleteFromIndex.Count > 0) { - _indexProvider.Delete(indexName, deleteFromIndex); - Logger.Information("Added content items to index: {0}", addToIndex.Count); - } - } - catch (Exception ex) { - Logger.Warning(ex, "An error occured while removing a document from the index"); + Logger.Warning(ex, "Unable to index content item #{0} during rebuild", item.Id); } } } + + if (indexSettings.Mode == IndexingMode.Update) { + Logger.Information("Updating index"); + _indexingStatus = IndexingStatus.Updating; + + var contentItems = _taskRepository + .Fetch(x => x.Id > indexSettings.LastIndexedId) + .OrderBy(x => x.Id) + .Take(ContentItemsPerLoop) + .GroupBy(x => x.ContentItemRecord.Id) + .Select(group => new {TaskId = group.Max(task => task.Id), ContentItem = _contentManager.Get(group.Key, VersionOptions.Published)}) + .OrderBy(x => x.TaskId) + .ToArray(); + + foreach (var item in contentItems) { + try { + IDocumentIndex documentIndex = ExtractDocumentIndex(item.ContentItem); + + if (documentIndex == null) { + deleteFromIndex.Add(item.ContentItem.Id); + } + else if (documentIndex.IsDirty) { + addToIndex.Add(documentIndex); + } + + indexSettings.LastIndexedId = item.TaskId; + } + catch (Exception ex) { + Logger.Warning(ex, "Unable to index content item #{0} during update", item.ContentItem.Id); + } + } + } + + // save current state of the index + indexSettings.LastIndexedUtc = _clock.UtcNow; + _appDataFolder.CreateFile(settingsFilename, indexSettings.ToString()); + + if (deleteFromIndex.Count == 0 && addToIndex.Count == 0) { + // nothing more to do + _indexingStatus = IndexingStatus.Idle; + return false; + } + + // save new and updated documents to the index + try { + if (addToIndex.Count > 0) { + _indexProvider.Store(indexName, addToIndex); + Logger.Information("Added content items to index: {0}", addToIndex.Count); + } + } + catch (Exception ex) { + Logger.Warning(ex, "An error occured while adding a document to the index"); + } + + // removing documents from the index + try { + if (deleteFromIndex.Count > 0) { + _indexProvider.Delete(indexName, deleteFromIndex); + Logger.Information("Added content items to index: {0}", addToIndex.Count); + } + } + catch (Exception ex) { + Logger.Warning(ex, "An error occured while removing a document from the index"); + } + + return true; } /// @@ -224,18 +233,17 @@ namespace Orchard.Indexing.Services { /// Creates a IDocumentIndex instance for a specific content item id. If the content /// item is no more published, it returns null. /// - private IDocumentIndex ExtractDocumentIndex(int id) { - var contentItem = _contentManager.Get(id, VersionOptions.Published); - + private IDocumentIndex ExtractDocumentIndex(ContentItem contentItem) { // ignore deleted or unpublished items - if(contentItem == null || !contentItem.IsPublished()) { + if (contentItem == null || !contentItem.IsPublished()) { return null; } // skip items from types which are not indexed var settings = GetTypeIndexingSettings(contentItem); - if (!settings.Included) + if (!settings.Included) { return null; + } var documentIndex = _indexProvider.New(contentItem.Id); @@ -244,11 +252,11 @@ namespace Orchard.Indexing.Services { return documentIndex; } - static TypeIndexing GetTypeIndexingSettings(ContentItem contentItem) { + private static TypeIndexing GetTypeIndexingSettings(ContentItem contentItem) { if (contentItem == null || contentItem.TypeDefinition == null || contentItem.TypeDefinition.Settings == null) { - return new TypeIndexing { Included = false }; + return new TypeIndexing {Included = false}; } return contentItem.TypeDefinition.Settings.GetModel(); } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs index 993f962e9..2726b0715 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskManager.cs @@ -51,9 +51,5 @@ namespace Orchard.Indexing.Services { CreateTask(contentItem, IndexingTaskRecord.Delete); Logger.Information("Deleting index task created for [{0}:{1}]", contentItem.ContentType, contentItem.Id); } - - public DateTime GetLastTaskDateTime() { - return _repository.Table.Max(t => t.CreatedUtc) ?? new DateTime(1980, 1, 1); - } } } diff --git a/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs b/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs index 85b6b02f2..aab78c650 100644 --- a/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs +++ b/src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs @@ -12,10 +12,5 @@ namespace Orchard.Tasks.Indexing { /// Adds a new entry in the index task table in order to delete an existing index for the specified content item. /// void CreateDeleteIndexTask(ContentItem contentItem); - - /// - /// Returns the Date Time of the last task created - /// - DateTime GetLastTaskDateTime(); } } \ No newline at end of file From 7e16aebadae3740d07f83bba50ab2562406c04da Mon Sep 17 00:00:00 2001 From: Sebastien Ros Date: Sat, 5 Mar 2011 10:47:20 -0800 Subject: [PATCH 6/7] Using IProcessEngine to schedule batched indexing; Modifying IIndexingTaskExecutor to handle index deletiong in order to prevent concurrency issues. --HG-- branch : indexing --- .../Indexing/IndexingTaskExecutorTests.cs | 39 +++++++------ .../Commands/IndexingCommands.cs | 1 - .../Controllers/AdminController.cs | 1 - .../Orchard.Indexing/Models/IndexSettings.cs | 4 +- .../Orchard.Indexing/Orchard.Indexing.csproj | 3 + .../Services/IIndexingTaskExecutor.cs | 6 ++ .../Services/IUpdateIndexScheduler.cs | 5 ++ .../Orchard.Indexing/Services/IndexService.cs | 26 +++++---- .../Services/IndexingTaskExecutor.cs | 57 +++++++++++++++---- .../Services/UpdateIndexScheduler.cs | 42 ++++++++++++++ 10 files changed, 136 insertions(+), 48 deletions(-) create mode 100644 src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexingTaskExecutor.cs create mode 100644 src/Orchard.Web/Modules/Orchard.Indexing/Services/IUpdateIndexScheduler.cs create mode 100644 src/Orchard.Web/Modules/Orchard.Indexing/Services/UpdateIndexScheduler.cs diff --git a/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs b/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs index 01b77337a..68e0b46fa 100644 --- a/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs +++ b/src/Orchard.Tests.Modules/Indexing/IndexingTaskExecutorTests.cs @@ -34,7 +34,7 @@ namespace Orchard.Tests.Modules.Indexing { private IIndexProvider _provider; private IAppDataFolder _appDataFolder; private ShellSettings _shellSettings; - private IIndexNotifierHandler _indexNotifier; + private IIndexingTaskExecutor _indexTaskExecutor; private IContentManager _contentManager; private Mock _contentDefinitionManager; private StubLogger _logger; @@ -43,7 +43,6 @@ namespace Orchard.Tests.Modules.Indexing { private const string IndexName = "Search"; private readonly string _basePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); - [TestFixtureTearDown] public void Clean() { if (Directory.Exists(_basePath)) { @@ -61,7 +60,7 @@ namespace Orchard.Tests.Modules.Indexing { builder.RegisterType().As(); builder.RegisterInstance(_appDataFolder).As(); - builder.RegisterType().As(); + builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); @@ -105,9 +104,9 @@ namespace Orchard.Tests.Modules.Indexing { base.Init(); _lockFileManager = _container.Resolve(); _provider = _container.Resolve(); - _indexNotifier = _container.Resolve(); + _indexTaskExecutor = _container.Resolve(); _contentManager = _container.Resolve(); - ((IndexingTaskExecutor)_indexNotifier).Logger = _logger = new StubLogger(); + ((IndexingTaskExecutor)_indexTaskExecutor).Logger = _logger = new StubLogger(); var thingType = new ContentTypeDefinitionBuilder() .Named(ThingDriver.ContentTypeName) @@ -121,7 +120,7 @@ namespace Orchard.Tests.Modules.Indexing { [Test] public void IndexShouldBeEmptyWhenThereIsNoContent() { - _indexNotifier.UpdateIndex(IndexName); + while(_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0)); } @@ -137,7 +136,7 @@ namespace Orchard.Tests.Modules.Indexing { _contentManager.Create("alpha"); - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0)); } @@ -154,7 +153,7 @@ namespace Orchard.Tests.Modules.Indexing { _contentManager.Create("alpha"); - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(0)); } @@ -163,22 +162,22 @@ namespace Orchard.Tests.Modules.Indexing { var content = _contentManager.Create(ThingDriver.ContentTypeName); content.Text = "Lorem ipsum"; - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); } [Test] public void ShouldUpdateTheIndexWhenContentIsPublished() { _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); // there should be nothing done - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2)); } @@ -187,13 +186,13 @@ namespace Orchard.Tests.Modules.Indexing { ILockFile lockFile = null; _lockFileManager.TryAcquireLock("Sites/My Site/Search.settings.xml.lock", ref lockFile); using (lockFile) { - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_logger.LogEntries.Count, Is.EqualTo(1)); Assert.That(_logger.LogEntries, Has.Some.Matches(entry => entry.LogFormat == "Index was requested but is already running")); } _logger.LogEntries.Clear(); - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_logger.LogEntries, Has.None.Matches(entry => entry.LogFormat == "Index was requested but is already running")); } @@ -201,18 +200,18 @@ namespace Orchard.Tests.Modules.Indexing { public void ShouldUpdateTheIndexWhenContentIsUnPublished() { _contentManager.Create(ThingDriver.ContentTypeName).Text = "Lorem ipsum"; - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); var content = _contentManager.Create(ThingDriver.ContentTypeName); content.Text = "Lorem ipsum"; - - _indexNotifier.UpdateIndex(IndexName); + + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(2)); _contentManager.Unpublish(content.ContentItem); - - _indexNotifier.UpdateIndex(IndexName); + + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(1)); } @@ -223,7 +222,7 @@ namespace Orchard.Tests.Modules.Indexing { var content = _contentManager.Create(ThingDriver.ContentTypeName); content.Text = "Lorem ipsum " + i; } - _indexNotifier.UpdateIndex(IndexName); + while (_indexTaskExecutor.UpdateIndexBatch(IndexName)) {} Assert.That(_provider.NumDocs(IndexName), Is.EqualTo(999)); } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs index 18c95a234..f09dc8aa5 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Commands/IndexingCommands.cs @@ -43,7 +43,6 @@ namespace Orchard.Indexing.Commands { [CommandHelp("index rebuild \r\n\t" + "Rebuilds the search index")] public string Rebuild() { _indexingService.RebuildIndex(SearchIndexName); - _indexingService.UpdateIndex(SearchIndexName); return T("Index is now being rebuilt...").Text; } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs index 2a7bccc7c..3057b3a4b 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Controllers/AdminController.cs @@ -44,7 +44,6 @@ namespace Orchard.Indexing.Controllers { return new HttpUnauthorizedResult(); _indexingService.RebuildIndex(DefaultIndexName); - _indexingService.UpdateIndex(DefaultIndexName); return RedirectToAction("Index"); } diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs index c895bc474..93276adc9 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Models/IndexSettings.cs @@ -36,7 +36,7 @@ namespace Orchard.Indexing.Models Mode = (IndexingMode) Enum.Parse(typeof (IndexingMode), doc.Descendants(TagMode).First().Value), LastIndexedId = Int32.Parse(doc.Descendants(TagLastIndexedId).First().Value), LastContentId = Int32.Parse(doc.Descendants(TagLastContentId).First().Value), - LastIndexedUtc = DateTime.Parse(doc.Descendants(TagLastIndexedUtc).First().Value) + LastIndexedUtc = DateTime.Parse(doc.Descendants(TagLastIndexedUtc).First().Value).ToUniversalTime() }; } catch { @@ -44,7 +44,7 @@ namespace Orchard.Indexing.Models } } - public override string ToString() { + public string ToXml() { return new XDocument( new XElement(TagSettings, new XElement(TagMode, Mode), diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj b/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj index d33b84836..651d5d389 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Orchard.Indexing.csproj @@ -60,6 +60,9 @@ Code + + + diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexingTaskExecutor.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexingTaskExecutor.cs new file mode 100644 index 000000000..89e385e6f --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IIndexingTaskExecutor.cs @@ -0,0 +1,6 @@ +namespace Orchard.Indexing.Services { + public interface IIndexingTaskExecutor : IDependency { + bool DeleteIndex(string indexName); + bool UpdateIndexBatch(string indexName); + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IUpdateIndexScheduler.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IUpdateIndexScheduler.cs new file mode 100644 index 000000000..eae037600 --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IUpdateIndexScheduler.cs @@ -0,0 +1,5 @@ +namespace Orchard.Indexing.Services { + public interface IUpdateIndexScheduler : IDependency { + void Schedule(string indexName); + } +} \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs index 30705b750..aad07cd8d 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexService.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using Orchard.Localization; using Orchard.UI.Notify; @@ -9,38 +8,41 @@ namespace Orchard.Indexing.Services private readonly IIndexManager _indexManager; private readonly IEnumerable _indexNotifierHandlers; private readonly IIndexStatisticsProvider _indexStatisticsProvider; + private readonly IIndexingTaskExecutor _indexingTaskExecutor; public IndexingService( IOrchardServices services, IIndexManager indexManager, IEnumerable indexNotifierHandlers, - IIndexStatisticsProvider indexStatisticsProvider) { + IIndexStatisticsProvider indexStatisticsProvider, + IIndexingTaskExecutor indexingTaskExecutor) { Services = services; _indexManager = indexManager; _indexNotifierHandlers = indexNotifierHandlers; _indexStatisticsProvider = indexStatisticsProvider; + _indexingTaskExecutor = indexingTaskExecutor; T = NullLocalizer.Instance; } public IOrchardServices Services { get; set; } public Localizer T { get; set; } - void IIndexingService.RebuildIndex(string indexName) { + public void RebuildIndex(string indexName) { if (!_indexManager.HasIndexProvider()) { Services.Notifier.Warning(T("There is no search index to rebuild.")); return; } - var searchProvider = _indexManager.GetSearchIndexProvider(); - if (searchProvider.Exists(indexName)) - searchProvider.DeleteIndex(indexName); - - searchProvider.CreateIndex(indexName); // or just reset the updated date and let the background process recreate the index - - Services.Notifier.Information(T("The index {0} has been rebuilt.", indexName)); + if(_indexingTaskExecutor.DeleteIndex(indexName)) { + Services.Notifier.Information(T("The index {0} has been rebuilt.", indexName)); + UpdateIndex(indexName); + } + else { + Services.Notifier.Warning(T("The index {0} could no ben rebuilt. It might already be in use, please try again later.", indexName)); + } } - void IIndexingService.UpdateIndex(string indexName) { + public void UpdateIndex(string indexName) { foreach(var handler in _indexNotifierHandlers) { handler.UpdateIndex(indexName); diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs index 14ae41bc4..9526e1f69 100644 --- a/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/IndexingTaskExecutor.cs @@ -22,7 +22,8 @@ namespace Orchard.Indexing.Services { /// and singleton locks would not be shared accross those two. /// [UsedImplicitly] - public class IndexingTaskExecutor : IIndexNotifierHandler, IIndexStatisticsProvider { + public class IndexingTaskExecutor : IIndexingTaskExecutor, IIndexStatisticsProvider + { private readonly IRepository _taskRepository; private readonly IRepository _contentRepository; private IIndexProvider _indexProvider; @@ -32,7 +33,7 @@ namespace Orchard.Indexing.Services { private readonly ShellSettings _shellSettings; private readonly ILockFileManager _lockFileManager; private readonly IClock _clock; - private const int ContentItemsPerLoop = 100; + private const int ContentItemsPerLoop = 50; private IndexingStatus _indexingStatus = IndexingStatus.Idle; public IndexingTaskExecutor( @@ -57,9 +58,31 @@ namespace Orchard.Indexing.Services { public ILogger Logger { get; set; } - public void UpdateIndex(string indexName) { - // What to do here to run next batch in a separate transaction - while (UpdateIndexBatch(indexName)) {} + public bool DeleteIndex(string indexName) { + ILockFile lockFile = null; + var settingsFilename = GetSettingsFileName(indexName); + var lockFilename = settingsFilename + ".lock"; + + // acquire a lock file on the index + if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile)) { + Logger.Information("Could not delete the index. Already in use."); + return false; + } + + using (lockFile) { + if (!_indexManager.HasIndexProvider()) { + return false; + } + + var searchProvider = _indexManager.GetSearchIndexProvider(); + if (searchProvider.Exists(indexName)) { + searchProvider.DeleteIndex(indexName); + } + + DeleteSettings(indexName); + } + + return true; } public bool UpdateIndexBatch(string indexName) { @@ -83,10 +106,8 @@ namespace Orchard.Indexing.Services { _indexProvider = _indexManager.GetSearchIndexProvider(); - // should the index be rebuilt - if (!_indexProvider.Exists(indexName)) { + if (indexSettings.Mode == IndexingMode.Rebuild && indexSettings.LastContentId == 0) { _indexProvider.CreateIndex(indexName); - indexSettings = new IndexSettings(); // mark the last available task at the moment the process is started. // once the Rebuild is done, Update will start at this point of the table @@ -120,7 +141,7 @@ namespace Orchard.Indexing.Services { // load all content items var contentItems = _contentRepository .Fetch( - versionRecord => versionRecord.Published && versionRecord.ContentItemRecord.Id > indexSettings.LastContentId, + versionRecord => versionRecord.Published && versionRecord.Id > indexSettings.LastContentId, order => order.Asc(versionRecord => versionRecord.Id)) .Take(ContentItemsPerLoop) .Select(versionRecord => _contentManager.Get(versionRecord.ContentItemRecord.Id, VersionOptions.VersionRecord(versionRecord.Id))) @@ -182,7 +203,7 @@ namespace Orchard.Indexing.Services { // save current state of the index indexSettings.LastIndexedUtc = _clock.UtcNow; - _appDataFolder.CreateFile(settingsFilename, indexSettings.ToString()); + _appDataFolder.CreateFile(settingsFilename, indexSettings.ToXml()); if (deleteFromIndex.Count == 0 && addToIndex.Count == 0) { // nothing more to do @@ -218,10 +239,12 @@ namespace Orchard.Indexing.Services { /// /// Loads the settings file or create a new default one if it doesn't exist /// - public IndexSettings LoadSettings(string indexName) { + public IndexSettings LoadSettings(string indexName) + { var indexSettings = new IndexSettings(); var settingsFilename = GetSettingsFileName(indexName); - if (_appDataFolder.FileExists(settingsFilename)) { + if (_appDataFolder.FileExists(settingsFilename)) + { var content = _appDataFolder.ReadFile(settingsFilename); indexSettings = IndexSettings.Parse(content); } @@ -229,6 +252,16 @@ namespace Orchard.Indexing.Services { return indexSettings; } + /// + /// Deletes the settings file + /// + public void DeleteSettings(string indexName) { + var settingsFilename = GetSettingsFileName(indexName); + if (_appDataFolder.FileExists(settingsFilename)) { + _appDataFolder.DeleteFile(settingsFilename); + } + } + /// /// Creates a IDocumentIndex instance for a specific content item id. If the content /// item is no more published, it returns null. diff --git a/src/Orchard.Web/Modules/Orchard.Indexing/Services/UpdateIndexScheduler.cs b/src/Orchard.Web/Modules/Orchard.Indexing/Services/UpdateIndexScheduler.cs new file mode 100644 index 000000000..878f5a65d --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.Indexing/Services/UpdateIndexScheduler.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using Orchard.Environment.Configuration; +using Orchard.Environment.Descriptor; +using Orchard.Environment.State; + +namespace Orchard.Indexing.Services { + public class UpdateIndexScheduler : IUpdateIndexScheduler, IIndexNotifierHandler { + private readonly IProcessingEngine _processingEngine; + private readonly ShellSettings _shellSettings; + private readonly IShellDescriptorManager _shellDescriptorManager; + private readonly Lazy _indexingTaskExecutor; + + public UpdateIndexScheduler( + IProcessingEngine processingEngine, + ShellSettings shellSettings, + IShellDescriptorManager shellDescriptorManager, + Lazy indexingTaskExecutor + ) { + _processingEngine = processingEngine; + _shellSettings = shellSettings; + _shellDescriptorManager = shellDescriptorManager; + _indexingTaskExecutor = indexingTaskExecutor; + } + + public void Schedule(string indexName) { + var shellDescriptor = _shellDescriptorManager.GetShellDescriptor(); + _processingEngine.AddTask( + _shellSettings, + shellDescriptor, + "IIndexNotifierHandler.UpdateIndex", + new Dictionary { { "indexName", indexName } } + ); + } + + public void UpdateIndex(string indexName) { + if(_indexingTaskExecutor.Value.UpdateIndexBatch(indexName)) { + Schedule(indexName); + } + } + } +} \ No newline at end of file From b666892cbdbcde00415cda88deef8524ec33078a Mon Sep 17 00:00:00 2001 From: Sebastien Ros Date: Mon, 14 Mar 2011 17:08:19 -0700 Subject: [PATCH 7/7] Fixing missing styles definitions --HG-- branch : dev --- src/Orchard.Web/Modules/Orchard.jQuery/ResourceManifest.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Orchard.Web/Modules/Orchard.jQuery/ResourceManifest.cs b/src/Orchard.Web/Modules/Orchard.jQuery/ResourceManifest.cs index 30aae3a81..5af198ffc 100644 --- a/src/Orchard.Web/Modules/Orchard.jQuery/ResourceManifest.cs +++ b/src/Orchard.Web/Modules/Orchard.jQuery/ResourceManifest.cs @@ -49,6 +49,9 @@ namespace Orchard.UI.Resources { manifest.DefineScript("jQueryUtils_TimePicker").SetUrl("ui.timepickr.js").SetVersion("0.7.0a").SetDependencies("jQueryUtils", "jQueryUI_Core", "jQueryUI_Widget"); manifest.DefineStyle("jQueryUI_Orchard").SetUrl("jquery-ui-1.7.2.custom.css").SetVersion("1.7.2"); + manifest.DefineStyle("jQueryUI_DatePicker").SetUrl("ui.datepicker.css").SetDependencies("jQueryUI_Orchard").SetVersion("1.7.2"); + manifest.DefineStyle("jQueryUtils_TimePicker").SetUrl("ui.timepickr.css"); + } } }