lsm/Orchard
1
0
Fork 0
mirror of https://github.com/OrchardCMS/Orchard.git synced 2025-10-15 03:25:23 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Store index settings on local file system instead of db, for farm scenarios

Browse Source 
--HG--
branch : dev
This commit is contained in:
Sebastien Ros
2010-06-04 16:29:50 -07:00
parent 91a63dddb8
commit d93c9274c8
8 changed files with 95 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs
View File

@@ -181,5 +181,17 @@ namespace Orchard.Tests.Indexing {
Assert.That(searchBuilder.Get(2).Id, Is.EqualTo(2)); Assert.That(searchBuilder.Get(2).Id, Is.EqualTo(2));
Assert.That(searchBuilder.Get(3).Id, Is.EqualTo(3)); Assert.That(searchBuilder.Get(3).Id, Is.EqualTo(3));
} }
[Test]
public void ProviderShouldStoreSettings() {
_provider.CreateIndex("default");
Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(DefaultIndexProvider.DefaultMinDateTime));
_provider.SetLastIndexUtc("default", new DateTime(2010, 1, 1, 1, 1, 1, 1));
Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(new DateTime(2010, 1, 1, 1, 1, 1, 0)));
_provider.SetLastIndexUtc("default", new DateTime(1901, 1, 1, 1, 1, 1, 1));
Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(DefaultIndexProvider.DefaultMinDateTime));
}
} }
} }

46
src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs
View File

@@ -13,6 +13,7 @@ using Orchard.Indexing;
using Directory = Lucene.Net.Store.Directory; using Directory = Lucene.Net.Store.Directory;
using Version = Lucene.Net.Util.Version; using Version = Lucene.Net.Util.Version;
using Orchard.Logging; using Orchard.Logging;
using System.Xml.Linq;
namespace Orchard.Core.Indexing.Lucene { namespace Orchard.Core.Indexing.Lucene {
/// <summary> /// <summary>
@@ -24,6 +25,9 @@ namespace Orchard.Core.Indexing.Lucene {
public static readonly Version LuceneVersion = Version.LUCENE_29; public static readonly Version LuceneVersion = Version.LUCENE_29;
private readonly Analyzer _analyzer = new StandardAnalyzer(LuceneVersion); private readonly Analyzer _analyzer = new StandardAnalyzer(LuceneVersion);
private readonly string _basePath; private readonly string _basePath;
public static readonly DateTime DefaultMinDateTime = new DateTime(1980, 1, 1);
public static readonly string Settings = "Settings";
public static readonly string LastIndexUtc = "LastIndexedUtc";
public ILogger Logger { get; set; } public ILogger Logger { get; set; }
@@ -37,6 +41,10 @@ namespace Orchard.Core.Indexing.Lucene {
Logger = NullLogger.Instance; Logger = NullLogger.Instance;
// Ensures the directory exists // Ensures the directory exists
EnsureDirectoryExists();
}
private void EnsureDirectoryExists() {
var directory = new DirectoryInfo(_appDataFolder.MapPath(_basePath)); var directory = new DirectoryInfo(_appDataFolder.MapPath(_basePath));
if(!directory.Exists) { if(!directory.Exists) {
directory.Create(); directory.Create();
@@ -72,6 +80,11 @@ namespace Orchard.Core.Indexing.Lucene {
public void DeleteIndex(string indexName) { public void DeleteIndex(string indexName) {
new DirectoryInfo(Path.Combine(_appDataFolder.MapPath(Path.Combine(_basePath, indexName)))) new DirectoryInfo(Path.Combine(_appDataFolder.MapPath(Path.Combine(_basePath, indexName))))
.Delete(true); .Delete(true);
var settingsFileName = GetSettingsFileName(indexName);
if(File.Exists(settingsFileName)) {
File.Delete(settingsFileName);
}
} }
public void Store(string indexName, IIndexDocument indexDocument) { public void Store(string indexName, IIndexDocument indexDocument) {
@@ -148,5 +161,38 @@ namespace Orchard.Core.Indexing.Lucene {
return new DefaultSearchBuilder(GetDirectory(indexName)); return new DefaultSearchBuilder(GetDirectory(indexName));
} }
private string GetSettingsFileName(string indexName) {
return Path.Combine(_appDataFolder.MapPath(_basePath), indexName + ".settings.xml");
}
public DateTime GetLastIndexUtc(string indexName) {
var settingsFileName = GetSettingsFileName(indexName);
return File.Exists(settingsFileName)
? DateTime.Parse(XDocument.Load(settingsFileName).Descendants(LastIndexUtc).First().Value)
: DefaultMinDateTime;
}
public void SetLastIndexUtc(string indexName, DateTime lastIndexUtc) {
if ( lastIndexUtc < DefaultMinDateTime ) {
lastIndexUtc = DefaultMinDateTime;
}
XDocument doc;
var settingsFileName = GetSettingsFileName(indexName);
if ( !File.Exists(settingsFileName) ) {
EnsureDirectoryExists();
doc = new XDocument(
new XElement(Settings,
new XElement(LastIndexUtc, lastIndexUtc.ToString("s"))));
}
else {
doc = XDocument.Load(settingsFileName);
doc.Element(Settings).Element(LastIndexUtc).Value = lastIndexUtc.ToString("s");
}
doc.Save(settingsFileName);
}
} }
} }

8
src/Orchard.Web/Core/Indexing/Models/IndexingSettingsRecord.cs
View File

@@ -1,8 +0,0 @@
using System;
namespace Orchard.Core.Indexing.Models {
public class IndexingSettingsRecord {
public virtual int Id { get; set; }
public virtual DateTime? LatestIndexingUtc { get; set; }
}
}

60
src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs
View File

@@ -19,7 +19,6 @@ namespace Orchard.Core.Indexing.Services {
public class IndexingTaskExecutor : IBackgroundTask { public class IndexingTaskExecutor : IBackgroundTask {
private readonly IClock _clock; private readonly IClock _clock;
private readonly IRepository<IndexingTaskRecord> _repository; private readonly IRepository<IndexingTaskRecord> _repository;
private readonly IRepository<IndexingSettingsRecord> _settings;
private readonly IEnumerable<IContentHandler> _handlers; private readonly IEnumerable<IContentHandler> _handlers;
private IIndexProvider _indexProvider; private IIndexProvider _indexProvider;
private readonly IIndexManager _indexManager; private readonly IIndexManager _indexManager;
@@ -29,13 +28,11 @@ namespace Orchard.Core.Indexing.Services {
public IndexingTaskExecutor( public IndexingTaskExecutor(
IClock clock, IClock clock,
IRepository<IndexingTaskRecord> repository, IRepository<IndexingTaskRecord> repository,
IRepository<IndexingSettingsRecord> settings,
IEnumerable<IContentHandler> handlers, IEnumerable<IContentHandler> handlers,
IIndexManager indexManager, IIndexManager indexManager,
IContentManager contentManager) { IContentManager contentManager) {
_clock = clock; _clock = clock;
_repository = repository; _repository = repository;
_settings = settings;
_indexManager = indexManager; _indexManager = indexManager;
_handlers = handlers; _handlers = handlers;
_contentManager = contentManager; _contentManager = contentManager;
@@ -46,64 +43,41 @@ namespace Orchard.Core.Indexing.Services {
public void Sweep() { public void Sweep() {
if ( !_indexManager.HasIndexProvider() ) { if (!_indexManager.HasIndexProvider()) {
return; return;
} }
_indexProvider = _indexManager.GetSearchIndexProvider(); _indexProvider = _indexManager.GetSearchIndexProvider();
// retrieve last processed index time // retrieve last processed index time
var settingsRecord = _settings.Table.FirstOrDefault(); var lastIndexing = _indexProvider.GetLastIndexUtc(SearchIndexName);
_indexProvider.SetLastIndexUtc(SearchIndexName, _clock.UtcNow);
if ( settingsRecord == null ) { // retrieve not yet processed tasks
_settings.Create(settingsRecord = new IndexingSettingsRecord { LatestIndexingUtc = new DateTime(1980, 1, 1) });
}
var lastIndexing = settingsRecord.LatestIndexingUtc;
settingsRecord.LatestIndexingUtc = _clock.UtcNow;
// retrieved not yet processed tasks
var taskRecords = _repository.Fetch(x => x.CreatedUtc >= lastIndexing) var taskRecords = _repository.Fetch(x => x.CreatedUtc >= lastIndexing)
.ToArray(); .ToArray();
if ( taskRecords.Length == 0 ) if (taskRecords.Length == 0)
return; return;
Logger.Information("Processing {0} indexing tasks", taskRecords.Length); Logger.Information("Processing {0} indexing tasks", taskRecords.Length);
if (!_indexProvider.Exists(SearchIndexName)) {
if ( !_indexProvider.Exists(SearchIndexName) ) {
_indexProvider.CreateIndex(SearchIndexName); _indexProvider.CreateIndex(SearchIndexName);
} }
var updateIndexDocuments = new List<IIndexDocument>(); var updateIndexDocuments = new List<IIndexDocument>();
var deleteIndexDocuments = new List<int>();
// process Delete tasks // process Delete tasks
foreach ( var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete) ) {
var task = new IndexingTask(_contentManager, taskRecord);
deleteIndexDocuments.Add(taskRecord.ContentItemRecord.Id);
try { try {
_repository.Delete(taskRecord); _indexProvider.Delete(SearchIndexName, taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete).Select(t => t.Id));
} }
catch ( Exception ex ) { catch (Exception ex) {
Logger.Error(ex, "Could not delete task #{0}", taskRecord.Id); Logger.Warning(ex, "An error occured while removing a document from the index");
}
}
try {
if ( deleteIndexDocuments.Count > 0 ) {
_indexProvider.Delete(SearchIndexName, deleteIndexDocuments);
}
}
catch ( Exception ex ) {
Logger.Warning(ex, "An error occured while remove a document from the index");
} }
// process Update tasks // process Update tasks
foreach ( var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update) ) { foreach (var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update)) {
var task = new IndexingTask(_contentManager, taskRecord); var task = new IndexingTask(_contentManager, taskRecord);
try { try {
@@ -113,31 +87,29 @@ namespace Orchard.Core.Indexing.Services {
}; };
// dispatch to handlers to retrieve index information // dispatch to handlers to retrieve index information
foreach ( var handler in _handlers ) { foreach (var handler in _handlers) {
handler.Indexing(context); handler.Indexing(context);
} }
updateIndexDocuments.Add(context.IndexDocument); updateIndexDocuments.Add(context.IndexDocument);
foreach ( var handler in _handlers ) { foreach (var handler in _handlers) {
handler.Indexed(context); handler.Indexed(context);
} }
} }
catch ( Exception ex ) { catch (Exception ex) {
Logger.Warning(ex, "Unable to process indexing task #{0}", taskRecord.Id); Logger.Warning(ex, "Unable to process indexing task #{0}", taskRecord.Id);
} }
} }
if (updateIndexDocuments.Count > 0) {
try { try {
if ( updateIndexDocuments.Count > 0 ) {
_indexProvider.Store(SearchIndexName, updateIndexDocuments); _indexProvider.Store(SearchIndexName, updateIndexDocuments);
} }
} catch (Exception ex) {
catch ( Exception ex ) {
Logger.Warning(ex, "An error occured while adding a document to the index"); Logger.Warning(ex, "An error occured while adding a document to the index");
} }
}
_settings.Update(settingsRecord);
} }
} }
} }

43
src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs
View File

@@ -16,18 +16,15 @@ namespace Orchard.Core.Indexing.Services {
public class IndexingTaskManager : IIndexingTaskManager { public class IndexingTaskManager : IIndexingTaskManager {
private readonly IContentManager _contentManager; private readonly IContentManager _contentManager;
private readonly IRepository<IndexingTaskRecord> _repository; private readonly IRepository<IndexingTaskRecord> _repository;
private readonly IRepository<IndexingSettingsRecord> _settings;
private readonly IClock _clock; private readonly IClock _clock;
public IndexingTaskManager( public IndexingTaskManager(
IContentManager contentManager, IContentManager contentManager,
IRepository<IndexingTaskRecord> repository, IRepository<IndexingTaskRecord> repository,
IRepository<IndexingSettingsRecord> settings,
IClock clock) { IClock clock) {
_clock = clock; _clock = clock;
_repository = repository; _repository = repository;
_contentManager = contentManager; _contentManager = contentManager;
_settings = settings;
Logger = NullLogger.Instance; Logger = NullLogger.Instance;
} }
@@ -38,14 +35,7 @@ namespace Orchard.Core.Indexing.Services {
throw new ArgumentNullException("contentItem"); throw new ArgumentNullException("contentItem");
} }
// remove previous tasks for the same content item DeleteTasks(contentItem);
var tasks = _repository
.Fetch(x => x.ContentItemRecord.Id == contentItem.Id)
.ToArray();
foreach ( var task in tasks ) {
_repository.Delete(task);
}
var taskRecord = new IndexingTaskRecord { var taskRecord = new IndexingTaskRecord {
CreatedUtc = _clock.UtcNow, CreatedUtc = _clock.UtcNow,
@@ -77,37 +67,16 @@ namespace Orchard.Core.Indexing.Services {
.ToReadOnlyCollection(); .ToReadOnlyCollection();
} }
public void DeleteTasks(DateTime? createdBefore) { /// <summary>
Logger.Debug("Deleting Indexing tasks created before {0}", createdBefore); /// Removes existing tasks for the specified content item
/// </summary>
var tasks = _repository
.Fetch(x => x.CreatedUtc <= createdBefore);
foreach (var task in tasks) {
_repository.Delete(task);
}
}
public void DeleteTasks(ContentItem contentItem) { public void DeleteTasks(ContentItem contentItem) {
Logger.Debug("Deleting Indexing tasks for ContentItem [{0}:{1}]", contentItem.ContentType, contentItem.Id);
var tasks = _repository var tasks = _repository
.Fetch(x => x.Id == contentItem.Id); .Fetch(x => x.ContentItemRecord.Id == contentItem.Id)
.ToArray();
foreach (var task in tasks) { foreach (var task in tasks) {
_repository.Delete(task); _repository.Delete(task);
} }
} }
public void RebuildIndex() {
var settingsRecord = _settings.Table.FirstOrDefault();
if (settingsRecord == null) {
_settings.Create(settingsRecord = new IndexingSettingsRecord() );
}
settingsRecord.LatestIndexingUtc = new DateTime(1980, 1, 1);
_settings.Update(settingsRecord);
}
} }
} }

1
src/Orchard.Web/Core/Orchard.Core.csproj
View File

@@ -113,7 +113,6 @@
<Compile Include="Indexing\Lucene\DefaultIndexProvider.cs" /> <Compile Include="Indexing\Lucene\DefaultIndexProvider.cs" />
<Compile Include="Indexing\Lucene\DefaultSearchBuilder.cs" /> <Compile Include="Indexing\Lucene\DefaultSearchBuilder.cs" />
<Compile Include="Indexing\Lucene\DefaultSearchHit.cs" /> <Compile Include="Indexing\Lucene\DefaultSearchHit.cs" />
<Compile Include="Indexing\Models\IndexingSettingsRecord.cs" />
<Compile Include="Indexing\Models\IndexingTask.cs" /> <Compile Include="Indexing\Models\IndexingTask.cs" />
<Compile Include="Indexing\Models\IndexingTaskRecord.cs" /> <Compile Include="Indexing\Models\IndexingTaskRecord.cs" />
<Compile Include="Indexing\Services\CreateIndexingTaskHandler.cs" /> <Compile Include="Indexing\Services\CreateIndexingTaskHandler.cs" />

14
src/Orchard/Indexing/IIndexProvider.cs
View File

@@ -1,4 +1,5 @@
using System.Collections.Generic; using System;
using System.Collections.Generic;
namespace Orchard.Indexing { namespace Orchard.Indexing {
public interface IIndexProvider : IDependency { public interface IIndexProvider : IDependency {
@@ -48,5 +49,16 @@ namespace Orchard.Indexing {
/// </summary> /// </summary>
/// <returns>A search builder instance</returns> /// <returns>A search builder instance</returns>
ISearchBuilder CreateSearchBuilder(string indexName); ISearchBuilder CreateSearchBuilder(string indexName);
/// <summary>
/// Returns the date and time when the index was last processed
/// </summary>
DateTime GetLastIndexUtc(string indexName);
/// <summary>
/// Sets the date and time when the index was last processed
/// </summary>
void SetLastIndexUtc(string indexName, DateTime lastIndexUtc);
} }
} }

5
src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs
View File

@@ -19,11 +19,6 @@ namespace Orchard.Tasks.Indexing {
/// </summary> /// </summary>
IEnumerable<IIndexingTask> GetTasks(DateTime? createdAfter); IEnumerable<IIndexingTask> GetTasks(DateTime? createdAfter);
/// <summary>
/// Deletes all indexing tasks previous to a specific date and time
/// </summary>
void DeleteTasks(DateTime? createdBefore);
/// <summary> /// <summary>
/// Deletes all indexing tasks assigned to a specific content item /// Deletes all indexing tasks assigned to a specific content item
/// </summary> /// </summary>