lsm/Orchard
1
0
Fork 0
mirror of https://github.com/OrchardCMS/Orchard.git synced 2025-10-14 10:54:50 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Store index settings on local file system instead of db, for farm scenarios

Browse Source 
--HG--
branch : dev
This commit is contained in:
Sebastien Ros
2010-06-04 16:29:50 -07:00
parent 91a63dddb8
commit d93c9274c8
8 changed files with 95 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs
View File

@@ -181,5 +181,17 @@ namespace Orchard.Tests.Indexing {
Assert.That(searchBuilder.Get(2).Id, Is.EqualTo(2));
Assert.That(searchBuilder.Get(3).Id, Is.EqualTo(3));
}
[Test]
public void ProviderShouldStoreSettings() {
_provider.CreateIndex("default");
Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(DefaultIndexProvider.DefaultMinDateTime));
_provider.SetLastIndexUtc("default", new DateTime(2010, 1, 1, 1, 1, 1, 1));
Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(new DateTime(2010, 1, 1, 1, 1, 1, 0)));
_provider.SetLastIndexUtc("default", new DateTime(1901, 1, 1, 1, 1, 1, 1));
Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(DefaultIndexProvider.DefaultMinDateTime));
}
}
}

46
src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs
View File

@@ -13,6 +13,7 @@ using Orchard.Indexing;
using Directory = Lucene.Net.Store.Directory;
using Version = Lucene.Net.Util.Version;
using Orchard.Logging;
using System.Xml.Linq;
namespace Orchard.Core.Indexing.Lucene {
/// <summary>
@@ -24,6 +25,9 @@ namespace Orchard.Core.Indexing.Lucene {
public static readonly Version LuceneVersion = Version.LUCENE_29;
private readonly Analyzer _analyzer = new StandardAnalyzer(LuceneVersion);
private readonly string _basePath;
public static readonly DateTime DefaultMinDateTime = new DateTime(1980, 1, 1);
public static readonly string Settings = "Settings";
public static readonly string LastIndexUtc = "LastIndexedUtc";
public ILogger Logger { get; set; }
@@ -37,6 +41,10 @@ namespace Orchard.Core.Indexing.Lucene {
Logger = NullLogger.Instance;
// Ensures the directory exists
EnsureDirectoryExists();
}
private void EnsureDirectoryExists() {
var directory = new DirectoryInfo(_appDataFolder.MapPath(_basePath));
if(!directory.Exists) {
directory.Create();
@@ -72,6 +80,11 @@ namespace Orchard.Core.Indexing.Lucene {
public void DeleteIndex(string indexName) {
new DirectoryInfo(Path.Combine(_appDataFolder.MapPath(Path.Combine(_basePath, indexName))))
.Delete(true);
var settingsFileName = GetSettingsFileName(indexName);
if(File.Exists(settingsFileName)) {
File.Delete(settingsFileName);
}
}
public void Store(string indexName, IIndexDocument indexDocument) {
@@ -148,5 +161,38 @@ namespace Orchard.Core.Indexing.Lucene {
return new DefaultSearchBuilder(GetDirectory(indexName));
}
private string GetSettingsFileName(string indexName) {
return Path.Combine(_appDataFolder.MapPath(_basePath), indexName + ".settings.xml");
}
public DateTime GetLastIndexUtc(string indexName) {
var settingsFileName = GetSettingsFileName(indexName);
return File.Exists(settingsFileName)
? DateTime.Parse(XDocument.Load(settingsFileName).Descendants(LastIndexUtc).First().Value)
: DefaultMinDateTime;
}
public void SetLastIndexUtc(string indexName, DateTime lastIndexUtc) {
if ( lastIndexUtc < DefaultMinDateTime ) {
lastIndexUtc = DefaultMinDateTime;
}
XDocument doc;
var settingsFileName = GetSettingsFileName(indexName);
if ( !File.Exists(settingsFileName) ) {
EnsureDirectoryExists();
doc = new XDocument(
new XElement(Settings,
new XElement(LastIndexUtc, lastIndexUtc.ToString("s"))));
}
else {
doc = XDocument.Load(settingsFileName);
doc.Element(Settings).Element(LastIndexUtc).Value = lastIndexUtc.ToString("s");
}
doc.Save(settingsFileName);
}
}
}

8
src/Orchard.Web/Core/Indexing/Models/IndexingSettingsRecord.cs
View File

@@ -1,8 +0,0 @@
using System;
namespace Orchard.Core.Indexing.Models {
public class IndexingSettingsRecord {
public virtual int Id { get; set; }
public virtual DateTime? LatestIndexingUtc { get; set; }
}
}

64
src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs
View File

@@ -19,7 +19,6 @@ namespace Orchard.Core.Indexing.Services {
public class IndexingTaskExecutor : IBackgroundTask {
private readonly IClock _clock;
private readonly IRepository<IndexingTaskRecord> _repository;
private readonly IRepository<IndexingSettingsRecord> _settings;
private readonly IEnumerable<IContentHandler> _handlers;
private IIndexProvider _indexProvider;
private readonly IIndexManager _indexManager;
@@ -29,13 +28,11 @@ namespace Orchard.Core.Indexing.Services {
public IndexingTaskExecutor(
IClock clock,
IRepository<IndexingTaskRecord> repository,
IRepository<IndexingSettingsRecord> settings,
IEnumerable<IContentHandler> handlers,
IIndexManager indexManager,
IContentManager contentManager) {
_clock = clock;
_repository = repository;
_settings = settings;
_indexManager = indexManager;
_handlers = handlers;
_contentManager = contentManager;
@@ -46,64 +43,41 @@ namespace Orchard.Core.Indexing.Services {
public void Sweep() {
if ( !_indexManager.HasIndexProvider() ) {
if (!_indexManager.HasIndexProvider()) {
return;
}
_indexProvider = _indexManager.GetSearchIndexProvider();
// retrieve last processed index time
var settingsRecord = _settings.Table.FirstOrDefault();
var lastIndexing = _indexProvider.GetLastIndexUtc(SearchIndexName);
_indexProvider.SetLastIndexUtc(SearchIndexName, _clock.UtcNow);
if ( settingsRecord == null ) {
_settings.Create(settingsRecord = new IndexingSettingsRecord { LatestIndexingUtc = new DateTime(1980, 1, 1) });
}
var lastIndexing = settingsRecord.LatestIndexingUtc;
settingsRecord.LatestIndexingUtc = _clock.UtcNow;
// retrieved not yet processed tasks
// retrieve not yet processed tasks
var taskRecords = _repository.Fetch(x => x.CreatedUtc >= lastIndexing)
.ToArray();
if ( taskRecords.Length == 0 )
if (taskRecords.Length == 0)
return;
Logger.Information("Processing {0} indexing tasks", taskRecords.Length);
if ( !_indexProvider.Exists(SearchIndexName) ) {
if (!_indexProvider.Exists(SearchIndexName)) {
_indexProvider.CreateIndex(SearchIndexName);
}
var updateIndexDocuments = new List<IIndexDocument>();
var deleteIndexDocuments = new List<int>();
// process Delete tasks
foreach ( var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete) ) {
var task = new IndexingTask(_contentManager, taskRecord);
deleteIndexDocuments.Add(taskRecord.ContentItemRecord.Id);
try {
_repository.Delete(taskRecord);
}
catch ( Exception ex ) {
Logger.Error(ex, "Could not delete task #{0}", taskRecord.Id);
}
}
try {
if ( deleteIndexDocuments.Count > 0 ) {
_indexProvider.Delete(SearchIndexName, deleteIndexDocuments);
}
_indexProvider.Delete(SearchIndexName, taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete).Select(t => t.Id));
}
catch ( Exception ex ) {
Logger.Warning(ex, "An error occured while remove a document from the index");
catch (Exception ex) {
Logger.Warning(ex, "An error occured while removing a document from the index");
}
// process Update tasks
foreach ( var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update) ) {
foreach (var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update)) {
var task = new IndexingTask(_contentManager, taskRecord);
try {
@@ -113,31 +87,29 @@ namespace Orchard.Core.Indexing.Services {
};
// dispatch to handlers to retrieve index information
foreach ( var handler in _handlers ) {
foreach (var handler in _handlers) {
handler.Indexing(context);
}
updateIndexDocuments.Add(context.IndexDocument);
foreach ( var handler in _handlers ) {
foreach (var handler in _handlers) {
handler.Indexed(context);
}
}
catch ( Exception ex ) {
catch (Exception ex) {
Logger.Warning(ex, "Unable to process indexing task #{0}", taskRecord.Id);
}
}
try {
if ( updateIndexDocuments.Count > 0 ) {
if (updateIndexDocuments.Count > 0) {
try {
_indexProvider.Store(SearchIndexName, updateIndexDocuments);
}
catch (Exception ex) {
Logger.Warning(ex, "An error occured while adding a document to the index");
}
}
catch ( Exception ex ) {
Logger.Warning(ex, "An error occured while adding a document to the index");
}
_settings.Update(settingsRecord);
}
}
}

43
src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs
View File

@@ -16,18 +16,15 @@ namespace Orchard.Core.Indexing.Services {
public class IndexingTaskManager : IIndexingTaskManager {
private readonly IContentManager _contentManager;
private readonly IRepository<IndexingTaskRecord> _repository;
private readonly IRepository<IndexingSettingsRecord> _settings;
private readonly IClock _clock;
public IndexingTaskManager(
IContentManager contentManager,
IRepository<IndexingTaskRecord> repository,
IRepository<IndexingSettingsRecord> settings,
IClock clock) {
_clock = clock;
_repository = repository;
_contentManager = contentManager;
_settings = settings;
Logger = NullLogger.Instance;
}
@@ -38,14 +35,7 @@ namespace Orchard.Core.Indexing.Services {
throw new ArgumentNullException("contentItem");
}
// remove previous tasks for the same content item
var tasks = _repository
.Fetch(x => x.ContentItemRecord.Id == contentItem.Id)
.ToArray();
foreach ( var task in tasks ) {
_repository.Delete(task);
}
DeleteTasks(contentItem);
var taskRecord = new IndexingTaskRecord {
CreatedUtc = _clock.UtcNow,
@@ -77,37 +67,16 @@ namespace Orchard.Core.Indexing.Services {
.ToReadOnlyCollection();
}
public void DeleteTasks(DateTime? createdBefore) {
Logger.Debug("Deleting Indexing tasks created before {0}", createdBefore);
var tasks = _repository
.Fetch(x => x.CreatedUtc <= createdBefore);
foreach (var task in tasks) {
_repository.Delete(task);
}
}
/// <summary>
/// Removes existing tasks for the specified content item
/// </summary>
public void DeleteTasks(ContentItem contentItem) {
Logger.Debug("Deleting Indexing tasks for ContentItem [{0}:{1}]", contentItem.ContentType, contentItem.Id);
var tasks = _repository
.Fetch(x => x.Id == contentItem.Id);
.Fetch(x => x.ContentItemRecord.Id == contentItem.Id)
.ToArray();
foreach (var task in tasks) {
_repository.Delete(task);
}
}
public void RebuildIndex() {
var settingsRecord = _settings.Table.FirstOrDefault();
if (settingsRecord == null) {
_settings.Create(settingsRecord = new IndexingSettingsRecord() );
}
settingsRecord.LatestIndexingUtc = new DateTime(1980, 1, 1);
_settings.Update(settingsRecord);
}
}
}

1
src/Orchard.Web/Core/Orchard.Core.csproj
View File

@@ -113,7 +113,6 @@
<Compile Include="Indexing\Lucene\DefaultIndexProvider.cs" />
<Compile Include="Indexing\Lucene\DefaultSearchBuilder.cs" />
<Compile Include="Indexing\Lucene\DefaultSearchHit.cs" />
<Compile Include="Indexing\Models\IndexingSettingsRecord.cs" />
<Compile Include="Indexing\Models\IndexingTask.cs" />
<Compile Include="Indexing\Models\IndexingTaskRecord.cs" />
<Compile Include="Indexing\Services\CreateIndexingTaskHandler.cs" />

14
src/Orchard/Indexing/IIndexProvider.cs
View File

@@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
namespace Orchard.Indexing {
public interface IIndexProvider : IDependency {
@@ -48,5 +49,16 @@ namespace Orchard.Indexing {
/// </summary>
/// <returns>A search builder instance</returns>
ISearchBuilder CreateSearchBuilder(string indexName);
/// <summary>
/// Returns the date and time when the index was last processed
/// </summary>
DateTime GetLastIndexUtc(string indexName);
/// <summary>
/// Sets the date and time when the index was last processed
/// </summary>
void SetLastIndexUtc(string indexName, DateTime lastIndexUtc);
}
}

5
src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs
View File

@@ -19,11 +19,6 @@ namespace Orchard.Tasks.Indexing {
/// </summary>
IEnumerable<IIndexingTask> GetTasks(DateTime? createdAfter);
/// <summary>
/// Deletes all indexing tasks previous to a specific date and time
/// </summary>
void DeleteTasks(DateTime? createdBefore);
/// <summary>
/// Deletes all indexing tasks assigned to a specific content item
/// </summary>