lsm/Orchard
1
0
Fork 0
mirror of https://github.com/OrchardCMS/Orchard.git synced 2025-10-14 19:04:51 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Background indexing tasks can now delete existing indexes

Browse Source 
Added new column to store the action type (update, delete)
Added new overloads to handle multiple indexing actions in the same transaction

--HG--
branch : dev
This commit is contained in:
Sebastien Ros
2010-06-03 13:12:07 -07:00
parent c677737a0d
commit bf665f71c4
10 changed files with 177 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/Orchard.Core.Tests/Indexing/DefaultIndexProviderTests.cs
View File

@@ -168,5 +168,18 @@ namespace Orchard.Tests.Indexing {
Assert.That(searchBuilder.WithField("body", "hr").Search().Count(), Is.EqualTo(1));
Assert.That(searchBuilder.WithField("body", "hr").Search().First().Id, Is.EqualTo(1));
}
[Test] public void ShouldAllowNullOrEmptyStrings() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("body", null));
_provider.Store("default", _provider.New(2).Add("body", ""));
_provider.Store("default", _provider.New(3).Add("body", "<hr></hr>", true));
var searchBuilder = _provider.CreateSearchBuilder("default");
Assert.That(searchBuilder.Get(1).Id, Is.EqualTo(1));
Assert.That(searchBuilder.Get(2).Id, Is.EqualTo(2));
Assert.That(searchBuilder.Get(3).Id, Is.EqualTo(3));
}
}
}

5
src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexDocument.cs
View File

@@ -25,6 +25,11 @@ namespace Orchard.Core.Indexing.Lucene {
public IIndexDocument Add(string name, string value, bool removeTags) {
AppendPreviousField();
if(value == null) {
value = String.Empty;
}
if(removeTags) {
value = value.RemoveTags();
}

66
src/Orchard.Web/Core/Indexing/Lucene/DefaultIndexProvider.cs
View File

@@ -1,5 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Documents;
@@ -14,7 +16,7 @@ using Orchard.Logging;
namespace Orchard.Core.Indexing.Lucene {
/// <summary>
/// Represents the default implementation of an IIndexProvider based on Lucene
/// Represents the default implementation of an IIndexProvider, based on Lucene
/// </summary>
public class DefaultIndexProvider : IIndexProvider {
private readonly IAppDataFolder _appDataFolder;
@@ -73,40 +75,65 @@ namespace Orchard.Core.Indexing.Lucene {
}
public void Store(string indexName, IIndexDocument indexDocument) {
Store(indexName, (DefaultIndexDocument)indexDocument);
Store(indexName, new [] { (DefaultIndexDocument)indexDocument });
}
public void Store(string indexName, DefaultIndexDocument indexDocument) {
public void Store(string indexName, IEnumerable<IIndexDocument> indexDocuments) {
Store(indexName, indexDocuments.Cast<DefaultIndexDocument>());
}
public void Store(string indexName, IEnumerable<DefaultIndexDocument> indexDocuments) {
if(indexDocuments.AsQueryable().Count() == 0) {
return;
}
var writer = new IndexWriter(GetDirectory(indexName), _analyzer, false, IndexWriter.MaxFieldLength.UNLIMITED);
DefaultIndexDocument current = null;
try {
var doc = CreateDocument(indexDocument);
writer.AddDocument(doc);
Logger.Debug("Document [{0}] indexed", indexDocument.Id);
foreach ( var indexDocument in indexDocuments ) {
current = indexDocument;
var doc = CreateDocument(indexDocument);
writer.AddDocument(doc);
Logger.Debug("Document [{0}] indexed", indexDocument.Id);
}
}
catch ( Exception ex ) {
Logger.Error(ex, "An unexpected error occured while removing the document [{0}] from the index [{1}].", indexDocument.Id, indexName);
Logger.Error(ex, "An unexpected error occured while add the document [{0}] from the index [{1}].", current.Id, indexName);
}
finally {
writer.Optimize();
writer.Close();
}
}
public void Delete(string indexName, int id) {
var reader = IndexReader.Open(GetDirectory(indexName), false);
public void Delete(string indexName, int documentId) {
Delete(indexName, new[] { documentId });
}
public void Delete(string indexName, IEnumerable<int> documentIds) {
if ( documentIds.AsQueryable().Count() == 0 ) {
return;
}
var reader = IndexReader.Open(GetDirectory(indexName), false);
try {
var term = new Term("id", id.ToString());
if ( reader.DeleteDocuments(term) != 0 ) {
Logger.Error("The document [{0}] could not be removed from the index [{1}]", id, indexName);
foreach (var id in documentIds) {
try {
var term = new Term("id", id.ToString());
if (reader.DeleteDocuments(term) != 0) {
Logger.Error("The document [{0}] could not be removed from the index [{1}]", id, indexName);
}
else {
Logger.Debug("Document [{0}] removed from index", id);
}
}
catch (Exception ex) {
Logger.Error(ex, "An unexpected error occured while removing the document [{0}] from the index [{1}].", id, indexName);
}
}
else {
Logger.Debug("Document [{0}] removed from index", id);
}
}
catch ( Exception ex ) {
Logger.Error(ex, "An unexpected error occured while removing the document [{0}] from the index [{1}].", id, indexName);
}
finally {
reader.Close();
@@ -121,8 +148,5 @@ namespace Orchard.Core.Indexing.Lucene {
return new DefaultSearchBuilder(GetDirectory(indexName));
}
public IIndexDocument Get(string indexName, int id) {
throw new NotImplementedException();
}
}
}

5
src/Orchard.Web/Core/Indexing/Models/IndexingTaskRecord.cs
View File

@@ -3,7 +3,12 @@ using Orchard.ContentManagement.Records;
namespace Orchard.Core.Indexing.Models {
public class IndexingTaskRecord {
public const int Update = 0;
public const int Delete = 1;
public virtual int Id { get; set; }
public virtual int Action { get; set; }
public virtual DateTime? CreatedUtc { get; set; }
public virtual ContentItemRecord ContentItemRecord { get; set; }
}

4
src/Orchard.Web/Core/Indexing/Services/CreateIndexingTaskHandler.cs
View File

@@ -19,11 +19,11 @@ namespace Orchard.Core.Indexing.Services {
}
void CreateIndexingTask(PublishContentContext context, ContentPart<CommonRecord> part) {
_indexingTaskManager.CreateTask(context.ContentItem);
_indexingTaskManager.CreateUpdateIndexTask(context.ContentItem);
}
void RemoveIndexingTask(RemoveContentContext context, ContentPart<CommonRecord> part) {
_indexingTaskManager.DeleteTasks(context.ContentItem);
_indexingTaskManager.CreateDeleteIndexTask(context.ContentItem);
}
}

67
src/Orchard.Web/Core/Indexing/Services/IndexingTaskExecutor.cs
View File

@@ -22,9 +22,9 @@ namespace Orchard.Core.Indexing.Services {
private readonly IRepository<IndexingSettingsRecord> _settings;
private readonly IEnumerable<IContentHandler> _handlers;
private IIndexProvider _indexProvider;
private IIndexManager _indexManager;
private readonly IIndexManager _indexManager;
private readonly IContentManager _contentManager;
private const string SearchIndexName = "search";
private const string SearchIndexName = "Search";
public IndexingTaskExecutor(
IClock clock,
@@ -46,7 +46,7 @@ namespace Orchard.Core.Indexing.Services {
public void Sweep() {
if(!_indexManager.HasIndexProvider()) {
if ( !_indexManager.HasIndexProvider() ) {
return;
}
@@ -55,8 +55,8 @@ namespace Orchard.Core.Indexing.Services {
// retrieve last processed index time
var settingsRecord = _settings.Table.FirstOrDefault();
if (settingsRecord == null) {
_settings.Create(settingsRecord = new IndexingSettingsRecord { LatestIndexingUtc = new DateTime(1980, 1, 1)});
if ( settingsRecord == null ) {
_settings.Create(settingsRecord = new IndexingSettingsRecord { LatestIndexingUtc = new DateTime(1980, 1, 1) });
}
var lastIndexing = settingsRecord.LatestIndexingUtc;
@@ -65,41 +65,76 @@ namespace Orchard.Core.Indexing.Services {
// retrieved not yet processed tasks
var taskRecords = _repository.Fetch(x => x.CreatedUtc >= lastIndexing)
.ToArray();
if (taskRecords.Length == 0)
if ( taskRecords.Length == 0 )
return;
Logger.Information("Processing {0} indexing tasks", taskRecords.Length);
if(!_indexProvider.Exists(SearchIndexName)) {
if ( !_indexProvider.Exists(SearchIndexName) ) {
_indexProvider.CreateIndex(SearchIndexName);
}
foreach (var taskRecord in taskRecords) {
var updateIndexDocuments = new List<IIndexDocument>();
var deleteIndexDocuments = new List<int>();
// process Delete tasks
foreach ( var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Delete) ) {
var task = new IndexingTask(_contentManager, taskRecord);
deleteIndexDocuments.Add(taskRecord.ContentItemRecord.Id);
try {
_repository.Delete(taskRecord);
}
catch ( Exception ex ) {
Logger.Error(ex, "Could not delete task #{0}", taskRecord.Id);
}
}
try {
if ( deleteIndexDocuments.Count > 0 ) {
_indexProvider.Delete(SearchIndexName, deleteIndexDocuments);
}
}
catch ( Exception ex ) {
Logger.Warning(ex, "An error occured while remove a document from the index");
}
// process Update tasks
foreach ( var taskRecord in taskRecords.Where(t => t.Action == IndexingTaskRecord.Update) ) {
var task = new IndexingTask(_contentManager, taskRecord);
try {
var task = new IndexingTask(_contentManager, taskRecord);
var context = new IndexContentContext {
ContentItem = task.ContentItem,
IndexDocument = _indexProvider.New(task.ContentItem.Id)
ContentItem = task.ContentItem,
IndexDocument = _indexProvider.New(task.ContentItem.Id)
};
// dispatch to handlers to retrieve index information
foreach (var handler in _handlers) {
foreach ( var handler in _handlers ) {
handler.Indexing(context);
}
_indexProvider.Store(SearchIndexName, context.IndexDocument);
updateIndexDocuments.Add(context.IndexDocument);
foreach ( var handler in _handlers ) {
handler.Indexed(context);
}
}
catch (Exception ex) {
catch ( Exception ex ) {
Logger.Warning(ex, "Unable to process indexing task #{0}", taskRecord.Id);
}
}
try {
if ( updateIndexDocuments.Count > 0 ) {
_indexProvider.Store(SearchIndexName, updateIndexDocuments);
}
}
catch ( Exception ex ) {
Logger.Warning(ex, "An error occured while adding a document to the index");
}
_settings.Update(settingsRecord);

26
src/Orchard.Web/Core/Indexing/Services/IndexingTaskManager.cs
View File

@@ -27,34 +27,46 @@ namespace Orchard.Core.Indexing.Services {
_clock = clock;
_repository = repository;
_contentManager = contentManager;
_settings = settings;
Logger = NullLogger.Instance;
}
public ILogger Logger { get; set; }
public void CreateTask(ContentItem contentItem) {
if (contentItem == null) {
private void CreateTask(ContentItem contentItem, int action) {
if ( contentItem == null ) {
throw new ArgumentNullException("contentItem");
}
// remove previous tasks for the same content item
var tasks = _repository
.Fetch(x => x.Id == contentItem.Id )
.Fetch(x => x.ContentItemRecord.Id == contentItem.Id)
.ToArray();
foreach (var task in tasks) {
foreach ( var task in tasks ) {
_repository.Delete(task);
}
var taskRecord = new IndexingTaskRecord {
CreatedUtc = _clock.UtcNow,
ContentItemRecord = contentItem.Record
};
CreatedUtc = _clock.UtcNow,
ContentItemRecord = contentItem.Record,
Action = action
};
_repository.Create(taskRecord);
}
public void CreateUpdateIndexTask(ContentItem contentItem) {
CreateTask(contentItem, IndexingTaskRecord.Update);
Logger.Information("Indexing task created for [{0}:{1}]", contentItem.ContentType, contentItem.Id);
}
public void CreateDeleteIndexTask(ContentItem contentItem) {
CreateTask(contentItem, IndexingTaskRecord.Delete);
Logger.Information("Deleting index task created for [{0}:{1}]", contentItem.ContentType, contentItem.Id);
}
public IEnumerable<IIndexingTask> GetTasks(DateTime? createdAfter) {

21
src/Orchard/Indexing/IIndexProvider.cs
View File

@@ -1,4 +1,6 @@
namespace Orchard.Indexing {
using System.Collections.Generic;
namespace Orchard.Indexing {
public interface IIndexProvider : IDependency {
/// <summary>
/// Creates a new index
@@ -15,11 +17,6 @@
/// </summary>
void DeleteIndex(string name);
/// <summary>
/// Loads an existing document
/// </summary>
IIndexDocument Get(string indexName, int documentId);
/// <summary>
/// Creates an empty document
/// </summary>
@@ -31,10 +28,20 @@
/// </summary>
void Store(string indexName, IIndexDocument indexDocument);
/// <summary>
/// Adds a set of new document to the index
/// </summary>
void Store(string indexName, IEnumerable<IIndexDocument> indexDocuments);
/// <summary>
/// Removes an existing document from the index
/// </summary>
void Delete(string indexName, int id);
void Delete(string indexName, int documentId);
/// <summary>
/// Removes a set of existing document from the index
/// </summary>
void Delete(string indexName, IEnumerable<int> documentIds);
/// <summary>
/// Creates a search builder for this provider

23
src/Orchard/Tasks/Indexing/IIndexingTaskManager.cs
View File

@@ -4,9 +4,30 @@ using Orchard.ContentManagement;
namespace Orchard.Tasks.Indexing {
public interface IIndexingTaskManager : IDependency {
void CreateTask(ContentItem contentItem);
/// <summary>
/// Adds a new entry in the index task table in order to create an index for the specified content item.
/// </summary>
void CreateUpdateIndexTask(ContentItem contentItem);
/// <summary>
/// Adds a new entry in the index task table in order to delete an existing index for the specified content item.
/// </summary>
void CreateDeleteIndexTask(ContentItem contentItem);
/// <summary>
/// Loads all indexing tasks created after to a specific date and time
/// </summary>
IEnumerable<IIndexingTask> GetTasks(DateTime? createdAfter);
/// <summary>
/// Deletes all indexing tasks previous to a specific date and time
/// </summary>
void DeleteTasks(DateTime? createdBefore);
/// <summary>
/// Deletes all indexing tasks assigned to a specific content item
/// </summary>
/// <param name="contentItem"></param>
void DeleteTasks(ContentItem contentItem);
}
}

2
src/Orchard/Tasks/SweepGenerator.cs
View File

@@ -14,7 +14,7 @@ namespace Orchard.Tasks {
_timer = new Timer();
_timer.Elapsed += Elapsed;
Logger = NullLogger.Instance;
Interval = TimeSpan.FromMinutes(5);
Interval = TimeSpan.FromMinutes(1);
}
public ILogger Logger { get; set; }