mirror of
https://github.com/OrchardCMS/Orchard.git
synced 2025-10-21 03:14:10 +08:00
Split Indexing in two disctint modules
- Added new Lucene module with Lucene.NET reference - Banner message is displayed if no index service is enabled when Orchard.Indexing is active - Added new MetadataExtensions for settings - Blogs, Page and Sandbox aredefined as indexed during setup --HG-- branch : dev
This commit is contained in:
Sebastien Ros
@@ -0,0 +1,24 @@
|
||||
using System.Collections.Generic;
|
||||
using Orchard.Localization;
|
||||
using Orchard.UI.Admin.Notification;
|
||||
using Orchard.UI.Notify;
|
||||
|
||||
namespace Orchard.Indexing.Services {
|
||||
public class IndexServiceNotificationProvider: INotificationProvider {
|
||||
private readonly IIndexManager _indexManager;
|
||||
|
||||
public IndexServiceNotificationProvider(IIndexManager indexManager) {
|
||||
_indexManager = indexManager;
|
||||
T = NullLocalizer.Instance;
|
||||
}
|
||||
|
||||
public Localizer T { get; set; }
|
||||
|
||||
public IEnumerable<NotifyEntry> GetNotifications() {
|
||||
|
||||
if(!_indexManager.HasIndexProvider()) {
|
||||
yield return new NotifyEntry { Message = T("You need to enable an index implementation module like Lucene." ), Type = NotifyType.Warning};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,256 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Lucene.Net.Analysis;
|
||||
using Lucene.Net.Analysis.Standard;
|
||||
using Lucene.Net.Documents;
|
||||
using Lucene.Net.Index;
|
||||
using Lucene.Net.Search;
|
||||
using Lucene.Net.Store;
|
||||
using Orchard.Environment.Configuration;
|
||||
using Orchard.FileSystems.AppData;
|
||||
using Orchard.Indexing.Models;
|
||||
using Orchard.Localization;
|
||||
using Orchard.Logging;
|
||||
using System.Xml.Linq;
|
||||
using Directory = Lucene.Net.Store.Directory;
|
||||
using Version = Lucene.Net.Util.Version;
|
||||
|
||||
namespace Orchard.Indexing.Services {
|
||||
/// <summary>
|
||||
/// Represents the default implementation of an IIndexProvider, based on Lucene
|
||||
/// </summary>
|
||||
public class LuceneIndexProvider : IIndexProvider {
|
||||
private readonly IAppDataFolder _appDataFolder;
|
||||
private readonly ShellSettings _shellSettings;
|
||||
public static readonly Version LuceneVersion = Version.LUCENE_29;
|
||||
private readonly Analyzer _analyzer ;
|
||||
private readonly string _basePath;
|
||||
public static readonly DateTime DefaultMinDateTime = new DateTime(1980, 1, 1);
|
||||
public static readonly string Settings = "Settings";
|
||||
public static readonly string LastIndexUtc = "LastIndexedUtc";
|
||||
|
||||
public LuceneIndexProvider(IAppDataFolder appDataFolder, ShellSettings shellSettings) {
|
||||
_appDataFolder = appDataFolder;
|
||||
_shellSettings = shellSettings;
|
||||
_analyzer = CreateAnalyzer();
|
||||
|
||||
// TODO: (sebros) Find a common way to get where tenant's specific files should go. "Sites/Tenant" is hard coded in multiple places
|
||||
_basePath = _appDataFolder.Combine("Sites", _shellSettings.Name, "Indexes");
|
||||
|
||||
Logger = NullLogger.Instance;
|
||||
|
||||
// Ensures the directory exists
|
||||
EnsureDirectoryExists();
|
||||
|
||||
T = NullLocalizer.Instance;
|
||||
Logger = NullLogger.Instance;
|
||||
}
|
||||
|
||||
public Localizer T { get; set; }
|
||||
public ILogger Logger { get; set; }
|
||||
|
||||
public static Analyzer CreateAnalyzer() {
|
||||
// StandardAnalyzer does lower-case and stop-word filtering. It also removes punctuation
|
||||
return new StandardAnalyzer(LuceneVersion);
|
||||
}
|
||||
|
||||
private void EnsureDirectoryExists() {
|
||||
var directory = new DirectoryInfo(_appDataFolder.MapPath(_basePath));
|
||||
if(!directory.Exists) {
|
||||
directory.Create();
|
||||
}
|
||||
}
|
||||
|
||||
protected virtual Directory GetDirectory(string indexName) {
|
||||
var directoryInfo = new DirectoryInfo(_appDataFolder.MapPath(_appDataFolder.Combine(_basePath, indexName)));
|
||||
return FSDirectory.Open(directoryInfo);
|
||||
}
|
||||
|
||||
private static Document CreateDocument(LuceneDocumentIndex indexDocument) {
|
||||
var doc = new Document();
|
||||
|
||||
indexDocument.PrepareForIndexing();
|
||||
foreach(var field in indexDocument.Fields) {
|
||||
doc.Add(field);
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
public bool Exists(string indexName) {
|
||||
return new DirectoryInfo(_appDataFolder.MapPath(_appDataFolder.Combine(_basePath, indexName))).Exists;
|
||||
}
|
||||
|
||||
public bool IsEmpty(string indexName) {
|
||||
if ( !Exists(indexName) ) {
|
||||
return true;
|
||||
}
|
||||
|
||||
var reader = IndexReader.Open(GetDirectory(indexName), true);
|
||||
|
||||
try {
|
||||
return reader.NumDocs() == 0;
|
||||
}
|
||||
finally {
|
||||
reader.Close();
|
||||
}
|
||||
}
|
||||
|
||||
public int NumDocs(string indexName) {
|
||||
if ( !Exists(indexName) ) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
var reader = IndexReader.Open(GetDirectory(indexName), true);
|
||||
|
||||
try {
|
||||
return reader.NumDocs();
|
||||
}
|
||||
finally {
|
||||
reader.Close();
|
||||
}
|
||||
}
|
||||
|
||||
public void CreateIndex(string indexName) {
|
||||
var writer = new IndexWriter(GetDirectory(indexName), _analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED);
|
||||
writer.Close();
|
||||
|
||||
Logger.Information("Index [{0}] created", indexName);
|
||||
}
|
||||
|
||||
public void DeleteIndex(string indexName) {
|
||||
new DirectoryInfo(_appDataFolder.MapPath(_appDataFolder.Combine(_basePath, indexName)))
|
||||
.Delete(true);
|
||||
|
||||
var settingsFileName = GetSettingsFileName(indexName);
|
||||
if(File.Exists(settingsFileName)) {
|
||||
File.Delete(settingsFileName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Store(string indexName, IDocumentIndex indexDocument) {
|
||||
Store(indexName, new [] { (LuceneDocumentIndex)indexDocument });
|
||||
}
|
||||
|
||||
public void Store(string indexName, IEnumerable<IDocumentIndex> indexDocuments) {
|
||||
Store(indexName, indexDocuments.Cast<LuceneDocumentIndex>());
|
||||
}
|
||||
|
||||
public void Store(string indexName, IEnumerable<LuceneDocumentIndex> indexDocuments) {
|
||||
if(indexDocuments.AsQueryable().Count() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Remove any previous document for these content items
|
||||
Delete(indexName, indexDocuments.Select(i => i.ContentItemId));
|
||||
|
||||
var writer = new IndexWriter(GetDirectory(indexName), _analyzer, false, IndexWriter.MaxFieldLength.UNLIMITED);
|
||||
LuceneDocumentIndex current = null;
|
||||
|
||||
try {
|
||||
|
||||
foreach ( var indexDocument in indexDocuments ) {
|
||||
current = indexDocument;
|
||||
var doc = CreateDocument(indexDocument);
|
||||
|
||||
writer.AddDocument(doc);
|
||||
Logger.Debug("Document [{0}] indexed", indexDocument.ContentItemId);
|
||||
}
|
||||
}
|
||||
catch ( Exception ex ) {
|
||||
Logger.Error(ex, "An unexpected error occured while add the document [{0}] from the index [{1}].", current.ContentItemId, indexName);
|
||||
}
|
||||
finally {
|
||||
writer.Optimize();
|
||||
writer.Close();
|
||||
}
|
||||
}
|
||||
|
||||
public void Delete(string indexName, int documentId) {
|
||||
Delete(indexName, new[] { documentId });
|
||||
}
|
||||
|
||||
public void Delete(string indexName, IEnumerable<int> documentIds) {
|
||||
if (!documentIds.Any()) {
|
||||
return;
|
||||
}
|
||||
|
||||
var writer = new IndexWriter(GetDirectory(indexName), _analyzer, false, IndexWriter.MaxFieldLength.UNLIMITED);
|
||||
|
||||
try {
|
||||
var query = new BooleanQuery();
|
||||
|
||||
try {
|
||||
foreach (var id in documentIds) {
|
||||
query.Add(new BooleanClause(new TermQuery(new Term("id", id.ToString())), BooleanClause.Occur.SHOULD));
|
||||
}
|
||||
|
||||
writer.DeleteDocuments(query);
|
||||
}
|
||||
catch (Exception ex) {
|
||||
Logger.Error(ex, "An unexpected error occured while removing the documents [{0}] from the index [{1}].", String.Join(", ", documentIds), indexName);
|
||||
}
|
||||
}
|
||||
finally {
|
||||
writer.Close();
|
||||
}
|
||||
}
|
||||
|
||||
public IDocumentIndex New(int documentId) {
|
||||
return new LuceneDocumentIndex(documentId, T);
|
||||
}
|
||||
|
||||
public ISearchBuilder CreateSearchBuilder(string indexName) {
|
||||
return new LuceneSearchBuilder(GetDirectory(indexName));
|
||||
}
|
||||
|
||||
private string GetSettingsFileName(string indexName) {
|
||||
return _appDataFolder.MapPath(_appDataFolder.Combine(_basePath, indexName + ".settings.xml"));
|
||||
}
|
||||
|
||||
public DateTime GetLastIndexUtc(string indexName) {
|
||||
var settingsFileName = GetSettingsFileName(indexName);
|
||||
|
||||
return File.Exists(settingsFileName)
|
||||
? DateTime.Parse(XDocument.Load(settingsFileName).Descendants(LastIndexUtc).First().Value)
|
||||
: DefaultMinDateTime;
|
||||
}
|
||||
|
||||
public void SetLastIndexUtc(string indexName, DateTime lastIndexUtc) {
|
||||
if ( lastIndexUtc < DefaultMinDateTime ) {
|
||||
lastIndexUtc = DefaultMinDateTime;
|
||||
}
|
||||
|
||||
XDocument doc;
|
||||
var settingsFileName = GetSettingsFileName(indexName);
|
||||
if ( !File.Exists(settingsFileName) ) {
|
||||
EnsureDirectoryExists();
|
||||
doc = new XDocument(
|
||||
new XElement(Settings,
|
||||
new XElement(LastIndexUtc, lastIndexUtc.ToString("s"))));
|
||||
}
|
||||
else {
|
||||
doc = XDocument.Load(settingsFileName);
|
||||
doc.Element(Settings).Element(LastIndexUtc).Value = lastIndexUtc.ToString("s");
|
||||
}
|
||||
|
||||
doc.Save(settingsFileName);
|
||||
}
|
||||
|
||||
public string[] GetFields(string indexName) {
|
||||
if ( !Exists(indexName) ) {
|
||||
return new string[0];
|
||||
}
|
||||
|
||||
var reader = IndexReader.Open(GetDirectory(indexName), true);
|
||||
|
||||
try {
|
||||
return reader.GetFieldNames(IndexReader.FieldOption.ALL).ToArray();
|
||||
}
|
||||
finally {
|
||||
reader.Close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,329 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using Lucene.Net.Analysis;
|
||||
using Lucene.Net.Index;
|
||||
using Lucene.Net.Search;
|
||||
using Lucene.Net.Store;
|
||||
using Orchard.Indexing.Models;
|
||||
using Orchard.Logging;
|
||||
using Lucene.Net.Documents;
|
||||
using Lucene.Net.QueryParsers;
|
||||
|
||||
namespace Orchard.Indexing.Services {
|
||||
public class LuceneSearchBuilder : ISearchBuilder {
|
||||
|
||||
private const int MaxResults = Int16.MaxValue;
|
||||
|
||||
private readonly Directory _directory;
|
||||
|
||||
private readonly List<BooleanClause> _clauses;
|
||||
private readonly List<BooleanClause> _filters;
|
||||
private int _count;
|
||||
private int _skip;
|
||||
private string _sort;
|
||||
private bool _sortDescending;
|
||||
private bool _asFilter;
|
||||
|
||||
// pending clause attributes
|
||||
private BooleanClause.Occur _occur;
|
||||
private bool _exactMatch;
|
||||
private float _boost;
|
||||
private Query _query;
|
||||
|
||||
public ILogger Logger { get; set; }
|
||||
|
||||
public LuceneSearchBuilder(Directory directory) {
|
||||
_directory = directory;
|
||||
Logger = NullLogger.Instance;
|
||||
|
||||
_count = MaxResults;
|
||||
_skip = 0;
|
||||
_clauses = new List<BooleanClause>();
|
||||
_filters = new List<BooleanClause>();
|
||||
_sort = String.Empty;
|
||||
_sortDescending = true;
|
||||
|
||||
InitPendingClause();
|
||||
}
|
||||
public ISearchBuilder Parse(string defaultField, string query) {
|
||||
return Parse(new string[] {defaultField}, query);
|
||||
}
|
||||
|
||||
public ISearchBuilder Parse(string[] defaultFields, string query) {
|
||||
if ( defaultFields.Length == 0 ) {
|
||||
throw new ArgumentException("Default field can't be empty");
|
||||
}
|
||||
|
||||
if ( String.IsNullOrWhiteSpace(query) ) {
|
||||
throw new ArgumentException("Query can't be empty");
|
||||
}
|
||||
|
||||
var analyzer = LuceneIndexProvider.CreateAnalyzer();
|
||||
foreach ( var defaultField in defaultFields ) {
|
||||
var clause = new BooleanClause(new QueryParser(LuceneIndexProvider.LuceneVersion, defaultField, analyzer).Parse(query), BooleanClause.Occur.SHOULD);
|
||||
_clauses.Add(clause);
|
||||
}
|
||||
|
||||
_query = null;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder WithField(string field, int value) {
|
||||
CreatePendingClause();
|
||||
_query = NumericRangeQuery.NewIntRange(field, value, value, true, true);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder WithinRange(string field, int min, int max) {
|
||||
CreatePendingClause();
|
||||
_query = NumericRangeQuery.NewIntRange(field, min, max, true, true);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder WithField(string field, float value) {
|
||||
CreatePendingClause();
|
||||
_query = NumericRangeQuery.NewFloatRange(field, value, value, true, true);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder WithinRange(string field, float min, float max) {
|
||||
CreatePendingClause();
|
||||
_query = NumericRangeQuery.NewFloatRange(field, min, max, true, true);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder WithField(string field, bool value) {
|
||||
CreatePendingClause();
|
||||
_query = new TermQuery(new Term(field, value.ToString()));
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder WithField(string field, DateTime value) {
|
||||
CreatePendingClause();
|
||||
_query = new TermQuery(new Term(field, DateTools.DateToString(value, DateTools.Resolution.SECOND)));
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder WithinRange(string field, DateTime min, DateTime max) {
|
||||
CreatePendingClause();
|
||||
_query = new TermRangeQuery(field, DateTools.DateToString(min, DateTools.Resolution.SECOND), DateTools.DateToString(max, DateTools.Resolution.SECOND), true, true);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder WithinRange(string field, string min, string max) {
|
||||
CreatePendingClause();
|
||||
_query = new TermRangeQuery(field, QueryParser.Escape(min.ToLower()), QueryParser.Escape(min.ToLower()), true, true);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder WithField(string field, string value) {
|
||||
CreatePendingClause();
|
||||
|
||||
if ( !String.IsNullOrWhiteSpace(value) ) {
|
||||
_query = new TermQuery(new Term(field, QueryParser.Escape(value.ToLower())));
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder Mandatory() {
|
||||
_occur = BooleanClause.Occur.MUST;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder Forbidden() {
|
||||
_occur = BooleanClause.Occur.MUST_NOT;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder ExactMatch() {
|
||||
_exactMatch = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder Weighted(float weight) {
|
||||
_boost = weight;
|
||||
return this;
|
||||
}
|
||||
|
||||
private void InitPendingClause() {
|
||||
_occur = BooleanClause.Occur.SHOULD;
|
||||
_exactMatch = false;
|
||||
_query = null;
|
||||
_boost = 0;
|
||||
_asFilter = false;
|
||||
}
|
||||
|
||||
private void CreatePendingClause() {
|
||||
if(_query == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (_boost != 0) {
|
||||
_query.SetBoost(_boost);
|
||||
}
|
||||
|
||||
if(!_exactMatch) {
|
||||
var termQuery = _query as TermQuery;
|
||||
if(termQuery != null) {
|
||||
_query = new PrefixQuery(termQuery.GetTerm());
|
||||
}
|
||||
}
|
||||
if ( _asFilter ) {
|
||||
_filters.Add(new BooleanClause(_query, _occur));
|
||||
}
|
||||
else {
|
||||
_clauses.Add(new BooleanClause(_query, _occur));
|
||||
}
|
||||
}
|
||||
|
||||
public ISearchBuilder SortBy(string name) {
|
||||
_sort = name;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder Ascending() {
|
||||
_sortDescending = false;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder AsFilter() {
|
||||
_asFilter = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ISearchBuilder Slice(int skip, int count) {
|
||||
if ( skip < 0 ) {
|
||||
throw new ArgumentException("Skip must be greater or equal to zero");
|
||||
}
|
||||
|
||||
if ( count <= 0 ) {
|
||||
throw new ArgumentException("Count must be greater than zero");
|
||||
}
|
||||
|
||||
_skip = skip;
|
||||
_count = count;
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
private Query CreateQuery() {
|
||||
CreatePendingClause();
|
||||
|
||||
var query = new BooleanQuery();
|
||||
|
||||
foreach( var clause in _clauses)
|
||||
query.Add(clause);
|
||||
|
||||
|
||||
if ( query.Clauses().Count == 0 ) { // get all documents ?
|
||||
query.Add(new TermRangeQuery("id", "0", "9", true, true), BooleanClause.Occur.SHOULD);
|
||||
}
|
||||
|
||||
Query finalQuery = query;
|
||||
|
||||
if(_filters.Count > 0) {
|
||||
var filter = new BooleanQuery();
|
||||
foreach( var clause in _filters)
|
||||
filter.Add(clause);
|
||||
var queryFilter = new QueryWrapperFilter(filter);
|
||||
|
||||
finalQuery = new FilteredQuery(query, queryFilter);
|
||||
}
|
||||
|
||||
Logger.Debug("New search query: {0}", finalQuery.ToString());
|
||||
return finalQuery;
|
||||
}
|
||||
|
||||
public IEnumerable<ISearchHit> Search() {
|
||||
var query = CreateQuery();
|
||||
|
||||
IndexSearcher searcher;
|
||||
|
||||
try {
|
||||
searcher = new IndexSearcher(_directory, true);
|
||||
}
|
||||
catch {
|
||||
// index might not exist if it has been rebuilt
|
||||
Logger.Information("Attempt to read a none existing index");
|
||||
return Enumerable.Empty<ISearchHit>();
|
||||
}
|
||||
|
||||
try {
|
||||
var sort = String.IsNullOrEmpty(_sort)
|
||||
? Sort.RELEVANCE
|
||||
: new Sort(new SortField(_sort, CultureInfo.InvariantCulture, _sortDescending));
|
||||
var collector = TopFieldCollector.create(
|
||||
sort,
|
||||
_count + _skip,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
true);
|
||||
|
||||
Logger.Debug("Searching: {0}", query.ToString());
|
||||
searcher.Search(query, collector);
|
||||
|
||||
var results = collector.TopDocs().scoreDocs
|
||||
.Skip(_skip)
|
||||
.Select(scoreDoc => new LuceneSearchHit(searcher.Doc(scoreDoc.doc), scoreDoc.score))
|
||||
.ToList();
|
||||
|
||||
Logger.Debug("Search results: {0}", results.Count);
|
||||
|
||||
return results;
|
||||
}
|
||||
finally {
|
||||
searcher.Close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public int Count() {
|
||||
var query = CreateQuery();
|
||||
IndexSearcher searcher;
|
||||
|
||||
try {
|
||||
searcher = new IndexSearcher(_directory, true);
|
||||
}
|
||||
catch {
|
||||
// index might not exist if it has been rebuilt
|
||||
Logger.Information("Attempt to read a none existing index");
|
||||
return 0;
|
||||
}
|
||||
|
||||
try {
|
||||
var hits = searcher.Search(query, Int16.MaxValue);
|
||||
Logger.Information("Search results: {0}", hits.scoreDocs.Length);
|
||||
var length = hits.scoreDocs.Length;
|
||||
return Math.Min(length - _skip, _count) ;
|
||||
}
|
||||
finally {
|
||||
searcher.Close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public ISearchHit Get(int documentId) {
|
||||
var query = new TermQuery(new Term("id", documentId.ToString()));
|
||||
|
||||
var searcher = new IndexSearcher(_directory, true);
|
||||
try {
|
||||
var hits = searcher.Search(query, 1);
|
||||
Logger.Information("Search results: {0}", hits.scoreDocs.Length);
|
||||
if ( hits.scoreDocs.Length > 0 ) {
|
||||
return new LuceneSearchHit(searcher.Doc(hits.scoreDocs[0].doc), hits.scoreDocs[0].score);
|
||||
}
|
||||
else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
finally {
|
||||
searcher.Close();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user