From a31111d33155ff591435b8d9f12fcfae615c0b8c Mon Sep 17 00:00:00 2001 From: Sebastien Ros Date: Wed, 16 Jun 2010 14:00:08 -0700 Subject: [PATCH] Update ISearchBuilder to pass the string unprocessed to Lucene --HG-- branch : dev --- .../Indexing/DefaultSearchBuilderTests.cs | 58 ++++++++- .../Indexing/Lucene/DefaultSearchBuilder.cs | 119 ++++++++++++++---- .../Orchard.Search/Services/SearchService.cs | 5 +- src/Orchard/Indexing/ISearchBuilder.cs | 8 +- 4 files changed, 154 insertions(+), 36 deletions(-) diff --git a/src/Orchard.Core.Tests/Indexing/DefaultSearchBuilderTests.cs b/src/Orchard.Core.Tests/Indexing/DefaultSearchBuilderTests.cs index b4f2d5c37..a9b56e16f 100644 --- a/src/Orchard.Core.Tests/Indexing/DefaultSearchBuilderTests.cs +++ b/src/Orchard.Core.Tests/Indexing/DefaultSearchBuilderTests.cs @@ -3,17 +3,13 @@ using System.IO; using System.Linq; using Autofac; using NUnit.Framework; -using Orchard.Environment; using Orchard.Environment.Configuration; using Orchard.FileSystems.AppData; -using Orchard.FileSystems.VirtualPath; using Orchard.Indexing; using Orchard.Core.Indexing.Lucene; -using Orchard.Services; -using Orchard.Tests.Environment.Configuration; using Orchard.Tests.FileSystems.AppData; -namespace Orchard.Tests.Indexing { +namespace Orchard.Core.Tests.Indexing { public class DefaultSearchBuilderTests { private IContainer _container; private IIndexProvider _provider; @@ -194,5 +190,57 @@ namespace Orchard.Tests.Indexing { Assert.That(cpp.Count(), Is.EqualTo(2)); } + + [Test] + public void ShouldHandleMandatoryFields() { + _provider.CreateIndex("default"); + _provider.Store("default", _provider.New(1).Add("body", "Orchard has been developped in C#")); + _provider.Store("default", _provider.New(2).Add("body", "Windows has been developped in C++")); + + Assert.That(_searchBuilder.WithField("body", "develop").Search().ToList().Count(), Is.EqualTo(2)); + Assert.That(_searchBuilder.WithField("body", "develop").WithField("body", "Orchard").Search().ToList().Count(), Is.EqualTo(2)); + Assert.That(_searchBuilder.WithField("body", "develop").WithField("body", "Orchard").Mandatory().Search().ToList().Count(), Is.EqualTo(1)); + Assert.That(_searchBuilder.WithField("body", "develop").WithField("body", "Orchard").Mandatory().Search().First().Id, Is.EqualTo(1)); + } + + [Test] + public void ShouldHandleForbiddenFields() { + _provider.CreateIndex("default"); + _provider.Store("default", _provider.New(1).Add("body", "Orchard has been developped in C#")); + _provider.Store("default", _provider.New(2).Add("body", "Windows has been developped in C++")); + + Assert.That(_searchBuilder.WithField("body", "develop").Search().ToList().Count(), Is.EqualTo(2)); + Assert.That(_searchBuilder.WithField("body", "develop").WithField("body", "Orchard").Search().ToList().Count(), Is.EqualTo(2)); + Assert.That(_searchBuilder.WithField("body", "develop").WithField("body", "Orchard").Forbidden().Search().ToList().Count(), Is.EqualTo(1)); + Assert.That(_searchBuilder.WithField("body", "develop").WithField("body", "Orchard").Forbidden().Search().First().Id, Is.EqualTo(2)); + } + + [Test] + public void ShouldHandleWeight() { + _provider.CreateIndex("default"); + _provider.Store("default", _provider.New(1).Add("body", "Orchard has been developped in C#")); + _provider.Store("default", _provider.New(2).Add("body", "Windows has been developped in C++")); + + Assert.That(_searchBuilder.WithField("body", "develop").WithField("body", "Orchard").Weighted(2).Search().First().Id, Is.EqualTo(1)); + } + + [Test] + public void ShouldParseLuceneQueries() { + _provider.CreateIndex("default"); + _provider.Store("default", _provider.New(1).Add("body", "Bradley is in the kitchen.").Add("title", "Beer and takos")); + _provider.Store("default", _provider.New(2).Add("body", "Renaud is also in the kitchen.").Add("title", "A love affair")); + _provider.Store("default", _provider.New(3).Add("body", "Bertrand is a little bit jealous.").Add("title", "Soap opera")); + + Assert.That(_searchBuilder.Parse(new[] { "body" }, "kitchen").Count(), Is.EqualTo(2)); + Assert.That(_searchBuilder.Parse(new[] { "body" }, "kitchen bertrand").Count(), Is.EqualTo(3)); + Assert.That(_searchBuilder.Parse(new[] { "body" }, "kitchen +bertrand").Count(), Is.EqualTo(1)); + Assert.That(_searchBuilder.Parse(new[] { "body" }, "+kitchen +bertrand").Count(), Is.EqualTo(0)); + Assert.That(_searchBuilder.Parse(new[] { "body" }, "kit").Count(), Is.EqualTo(0)); + Assert.That(_searchBuilder.Parse(new[] { "body" }, "kit*").Count(), Is.EqualTo(2)); + Assert.That(_searchBuilder.Parse(new[] { "body", "title" }, "bradley love^3 soap").Count(), Is.EqualTo(3)); + Assert.That(_searchBuilder.Parse(new[] { "body", "title" }, "bradley love^3 soap").Search().First().Id, Is.EqualTo(2)); + } + + } } diff --git a/src/Orchard.Web/Core/Indexing/Lucene/DefaultSearchBuilder.cs b/src/Orchard.Web/Core/Indexing/Lucene/DefaultSearchBuilder.cs index a68965594..a9878c268 100644 --- a/src/Orchard.Web/Core/Indexing/Lucene/DefaultSearchBuilder.cs +++ b/src/Orchard.Web/Core/Indexing/Lucene/DefaultSearchBuilder.cs @@ -19,7 +19,7 @@ namespace Orchard.Core.Indexing.Lucene { private readonly Directory _directory; - private readonly Dictionary _fields; + private readonly Dictionary _fields; private int _count; private int _skip; private readonly Dictionary _before; @@ -28,7 +28,15 @@ namespace Orchard.Core.Indexing.Lucene { private bool _sortDescending; private string _parse; private readonly Analyzer _analyzer; - private string _defaultField; + private string[] _defaultFields; + + // pending clause attributes + private string _field; + private string _terms; + private BooleanClause.Occur _occur; + private bool _prefix; + private bool _stem; + private float _boost; public ILogger Logger { get; set; } @@ -40,15 +48,17 @@ namespace Orchard.Core.Indexing.Lucene { _skip = 0; _before = new Dictionary(); _after = new Dictionary(); - _fields = new Dictionary(); + _fields = new Dictionary(); _sort = String.Empty; _sortDescending = true; _parse = String.Empty; _analyzer = DefaultIndexProvider.CreateAnalyzer(); + + InitPendingClause(); } - public ISearchBuilder Parse(string defaultField, string query) { - if ( String.IsNullOrWhiteSpace(defaultField) ) { + public ISearchBuilder Parse(string[] defaultFields, string query) { + if ( defaultFields.Length == 0 ) { throw new ArgumentException("Default field can't be empty"); } @@ -56,32 +66,84 @@ namespace Orchard.Core.Indexing.Lucene { throw new ArgumentException("Query can't be empty"); } - _defaultField = defaultField; + _defaultFields = defaultFields; _parse = query; return this; } public ISearchBuilder WithField(string field, string value) { - return WithField(field, value, true); + CreatePendingClause(); + + _field = field; + _terms = value; + + return this; } - public ISearchBuilder WithField(string field, string value, bool wildcardSearch) { + public ISearchBuilder Mandatory() { + _occur = BooleanClause.Occur.MUST; + return this; + } + + public ISearchBuilder Forbidden() { + _occur = BooleanClause.Occur.MUST_NOT; + return this; + } + + public ISearchBuilder ExactMatch() { + _prefix = false; + _stem = false; + return this; + } + + public ISearchBuilder Weighted(float weight) { + _boost = weight; + return this; + } + + private void InitPendingClause() { + _field = String.Empty; + _terms = String.Empty; + _occur = BooleanClause.Occur.SHOULD; + _prefix = true; + _stem = true; + _boost = 0; + } + + private void CreatePendingClause() { + if(String.IsNullOrWhiteSpace(_field) || String.IsNullOrWhiteSpace(_terms)) { + return; + } var tokens = new List(); - using(var sr = new System.IO.StringReader(value)) { - var stream = _analyzer.TokenStream(field, sr); - while(stream.IncrementToken()) { - tokens.Add(((TermAttribute)stream.GetAttribute(typeof(TermAttribute))).Term()); + using ( var sr = new System.IO.StringReader(_terms) ) { + var stream = _analyzer.TokenStream(_field, sr); + + if(_stem) { + stream = new PorterStemFilter(stream); + } + + while ( stream.IncrementToken() ) { + tokens.Add(( (TermAttribute)stream.GetAttribute(typeof(TermAttribute)) ).Term()); } } - _fields[field] = tokens - .Where(k => !String.IsNullOrWhiteSpace(k)) - .Select(QueryParser.Escape) - .Select(k => wildcardSearch ? (Query)new PrefixQuery(new Term(field, k)) : new TermQuery(new Term(k))) - .ToArray(); - - return this; + var clauses = tokens + .Where(k => !String.IsNullOrWhiteSpace(k)) // remove empty strings + .Select(QueryParser.Escape) // escape special chars (e.g. C#) + .Select(k => new Term(_field, k)) // creates the Term instance + .Select(t => _prefix ? new PrefixQuery(t) as Query : new TermQuery(t) as Query) // apply the corresponding Query + .Select(q => { + if (_boost != 0) q.SetBoost(_boost); + return q; + }) + .Select(q => new BooleanClause(q, _occur)); // apply the corresponding clause + + if ( !_fields.ContainsKey(_field) ) { + _fields[_field] = new BooleanClause[0]; + } + + _fields[_field] = _fields[_field].Union(clauses).ToArray(); } public ISearchBuilder After(string name, DateTime date) { @@ -120,16 +182,23 @@ namespace Orchard.Core.Indexing.Lucene { } private Query CreateQuery() { - if(!String.IsNullOrWhiteSpace(_parse)) { - return new QueryParser(DefaultIndexProvider.LuceneVersion, _defaultField, DefaultIndexProvider.CreateAnalyzer()).Parse(_parse); - } + CreatePendingClause(); var query = new BooleanQuery(); + if(!String.IsNullOrWhiteSpace(_parse)) { + + foreach ( var defaultField in _defaultFields ) { + var clause = new BooleanClause(new QueryParser(DefaultIndexProvider.LuceneVersion, defaultField, DefaultIndexProvider.CreateAnalyzer()).Parse(_parse), BooleanClause.Occur.SHOULD); + query.Add(clause); + } + } + + if ( _fields.Keys.Count > 0 ) { // apply specific filters if defined - foreach ( var filters in _fields.Values ) { - foreach(var filter in filters) - query.Add(filter, BooleanClause.Occur.SHOULD); + foreach ( var clauses in _fields.Values ) { + foreach( var clause in clauses) + query.Add(clause); } } diff --git a/src/Orchard.Web/Modules/Orchard.Search/Services/SearchService.cs b/src/Orchard.Web/Modules/Orchard.Search/Services/SearchService.cs index 338d101bc..d21d4f991 100644 --- a/src/Orchard.Web/Modules/Orchard.Search/Services/SearchService.cs +++ b/src/Orchard.Web/Modules/Orchard.Search/Services/SearchService.cs @@ -37,11 +37,10 @@ namespace Orchard.Search.Services return null; var searchBuilder = _indexManager.GetSearchIndexProvider().CreateSearchBuilder(SearchIndexName) - .WithField("title", query) - .WithField("body", query); + .Parse(new [] {"title", "body"}, query); if(HttpContext.Current != null) { - searchBuilder.WithField("culture", _cultureManager.GetCurrentCulture(HttpContext.Current)); + searchBuilder.WithField("culture", _cultureManager.GetCurrentCulture(HttpContext.Current)).Mandatory(); } var totalCount = searchBuilder.Count(); diff --git a/src/Orchard/Indexing/ISearchBuilder.cs b/src/Orchard/Indexing/ISearchBuilder.cs index 83e3ffda4..ebbf3f890 100644 --- a/src/Orchard/Indexing/ISearchBuilder.cs +++ b/src/Orchard/Indexing/ISearchBuilder.cs @@ -3,11 +3,13 @@ using System.Collections.Generic; namespace Orchard.Indexing { public interface ISearchBuilder { - - ISearchBuilder Parse(string defaultField, string query); + ISearchBuilder Parse(string[] defaultFields, string query); ISearchBuilder WithField(string field, string value); - ISearchBuilder WithField(string field, string value, bool wildcardSearch); + ISearchBuilder Mandatory(); + ISearchBuilder Forbidden(); + ISearchBuilder ExactMatch(); + ISearchBuilder Weighted(float weight); ISearchBuilder After(string name, DateTime date); ISearchBuilder Before(string name, DateTime date);