Complete refactoring on Indexing module

Changed Parse() to use lucene syntax
Moved everything to Orchard.Indexing
New filters for search

--HG--
branch : dev
This commit is contained in:
Sebastien Ros
2010-06-17 16:21:29 -07:00
parent a1cc5d579b
commit 40769d6f2a
47 changed files with 775 additions and 413 deletions

View File

@@ -0,0 +1,253 @@
using System;
using System.IO;
using System.Linq;
using Autofac;
using NUnit.Framework;
using Orchard.Environment.Configuration;
using Orchard.FileSystems.AppData;
using Orchard.Indexing;
using Orchard.Indexing.Services;
using Orchard.Tests.FileSystems.AppData;
namespace Orchard.Tests.Modules.Indexing {
public class LuceneIndexProviderTests {
private IContainer _container;
private IIndexProvider _provider;
private IAppDataFolder _appDataFolder;
private ShellSettings _shellSettings;
private readonly string _basePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
[TestFixtureTearDown]
public void Clean() {
Directory.Delete(_basePath, true);
}
[SetUp]
public void Setup() {
if (Directory.Exists(_basePath)) {
Directory.Delete(_basePath, true);
}
Directory.CreateDirectory(_basePath);
_appDataFolder = AppDataFolderTests.CreateAppDataFolder(_basePath);
var builder = new ContainerBuilder();
builder.RegisterType<LuceneIndexProvider>().As<IIndexProvider>();
builder.RegisterInstance(_appDataFolder).As<IAppDataFolder>();
// setting up a ShellSettings instance
_shellSettings = new ShellSettings { Name = "My Site" };
builder.RegisterInstance(_shellSettings).As<ShellSettings>();
_container = builder.Build();
_provider = _container.Resolve<IIndexProvider>();
}
private string[] Indexes() {
return new DirectoryInfo(Path.Combine(_basePath, "Sites", "My Site", "Indexes")).GetDirectories().Select(d => d.Name).ToArray();
}
[Test]
public void IndexProviderShouldCreateNewIndex() {
Assert.That(Indexes().Length, Is.EqualTo(0));
_provider.CreateIndex("default");
Assert.That(Indexes().Length, Is.EqualTo(1));
}
[Test]
public void IndexProviderShouldOverwriteAlreadyExistingIndex() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("body", null));
Assert.That(_provider.IsEmpty("default"), Is.False);
_provider.CreateIndex("default");
Assert.That(_provider.IsEmpty("default"), Is.True);
}
[Test]
public void IndexProviderShouldDeleteExistingIndex() {
Assert.That(Indexes().Length, Is.EqualTo(0));
_provider.CreateIndex("default");
Assert.That(Indexes().Length, Is.EqualTo(1));
_provider.DeleteIndex("default");
Assert.That(Indexes().Length, Is.EqualTo(0));
}
[Test]
public void IndexProviderShouldListExistingIndexes() {
Assert.That(Indexes().Length, Is.EqualTo(0));
_provider.CreateIndex("default");
Assert.That(Indexes().Length, Is.EqualTo(1));
Assert.That(Indexes()[0], Is.EqualTo("default"));
_provider.CreateIndex("foo");
Assert.That(Indexes().Length, Is.EqualTo(2));
}
[Test]
public void ANewIndexShouldBeEmpty() {
_provider.CreateIndex("default");
var searchBuilder = _provider.CreateSearchBuilder("default");
var hits = searchBuilder.Search();
Assert.That(hits.Count(), Is.EqualTo(0));
}
[Test]
public void DocumentsShouldBeSearchableById() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(42));
var searchBuilder = _provider.CreateSearchBuilder("default");
var hit = searchBuilder.Get(42);
Assert.IsNotNull(hit);
Assert.That(hit.ContentItemId, Is.EqualTo(42));
hit = searchBuilder.Get(1);
Assert.IsNull(hit);
}
[Test]
public void PropertiesShouldNotBeLost() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(42).Add("prop1", "value1").Store());
var hit = _provider.CreateSearchBuilder("default").Get(42);
Assert.IsNotNull(hit);
Assert.That(hit.ContentItemId, Is.EqualTo(42));
Assert.That(hit.GetString("prop1"), Is.EqualTo("value1"));
}
[Test]
public void ShouldHandleMultipleIndexes() {
_provider.CreateIndex("default1");
_provider.Store("default1", _provider.New(1));
_provider.CreateIndex("default2");
_provider.Store("default2", _provider.New(2));
_provider.CreateIndex("default3");
_provider.Store("default3", _provider.New(3));
Assert.IsNotNull(_provider.CreateSearchBuilder("default1").Get(1));
Assert.IsNotNull(_provider.CreateSearchBuilder("default2").Get(2));
Assert.IsNotNull(_provider.CreateSearchBuilder("default3").Get(3));
Assert.IsNull(_provider.CreateSearchBuilder("default1").Get(2));
Assert.IsNull(_provider.CreateSearchBuilder("default2").Get(3));
Assert.IsNull(_provider.CreateSearchBuilder("default3").Get(1));
}
[Test]
public void IdentifierShouldNotCollide() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("field", "value1"));
_provider.Store("default", _provider.New(11).Add("field", "value11"));
_provider.Store("default", _provider.New(111).Add("field", "value111"));
var searchBuilder = _provider.CreateSearchBuilder("default");
Assert.That(searchBuilder.Get(1).ContentItemId, Is.EqualTo(1));
Assert.That(searchBuilder.Get(11).ContentItemId, Is.EqualTo(11));
Assert.That(searchBuilder.Get(111).ContentItemId, Is.EqualTo(111));
}
[Test]
public void TagsShouldBeRemoved() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("body", "<hr>some content</hr>").Analyze());
_provider.Store("default", _provider.New(2).Add("body", "<hr>some content</hr>", true).Analyze());
var searchBuilder = _provider.CreateSearchBuilder("default");
Assert.That(searchBuilder.WithField("body", "hr").Search().Count(), Is.EqualTo(1));
Assert.That(searchBuilder.WithField("body", "hr").Search().First().ContentItemId, Is.EqualTo(1));
}
[Test] public void ShouldAllowNullOrEmptyStrings() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("body", null));
_provider.Store("default", _provider.New(2).Add("body", ""));
_provider.Store("default", _provider.New(3).Add("body", "<hr></hr>", true));
var searchBuilder = _provider.CreateSearchBuilder("default");
Assert.That(searchBuilder.Get(1).ContentItemId, Is.EqualTo(1));
Assert.That(searchBuilder.Get(2).ContentItemId, Is.EqualTo(2));
Assert.That(searchBuilder.Get(3).ContentItemId, Is.EqualTo(3));
}
[Test]
public void ProviderShouldStoreSettings() {
_provider.CreateIndex("default");
Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(LuceneIndexProvider.DefaultMinDateTime));
_provider.SetLastIndexUtc("default", new DateTime(2010, 1, 1, 1, 1, 1, 1));
Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(new DateTime(2010, 1, 1, 1, 1, 1, 0)));
_provider.SetLastIndexUtc("default", new DateTime(1901, 1, 1, 1, 1, 1, 1));
Assert.That(_provider.GetLastIndexUtc("default"), Is.EqualTo(LuceneIndexProvider.DefaultMinDateTime));
}
[Test]
public void IsEmptyShouldBeTrueForNoneExistingIndexes() {
_provider.IsEmpty("dummy");
Assert.That(_provider.IsEmpty("default"), Is.True);
}
[Test]
public void IsEmptyShouldBeTrueForJustNewIndexes() {
_provider.CreateIndex("default");
Assert.That(_provider.IsEmpty("default"), Is.True);
}
[Test]
public void IsEmptyShouldBeFalseWhenThereIsADocument() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("body", null));
Assert.That(_provider.IsEmpty("default"), Is.False);
}
[Test]
public void IsDirtyShouldBeFalseForNewDocuments() {
IDocumentIndex doc = _provider.New(1);
Assert.That(doc.IsDirty, Is.False);
}
[Test]
public void IsDirtyShouldBeTrueWhenIndexIsModified() {
IDocumentIndex doc;
doc = _provider.New(1);
doc.Add("foo", "value");
Assert.That(doc.IsDirty, Is.True);
doc = _provider.New(1);
doc.Add("foo", false);
Assert.That(doc.IsDirty, Is.True);
doc = _provider.New(1);
doc.Add("foo", (float)1.0);
Assert.That(doc.IsDirty, Is.True);
doc = _provider.New(1);
doc.Add("foo", 1);
Assert.That(doc.IsDirty, Is.True);
doc = _provider.New(1);
doc.Add("foo", DateTime.Now);
Assert.That(doc.IsDirty, Is.True);
}
}
}

View File

@@ -0,0 +1,258 @@
using System;
using System.IO;
using System.Linq;
using Autofac;
using NUnit.Framework;
using Orchard.Environment.Configuration;
using Orchard.FileSystems.AppData;
using Orchard.Indexing;
using Orchard.Indexing.Services;
using Orchard.Tests.FileSystems.AppData;
namespace Orchard.Tests.Modules.Indexing {
public class LuceneSearchBuilderTests {
private IContainer _container;
private IIndexProvider _provider;
private IAppDataFolder _appDataFolder;
private ShellSettings _shellSettings;
private readonly string _basePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
[TestFixtureTearDown]
public void Clean() {
Directory.Delete(_basePath, true);
}
[SetUp]
public void Setup() {
if (Directory.Exists(_basePath)) {
Directory.Delete(_basePath, true);
}
Directory.CreateDirectory(_basePath);
_appDataFolder = AppDataFolderTests.CreateAppDataFolder(_basePath);
var builder = new ContainerBuilder();
builder.RegisterType<LuceneIndexProvider>().As<IIndexProvider>();
builder.RegisterInstance(_appDataFolder).As<IAppDataFolder>();
// setting up a ShellSettings instance
_shellSettings = new ShellSettings { Name = "My Site" };
builder.RegisterInstance(_shellSettings).As<ShellSettings>();
_container = builder.Build();
_provider = _container.Resolve<IIndexProvider>();
}
private ISearchBuilder _searchBuilder { get { return _provider.CreateSearchBuilder("default"); } }
[Test]
public void SearchTermsShouldBeFoundInMultipleFields() {
_provider.CreateIndex("default");
_provider.Store("default",
_provider.New(42)
.Add("title", "title1 title2 title3").Analyze()
.Add("date", new DateTime(2010, 05, 28, 14, 13, 56, 123))
);
Assert.IsNotNull(_provider.CreateSearchBuilder("default").Get(42));
Assert.IsNotNull(_provider.CreateSearchBuilder("default").WithField("title", "title1").Search().FirstOrDefault());
Assert.IsNotNull(_provider.CreateSearchBuilder("default").WithField("title", "title2").Search().FirstOrDefault());
Assert.IsNotNull(_provider.CreateSearchBuilder("default").WithField("title", "title3").Search().FirstOrDefault());
Assert.IsNull(_provider.CreateSearchBuilder("default").WithField("title", "title4").Search().FirstOrDefault());
}
[Test]
public void ShouldSearchById() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1));
_provider.Store("default", _provider.New(2));
_provider.Store("default", _provider.New(3));
Assert.That(_searchBuilder.Get(1).ContentItemId, Is.EqualTo(1));
Assert.That(_searchBuilder.Get(2).ContentItemId, Is.EqualTo(2));
Assert.That(_searchBuilder.Get(3).ContentItemId, Is.EqualTo(3));
}
[Test]
public void ShouldSearchWithField() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("title", "cat"));
_provider.Store("default", _provider.New(2).Add("title", "dog"));
_provider.Store("default", _provider.New(3).Add("title", "cat"));
Assert.That(_searchBuilder.WithField("title", "cat").Search().Count(), Is.EqualTo(2));
Assert.That(_searchBuilder.WithField("title", "cat").Search().Any(hit => new[] { 1, 3 }.Contains(hit.ContentItemId)), Is.True);
}
[Test]
public void ShouldCountResultsOnly() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("title", "cat"));
_provider.Store("default", _provider.New(2).Add("title", "dog"));
_provider.Store("default", _provider.New(3).Add("title", "cat"));
Assert.That(_searchBuilder.WithField("title", "dog").Count(), Is.EqualTo(1));
Assert.That(_searchBuilder.WithField("title", "cat").Count(), Is.EqualTo(2));
}
[Test]
public void ShouldFilterByDate() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("date", new DateTime(2010, 05, 28, 12, 30, 15)));
_provider.Store("default", _provider.New(2).Add("date", new DateTime(2010, 05, 28, 12, 30, 30)));
_provider.Store("default", _provider.New(3).Add("date", new DateTime(2010, 05, 28, 12, 30, 45)));
Assert.That(_searchBuilder.WithinRange("date", new DateTime(2010, 05, 28, 12, 30, 15), DateTime.MaxValue).Count(), Is.EqualTo(3));
Assert.That(_searchBuilder.WithinRange("date", DateTime.MinValue, new DateTime(2010, 05, 28, 12, 30, 45)).Count(), Is.EqualTo(3));
Assert.That(_searchBuilder.WithinRange("date", new DateTime(2010, 05, 28, 12, 30, 15), new DateTime(2010, 05, 28, 12, 30, 45)).Count(), Is.EqualTo(3));
Assert.That(_searchBuilder.WithinRange("date", new DateTime(2010, 05, 28, 12, 30, 16), new DateTime(2010, 05, 28, 12, 30, 44)).Count(), Is.EqualTo(1));
Assert.That(_searchBuilder.WithinRange("date", new DateTime(2010, 05, 28, 12, 30, 46), DateTime.MaxValue).Count(), Is.EqualTo(0));
Assert.That(_searchBuilder.WithinRange("date", DateTime.MinValue, new DateTime(2010, 05, 28, 12, 30, 1)).Count(), Is.EqualTo(0));
}
[Test]
public void ShouldSliceResults() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1));
_provider.Store("default", _provider.New(22));
_provider.Store("default", _provider.New(333));
_provider.Store("default", _provider.New(4444));
_provider.Store("default", _provider.New(55555));
Assert.That(_searchBuilder.Count(), Is.EqualTo(5));
Assert.That(_searchBuilder.Slice(0, 3).Count(), Is.EqualTo(3));
Assert.That(_searchBuilder.Slice(1, 3).Count(), Is.EqualTo(3));
Assert.That(_searchBuilder.Slice(3, 3).Count(), Is.EqualTo(2));
// Count() and Search() should return the same results
Assert.That(_searchBuilder.Search().Count(), Is.EqualTo(5));
Assert.That(_searchBuilder.Slice(0, 3).Search().Count(), Is.EqualTo(3));
Assert.That(_searchBuilder.Slice(1, 3).Search().Count(), Is.EqualTo(3));
Assert.That(_searchBuilder.Slice(3, 3).Search().Count(), Is.EqualTo(2));
}
[Test]
public void ShouldSortByRelevance() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("body", "michael is in the kitchen").Analyze());
_provider.Store("default", _provider.New(2).Add("body", "michael as a cousin named michel").Analyze());
_provider.Store("default", _provider.New(3).Add("body", "speak inside the mic").Analyze());
_provider.Store("default", _provider.New(4).Add("body", "a dog is pursuing a cat").Analyze());
_provider.Store("default", _provider.New(5).Add("body", "the elephant can't catch up the dog").Analyze());
var michael = _searchBuilder.WithField("body", "michael").Search().ToList();
Assert.That(michael.Count(), Is.EqualTo(2));
Assert.That(michael[0].Score >= michael[1].Score, Is.True);
// Sorting on score is always descending
michael = _searchBuilder.WithField("body", "michael").Ascending().Search().ToList();
Assert.That(michael.Count(), Is.EqualTo(2));
Assert.That(michael[0].Score >= michael[1].Score, Is.True);
}
[Test]
public void ShouldSortByDate() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("date", new DateTime(2010, 05, 28, 12, 30, 15)).Store());
_provider.Store("default", _provider.New(2).Add("date", new DateTime(2010, 05, 28, 12, 30, 30)).Store());
_provider.Store("default", _provider.New(3).Add("date", new DateTime(2010, 05, 28, 12, 30, 45)).Store());
var date = _searchBuilder.SortBy("date").Search().ToList();
Assert.That(date.Count(), Is.EqualTo(3));
Assert.That(date[0].GetDateTime("date") > date[1].GetDateTime("date"), Is.True);
Assert.That(date[1].GetDateTime("date") > date[2].GetDateTime("date"), Is.True);
date = _searchBuilder.SortBy("date").Ascending().Search().ToList();
Assert.That(date.Count(), Is.EqualTo(3));
Assert.That(date[0].GetDateTime("date") < date[1].GetDateTime("date"), Is.True);
Assert.That(date[1].GetDateTime("date") < date[2].GetDateTime("date"), Is.True);
}
[Test]
public void ShouldEscapeSpecialChars() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("body", "Orchard has been developped in C#").Analyze());
_provider.Store("default", _provider.New(2).Add("body", "Windows has been developped in C++").Analyze());
var cs = _searchBuilder.Parse("body", "C#").Search().ToList();
Assert.That(cs.Count(), Is.EqualTo(2));
var cpp = _searchBuilder.Parse("body", "C++").Search().ToList();
Assert.That(cpp.Count(), Is.EqualTo(2));
}
[Test]
public void ShouldHandleMandatoryFields() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("body", "Orchard has been developped in C#").Analyze());
_provider.Store("default", _provider.New(2).Add("body", "Windows has been developped in C++").Analyze());
Assert.That(_searchBuilder.WithField("body", "develop").Search().ToList().Count(), Is.EqualTo(2));
Assert.That(_searchBuilder.WithField("body", "develop").WithField("body", "Orchard").Search().ToList().Count(), Is.EqualTo(2));
Assert.That(_searchBuilder.WithField("body", "develop").WithField("body", "Orchard").Mandatory().Search().ToList().Count(), Is.EqualTo(1));
Assert.That(_searchBuilder.WithField("body", "develop").WithField("body", "Orchard").Mandatory().Search().First().ContentItemId, Is.EqualTo(1));
}
[Test]
public void ShouldHandleForbiddenFields() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("body", "Orchard has been developped in C#").Analyze());
_provider.Store("default", _provider.New(2).Add("body", "Windows has been developped in C++").Analyze());
Assert.That(_searchBuilder.WithField("body", "developped").Search().ToList().Count(), Is.EqualTo(2));
Assert.That(_searchBuilder.WithField("body", "developped").WithField("body", "Orchard").Search().ToList().Count(), Is.EqualTo(2));
Assert.That(_searchBuilder.WithField("body", "developped").WithField("body", "Orchard").Forbidden().Search().ToList().Count(), Is.EqualTo(1));
Assert.That(_searchBuilder.WithField("body", "developped").WithField("body", "Orchard").Forbidden().Search().First().ContentItemId, Is.EqualTo(2));
}
[Test]
public void ShouldHandleWeight() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("body", "Orchard has been developped in C#").Analyze());
_provider.Store("default", _provider.New(2).Add("body", "Windows has been developped in C++").Analyze());
Assert.That(_searchBuilder.WithField("body", "developped").WithField("body", "Orchard").Weighted(2).Search().First().ContentItemId, Is.EqualTo(1));
}
[Test]
public void ShouldParseLuceneQueries() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("body", "Bradley is in the kitchen.").Analyze().Add("title", "Beer and takos").Analyze());
_provider.Store("default", _provider.New(2).Add("body", "Renaud is also in the kitchen.").Analyze().Add("title", "A love affair").Analyze());
_provider.Store("default", _provider.New(3).Add("body", "Bertrand is a little bit jealous.").Analyze().Add("title", "Soap opera").Analyze());
Assert.That(_searchBuilder.Parse(new[] { "body" }, "kitchen").Count(), Is.EqualTo(2));
Assert.That(_searchBuilder.Parse(new[] { "body" }, "kitchen bertrand").Count(), Is.EqualTo(3));
Assert.That(_searchBuilder.Parse(new[] { "body" }, "kitchen +bertrand").Count(), Is.EqualTo(1));
Assert.That(_searchBuilder.Parse(new[] { "body" }, "+kitchen +bertrand").Count(), Is.EqualTo(0));
Assert.That(_searchBuilder.Parse(new[] { "body" }, "kit").Count(), Is.EqualTo(0));
Assert.That(_searchBuilder.Parse(new[] { "body" }, "kit*").Count(), Is.EqualTo(2));
Assert.That(_searchBuilder.Parse(new[] { "body", "title" }, "bradley love^3 soap").Count(), Is.EqualTo(3));
Assert.That(_searchBuilder.Parse(new[] { "body", "title" }, "bradley love^3 soap").Search().First().ContentItemId, Is.EqualTo(2));
}
[Test]
public void ShouldFilterIntValues() {
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("field", 1));
_provider.Store("default", _provider.New(2).Add("field", 22));
_provider.Store("default", _provider.New(3).Add("field", 333));
Assert.That(_searchBuilder.WithField("field", 1).ExactMatch().Count(), Is.EqualTo(1));
Assert.That(_searchBuilder.WithField("field", 22).ExactMatch().Count(), Is.EqualTo(1));
Assert.That(_searchBuilder.WithField("field", 333).ExactMatch().Count(), Is.EqualTo(1));
Assert.That(_searchBuilder.WithField("field", 0).ExactMatch().Count(), Is.EqualTo(0));
Assert.That(_searchBuilder.WithField("field", 2).ExactMatch().Count(), Is.EqualTo(0));
Assert.That(_searchBuilder.WithField("field", 3).ExactMatch().Count(), Is.EqualTo(0));
}
}
}

View File

@@ -108,6 +108,8 @@
</ItemGroup>
<ItemGroup>
<Compile Include="DatabaseEnabledTestsBase.cs" />
<Compile Include="Indexing\LuceneIndexProviderTests.cs" />
<Compile Include="Indexing\LuceneSearchBuilderTests.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Roles\Controllers\AdminControllerTests.cs" />
<Compile Include="Roles\Services\RoleServiceTests.cs" />
@@ -128,6 +130,10 @@
<Project>{9916839C-39FC-4CEB-A5AF-89CA7E87119F}</Project>
<Name>Orchard.Core</Name>
</ProjectReference>
<ProjectReference Include="..\Orchard.Web\Modules\Orchard.Indexing\Orchard.Indexing.csproj">
<Project>{EA2B9121-EF54-40A6-A53E-6593C86EE696}</Project>
<Name>Orchard.Indexing</Name>
</ProjectReference>
<ProjectReference Include="..\Orchard.Web\Modules\Orchard.Pages\Orchard.Pages.csproj">
<Project>{4A9C04A6-0986-4A92-A610-5F59FF273FB9}</Project>
<Name>Orchard.Pages</Name>