#5664: Lucene should page operations

Fixes #5664

src/Orchard.Tests.Modules/Indexing/LuceneIndexProviderTests.cs

@@ -303,5 +303,24 @@ namespace Orchard.Tests.Modules.Indexing {
             _provider.Store("default", _provider.New(1).Add("field", "value2"));
             Assert.That(searchBuilder.WithField("id", "1").Count(), Is.EqualTo(1));
         }
+
+        [Test]
+        public void IndexProviderShouldDeleteMoreThanMaxTermsCount() {
+            _provider.CreateIndex("default");
+
+            var documents = Enumerable.Range(1, 1025).Select(i => _provider.New(i).Add("field", "value1"));
+            _provider.Store("default", documents);
+            
+            var searchBuilder = _provider.CreateSearchBuilder("default");
+
+            Assert.That(searchBuilder.Count(), Is.EqualTo(1025));
+            Assert.That(searchBuilder.Get(1).ContentItemId, Is.EqualTo(1));
+            Assert.That(searchBuilder.Get(1025).ContentItemId, Is.EqualTo(1025));
+
+            _provider.Delete("default", Enumerable.Range(1, 1025));
+
+
+            Assert.That(searchBuilder.Count(), Is.EqualTo(0));
+        }
     }
 }

src/Orchard.Web/Modules/Lucene/Services/LuceneIndexProvider.cs

@@ -29,6 +29,7 @@ namespace Lucene.Services {
 
         public static readonly Version LuceneVersion = Version.LUCENE_29;
         public static readonly DateTime DefaultMinDateTime = new DateTime(1980, 1, 1);
+        public static readonly int BatchSize = 50;
 
         public LuceneIndexProvider(
             IAppDataFolder appDataFolder, 
@@ -152,10 +153,17 @@ namespace Lucene.Services {
             }
 
             using (var writer = new IndexWriter(GetDirectory(indexName), _analyzerProvider.GetAnalyzer(indexName), false, IndexWriter.MaxFieldLength.UNLIMITED)) {
+                // Process documents by batch as there is a max number of terms a query can contain (1024 by default).
+                var pageCount = documentIds.Count() / BatchSize + 1;
+                for (int page = 0; page < pageCount; page++) {
                     var query = new BooleanQuery();
 
                     try {
-                    foreach (var id in documentIds) {
+                        var batch = documentIds
+                            .Skip(page * BatchSize)
+                            .Take(BatchSize);
+
+                        foreach (var id in batch) {
                             query.Add(new BooleanClause(new TermQuery(new Term("id", id.ToString(CultureInfo.InvariantCulture))), Occur.SHOULD));
                         }
 
@@ -166,6 +174,7 @@ namespace Lucene.Services {
                     }
                 }
             }
+        }
 
         public IDocumentIndex New(int documentId) {
             return new LuceneDocumentIndex(documentId, T);