From b3924ba1d8d421aa7ad16e61746ab25032aaa663 Mon Sep 17 00:00:00 2001 From: damoclarke Date: Fri, 29 Mar 2013 18:50:12 +1100 Subject: [PATCH] Added ContentIdentity resolvers, batch processing and improved performance for large imports. --HG-- branch : 1.x extra : source : 3a9c242225bfe36132da7f8760711574eb5a9b43 --- .../Services/ImportExportServiceTests.cs | 2 + .../Recipes/Services/RecipeManagerTests.cs | 2 + .../ContentManagement/ContentIdentityTests.cs | 65 +++++- .../ImportContentSessionTests.cs | 189 ++++++++++++++++ .../Orchard.Framework.Tests.csproj | 1 + .../Common/Handlers/IdentityPartHandler.cs | 27 ++- .../Controllers/AdminController.cs | 18 +- .../Models/ExportOptions.cs | 1 + .../Orchard.ImportExport.csproj | 7 + .../Services/IImportExportService.cs | 4 +- .../Services/ImportExportService.cs | 40 +++- .../Views/Admin/ImportResult.cshtml | 30 +++ .../Orchard.Recipes/Orchard.Recipes.csproj | 4 + .../RecipeHandlers/DataRecipeHandler.cs | 80 +++++-- .../Orchard.Recipes/Services/RecipeManager.cs | 7 +- .../Orchard.Recipes/Services/RecipeParser.cs | 4 + .../Services/RecipeStepExecutor.cs | 23 +- .../ContentManagement/ContentIdentity.cs | 37 +++- .../DefaultContentManager.cs | 16 +- .../Handlers/ContentHandler.cs | 5 + .../Handlers/ContentHandlerBase.cs | 1 + .../Handlers/IContentHandler.cs | 1 + .../RegisterIdentityResolverContext.cs | 28 +++ .../ContentManagement/IContentManager.cs | 2 + .../ContentManagement/ImportContentSession.cs | 203 +++++++++++++----- src/Orchard/Orchard.Framework.csproj | 2 + .../Events/IRecipeExecuteEventHandler.cs | 12 ++ src/Orchard/Recipes/Models/Recipe.cs | 4 +- 28 files changed, 715 insertions(+), 100 deletions(-) create mode 100644 src/Orchard.Tests/ContentManagement/ImportContentSessionTests.cs create mode 100644 src/Orchard.Web/Modules/Orchard.ImportExport/Views/Admin/ImportResult.cshtml create mode 100644 src/Orchard/ContentManagement/Handlers/RegisterIdentityResolverContext.cs create mode 100644 src/Orchard/Recipes/Events/IRecipeExecuteEventHandler.cs diff --git a/src/Orchard.Tests.Modules/ImportExport/Services/ImportExportServiceTests.cs b/src/Orchard.Tests.Modules/ImportExport/Services/ImportExportServiceTests.cs index 208e7d592..bd41d2249 100644 --- a/src/Orchard.Tests.Modules/ImportExport/Services/ImportExportServiceTests.cs +++ b/src/Orchard.Tests.Modules/ImportExport/Services/ImportExportServiceTests.cs @@ -20,6 +20,7 @@ using Orchard.Environment.Extensions.Loaders; using Orchard.FileSystems.AppData; using Orchard.FileSystems.WebSite; using Orchard.ImportExport.Services; +using Orchard.Recipes.Events; using Orchard.Recipes.Services; using Orchard.Services; using Orchard.Tests.ContentManagement; @@ -73,6 +74,7 @@ namespace Orchard.Tests.Modules.ImportExport.Services { builder.RegisterType().As(); builder.RegisterGeneric(typeof(Repository<>)).As(typeof(IRepository<>)); builder.RegisterInstance(new Mock().Object); + builder.RegisterInstance(new Mock().Object); _session = _sessionFactory.OpenSession(); builder.RegisterInstance(new DefaultContentManagerTests.TestSessionLocator(_session)).As(); diff --git a/src/Orchard.Tests.Modules/Recipes/Services/RecipeManagerTests.cs b/src/Orchard.Tests.Modules/Recipes/Services/RecipeManagerTests.cs index 300ef38ca..d9167ebc9 100644 --- a/src/Orchard.Tests.Modules/Recipes/Services/RecipeManagerTests.cs +++ b/src/Orchard.Tests.Modules/Recipes/Services/RecipeManagerTests.cs @@ -15,6 +15,7 @@ using Orchard.Recipes.Models; using Orchard.Recipes.Services; using Orchard.Services; using Orchard.Tests.Stubs; +using Orchard.Recipes.Events; namespace Orchard.Tests.Modules.Recipes.Services { [TestFixture] @@ -78,6 +79,7 @@ namespace Orchard.Tests.Modules.Recipes.Services { builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterInstance(_folders).As(); + builder.RegisterInstance(new Mock().Object); builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); diff --git a/src/Orchard.Tests/ContentManagement/ContentIdentityTests.cs b/src/Orchard.Tests/ContentManagement/ContentIdentityTests.cs index 4f06511d2..abefdfa23 100644 --- a/src/Orchard.Tests/ContentManagement/ContentIdentityTests.cs +++ b/src/Orchard.Tests/ContentManagement/ContentIdentityTests.cs @@ -26,7 +26,70 @@ namespace Orchard.Tests.ContentManagement { Assert.That(identity2.Get("foo"), Is.EqualTo("bar")); Assert.That(identity2.Get("abaz"), Is.EqualTo(@"quux/fr\ed=foo")); Assert.That(identity2.Get("yarg"), Is.EqualTo("yiu=foo")); - Assert.That(identity2.ToString(), Is.EqualTo(@"/foo=bar/abaz=quux\/fr\\ed=foo/yarg=yiu=foo")); + Assert.That(identity2.ToString(), Is.EqualTo(@"/abaz=quux\/fr\\ed=foo/foo=bar/yarg=yiu=foo")); + } + + [Test] + public void ContentIdentitiesWithKeysAddedInDifferentOrderAreEqual() { + var comparer = new ContentIdentity.ContentIdentityEqualityComparer(); + + var identity1 = new ContentIdentity("/foo=bar"); + Assert.That(comparer.Equals(identity1, new ContentIdentity(identity1.ToString()))); + + var identity2 = new ContentIdentity(@"/foo=bar/abaz=quux\/fr\\ed=foo/yarg=yiu=foo"); + Assert.That(comparer.Equals(identity2, new ContentIdentity(identity2.ToString()))); + } + + [Test] + public void IdentitiesCanBeAddedWithNoPriority() { + var contentIdentity = new ContentIdentity(); + + contentIdentity.Add("identifier", "CAEEB150-F5E9-481D-9FF9-3053D23329C1"); + contentIdentity.Add("alias", "some-unique-item-alias/sub-alias"); + + Assert.That("/alias=some-unique-item-alias\\/sub-alias/identifier=CAEEB150-F5E9-481D-9FF9-3053D23329C1", Is.EqualTo(contentIdentity.ToString())); + } + + [Test] + public void IdentitiesCanBeAddedWithSamePriority() { + var contentIdentity = new ContentIdentity(); + contentIdentity.Add("identifier", "CAEEB150-F5E9-481D-9FF9-3053D23329C1", 5); + contentIdentity.Add("alias", "some-unique-item-alias/sub-alias", 5); + + var contentIdentityNegative = new ContentIdentity(); + contentIdentityNegative.Add("identifier", "CAEEB150-F5E9-481D-9FF9-3053D23329C1", -5); + contentIdentityNegative.Add("alias", "some-unique-item-alias/sub-alias", -5); + + Assert.That("/alias=some-unique-item-alias\\/sub-alias/identifier=CAEEB150-F5E9-481D-9FF9-3053D23329C1", Is.EqualTo(contentIdentity.ToString())); + Assert.That("/alias=some-unique-item-alias\\/sub-alias/identifier=CAEEB150-F5E9-481D-9FF9-3053D23329C1", Is.EqualTo(contentIdentityNegative.ToString())); + } + + [Test] + public void LowestPriorityIdentityIsIgnored() { + var contentIdentity = new ContentIdentity(); + contentIdentity.Add("alias", "some-unique-item-alias/sub-alias", 0); + contentIdentity.Add("identifier", "CAEEB150-F5E9-481D-9FF9-3053D23329C1", 5); + + var contentIdentityNegative = new ContentIdentity(); + contentIdentityNegative.Add("alias", "some-unique-item-alias/sub-alias", -5); + contentIdentityNegative.Add("identifier", "CAEEB150-F5E9-481D-9FF9-3053D23329C1", 0); + + Assert.That("/identifier=CAEEB150-F5E9-481D-9FF9-3053D23329C1", Is.EqualTo(contentIdentity.ToString())); + Assert.That("/identifier=CAEEB150-F5E9-481D-9FF9-3053D23329C1", Is.EqualTo(contentIdentityNegative.ToString())); + } + + [Test] + public void HighestPriorityIdentityIsRetained() { + var contentIdentity = new ContentIdentity(); + contentIdentity.Add("identifier", "CAEEB150-F5E9-481D-9FF9-3053D23329C1", 5); + contentIdentity.Add("alias", "some-unique-item-alias/sub-alias", 0); + + var contentIdentityNegative = new ContentIdentity(); + contentIdentityNegative.Add("identifier", "CAEEB150-F5E9-481D-9FF9-3053D23329C1", 0); + contentIdentityNegative.Add("alias", "some-unique-item-alias/sub-alias", -5); + + Assert.That("/identifier=CAEEB150-F5E9-481D-9FF9-3053D23329C1", Is.EqualTo(contentIdentity.ToString())); + Assert.That("/identifier=CAEEB150-F5E9-481D-9FF9-3053D23329C1", Is.EqualTo(contentIdentityNegative.ToString())); } } } diff --git a/src/Orchard.Tests/ContentManagement/ImportContentSessionTests.cs b/src/Orchard.Tests/ContentManagement/ImportContentSessionTests.cs new file mode 100644 index 000000000..b50b05787 --- /dev/null +++ b/src/Orchard.Tests/ContentManagement/ImportContentSessionTests.cs @@ -0,0 +1,189 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Moq; +using NUnit.Framework; +using Orchard.ContentManagement; +using Orchard.ContentManagement.Records; + +namespace Orchard.Tests.ContentManagement { + [TestFixture] + public class ImportContentSessionTests { + private ContentIdentity _testItemIdentity1; + private ContentIdentity _testItemIdentity2; + private ContentIdentity _testItemIdentity3; + private ContentIdentity _testItemIdentity4; + private ContentIdentity _testItemIdentity5; + private Mock _contentManager; + + #region Init + [TestFixtureSetUp] + public void TestInit() { + _testItemIdentity1 = new ContentIdentity("/ItemId=1"); + _testItemIdentity2 = new ContentIdentity("/ItemId=2"); + _testItemIdentity3 = new ContentIdentity("/ItemId=3"); + _testItemIdentity4 = new ContentIdentity("/ItemId=4"); + _testItemIdentity5 = new ContentIdentity("/ItemId=5"); + var draftItem = new ContentItem { VersionRecord = new ContentItemVersionRecord { Id = 1234, Published = false, Latest = true, ContentItemRecord = new ContentItemRecord { Id = 1 } } }; + var publishedItem = new ContentItem { VersionRecord = new ContentItemVersionRecord { Id = 1234, Published = true, Latest = true, ContentItemRecord = new ContentItemRecord { Id = 1 } } }; + + var draftItem5 = new ContentItem { VersionRecord = new ContentItemVersionRecord { Id = 1234, Published = false, Latest = true, ContentItemRecord = new ContentItemRecord { Id = 5 } } }; + var publishedItem5 = new ContentItem { VersionRecord = new ContentItemVersionRecord { Id = 1234, Published = true, Latest = true, ContentItemRecord = new ContentItemRecord { Id = 5 } } }; + + _contentManager = new Mock(); + _contentManager.Setup(m => m.Get(It.Is(v => v == 1), It.Is(v => v.IsDraftRequired))).Returns(draftItem); + _contentManager.Setup(m => m.Get(It.Is(v => v == 1), It.Is(v => !v.IsDraftRequired))).Returns(publishedItem); + + _contentManager.Setup(m => m.Get(It.Is(v => v == 5), It.Is(v => v.IsDraftRequired))).Returns(draftItem5); + _contentManager.Setup(m => m.Get(It.Is(v => v == 5), It.Is(v => !v.IsDraftRequired))).Returns(publishedItem5); + + _contentManager.Setup(m => m.GetItemMetadata(It.Is(c => c.Id == 1))).Returns(new ContentItemMetadata { Identity = _testItemIdentity1 }); + _contentManager.Setup(m => m.GetItemMetadata(It.Is(c => c.Id == 5))).Returns(new ContentItemMetadata { Identity = _testItemIdentity5 }); + + _contentManager.Setup(m => m.New(It.IsAny())).Returns(draftItem5); + + _contentManager.Setup(m => m.HasResolverForIdentity(It.Is(id => id.Get("ItemId") != null))).Returns(true); + _contentManager.Setup(m => m.ResolveIdentity(It.Is(id => id.Get("ItemId") == "1"))).Returns(publishedItem); + } + #endregion + + [Test] + public void GetNextInBatchReturnsNullWhenNoItemsSet() { + var importContentSession = new ImportContentSession(_contentManager.Object); + + Assert.That(importContentSession.GetNextInBatch(), Is.Null); + } + + [Test] + public void GetNextInBatchReturnsNullWhenInitializedButNoItemsSet() { + var importContentSession = new ImportContentSession(_contentManager.Object); + importContentSession.InitializeBatch(0, 20); + Assert.That(importContentSession.GetNextInBatch(), Is.Null); + } + + [Test] + public void ItemsSetAndUninitializedReturnsAllItems() { + var importContentSession = new ImportContentSession(_contentManager.Object); + + importContentSession.Set("/Id=One", "TestType"); + importContentSession.Set("/Id=Two", "TestType"); + importContentSession.Set("/Id=Three", "TestType"); + + var comparer = new ContentIdentity.ContentIdentityEqualityComparer(); + Assert.That(comparer.Equals(importContentSession.GetNextInBatch(), new ContentIdentity("/Id=One"))); + Assert.That(comparer.Equals(importContentSession.GetNextInBatch(), new ContentIdentity("/Id=Two"))); + Assert.That(comparer.Equals(importContentSession.GetNextInBatch(), new ContentIdentity("/Id=Three"))); + Assert.That(importContentSession.GetNextInBatch(), Is.Null); + } + + [Test] + public void ItemsSetAndBatchInitialisedReturnsBatchedItems() { + var importContentSession = new ImportContentSession(_contentManager.Object); + + importContentSession.Set("/Id=One", "TestType"); + importContentSession.Set("/Id=Two", "TestType"); + importContentSession.Set("/Id=Three", "TestType"); + importContentSession.Set("/Id=Four", "TestType"); + importContentSession.Set("/Id=Five", "TestType"); + + importContentSession.InitializeBatch(1, 2); + + + var comparer = new ContentIdentity.ContentIdentityEqualityComparer(); + Assert.That(comparer.Equals(importContentSession.GetNextInBatch(), new ContentIdentity("/Id=Two"))); + Assert.That(comparer.Equals(importContentSession.GetNextInBatch(), new ContentIdentity("/Id=Three"))); + Assert.That(importContentSession.GetNextInBatch(), Is.Null); + + importContentSession.InitializeBatch(2, 5); + + //item with "/Id=Three" should not be returned twice in the same session + Assert.That(comparer.Equals(importContentSession.GetNextInBatch(), new ContentIdentity("/Id=Four"))); + Assert.That(comparer.Equals(importContentSession.GetNextInBatch(), new ContentIdentity("/Id=Five"))); + Assert.That(importContentSession.GetNextInBatch(), Is.Null); + } + + [Test] + public void GetItemExistsAndNoVersionOptionsReturnsPublishedItem() { + var session = new ImportContentSession(_contentManager.Object); + session.Set(_testItemIdentity1.ToString(), "TestContentType"); + var sessionItem = session.Get(_testItemIdentity1.ToString()); + Assert.IsNotNull(sessionItem); + Assert.AreEqual(1, sessionItem.Id); + Assert.IsTrue(sessionItem.IsPublished()); + } + + [Test] + public void GetItemExistsAndLatestVersionOptionReturnsPublishedItem() { + var session = new ImportContentSession(_contentManager.Object); + session.Set(_testItemIdentity1.ToString(), "TestContentType"); + var sessionItem = session.Get(_testItemIdentity1.ToString()); + Assert.IsNotNull(sessionItem); + Assert.AreEqual(1, sessionItem.Id); + Assert.IsTrue(sessionItem.IsPublished()); + } + + [Test] + public void GetItemExistsAndDraftRequiredVersionOptionReturnsDraft() { + var session = new ImportContentSession(_contentManager.Object); + session.Set(_testItemIdentity1.ToString(), "TestContentType"); + var sessionItem = session.Get(_testItemIdentity1.ToString(), VersionOptions.DraftRequired); + Assert.IsNotNull(sessionItem); + Assert.That(1, Is.EqualTo(sessionItem.Id)); + Assert.IsFalse(sessionItem.IsPublished()); + } + + [Test] + public void GetNextInBatchInitialisedWithOneItemReturnsOneItemThenNull() { + var session = new ImportContentSession(_contentManager.Object); + session.Set(_testItemIdentity1.ToString(), "TestContentType"); + session.InitializeBatch(0, 1); + var firstIdentity = session.GetNextInBatch(); + var secondIdentity = session.GetNextInBatch(); + + var comparer = new ContentIdentity.ContentIdentityEqualityComparer(); + Assert.That(comparer.Equals(_testItemIdentity1, firstIdentity)); + Assert.That(secondIdentity, Is.Null); + } + + + [Test] + public void GetNextInBatchInitialisedTwoBatchesReturnsItemsOnceEach() { + var session = new ImportContentSession(_contentManager.Object); + session.Set(_testItemIdentity1.ToString(), "TestContentType"); + session.Set(_testItemIdentity2.ToString(), "TestContentType"); + session.Set(_testItemIdentity3.ToString(), "TestContentType"); + session.Set(_testItemIdentity4.ToString(), "TestContentType"); + session.Set(_testItemIdentity5.ToString(), "TestContentType"); + + session.InitializeBatch(0, 2); + var firstIdentity = session.GetNextInBatch(); + //get later item as dependency + var dependencyItem = session.Get(_testItemIdentity5.ToString(), VersionOptions.Latest); + var dependencyIdentity = session.GetNextInBatch(); + var secondIdentity = session.GetNextInBatch(); + var afterBatch1 = session.GetNextInBatch(); + + session.InitializeBatch(2, 2); + var thirdIdentity = session.GetNextInBatch(); + var fourthdentity = session.GetNextInBatch(); + var afterBatch2 = session.GetNextInBatch(); + + session.InitializeBatch(4, 2); + var fifthIdentity = session.GetNextInBatch(); + var afterBatch3 = session.GetNextInBatch(); + + var comparer = new ContentIdentity.ContentIdentityEqualityComparer(); + Assert.That(comparer.Equals(_testItemIdentity1, firstIdentity)); + Assert.That(comparer.Equals(_testItemIdentity5, dependencyIdentity)); + Assert.That(comparer.Equals(_testItemIdentity2, secondIdentity)); + Assert.That(afterBatch1, Is.Null); + + Assert.That(comparer.Equals(_testItemIdentity3, thirdIdentity)); + Assert.That(comparer.Equals(_testItemIdentity4, fourthdentity)); + Assert.That(afterBatch2, Is.Null); + + Assert.That(fifthIdentity, Is.Null); //already processed as dependency + Assert.That(afterBatch3, Is.Null); + } + } +} \ No newline at end of file diff --git a/src/Orchard.Tests/Orchard.Framework.Tests.csproj b/src/Orchard.Tests/Orchard.Framework.Tests.csproj index ca4784e09..d634cecbd 100644 --- a/src/Orchard.Tests/Orchard.Framework.Tests.csproj +++ b/src/Orchard.Tests/Orchard.Framework.Tests.csproj @@ -174,6 +174,7 @@ + diff --git a/src/Orchard.Web/Core/Common/Handlers/IdentityPartHandler.cs b/src/Orchard.Web/Core/Common/Handlers/IdentityPartHandler.cs index 5565678cb..e2ae0ce47 100644 --- a/src/Orchard.Web/Core/Common/Handlers/IdentityPartHandler.cs +++ b/src/Orchard.Web/Core/Common/Handlers/IdentityPartHandler.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using JetBrains.Annotations; using Orchard.ContentManagement; using Orchard.Core.Common.Models; @@ -8,9 +9,14 @@ using Orchard.ContentManagement.Handlers; namespace Orchard.Core.Common.Handlers { [UsedImplicitly] public class IdentityPartHandler : ContentHandler { - public IdentityPartHandler(IRepository identityRepository) { + private readonly IContentManager _contentManager; + + public IdentityPartHandler(IRepository identityRepository, + IContentManager contentManager) { Filters.Add(StorageFilter.For(identityRepository)); OnInitializing(AssignIdentity); + + _contentManager = contentManager; } protected void AssignIdentity(InitializingContentContext context, IdentityPart part) { @@ -24,5 +30,24 @@ namespace Orchard.Core.Common.Handlers { context.Metadata.Identity.Add("Identifier", part.Identifier); } } + + protected override void RegisterIdentityResolver(RegisterIdentityResolverContext context) { + context.Register(id => id.Get("Identifier") != null, ResolveIdentity); + } + + private ContentItem ResolveIdentity(ContentIdentity identity) { + var identifier = identity.Get("Identifier"); + + if (identifier == null) { + return null; + } + + var comparer = new ContentIdentity.ContentIdentityEqualityComparer(); + return _contentManager + .Query() + .Where(p => p.Identifier == identifier) + .List() + .FirstOrDefault(c => comparer.Equals(identity, _contentManager.GetItemMetadata(c).Identity)); + } } } \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.ImportExport/Controllers/AdminController.cs b/src/Orchard.Web/Modules/Orchard.ImportExport/Controllers/AdminController.cs index ad28a6194..c6fbc6bdc 100644 --- a/src/Orchard.Web/Modules/Orchard.ImportExport/Controllers/AdminController.cs +++ b/src/Orchard.Web/Modules/Orchard.ImportExport/Controllers/AdminController.cs @@ -7,6 +7,7 @@ using Orchard.ContentManagement.MetaData; using Orchard.ImportExport.Services; using Orchard.ImportExport.ViewModels; using Orchard.Localization; +using Orchard.Recipes.Services; using Orchard.UI.Notify; using Orchard.ImportExport.Models; @@ -15,16 +16,19 @@ namespace Orchard.ImportExport.Controllers { private readonly IImportExportService _importExportService; private readonly IContentDefinitionManager _contentDefinitionManager; private readonly ICustomExportStep _customExportStep; + private readonly IRecipeJournal _recipeJournal; public AdminController( IOrchardServices services, IImportExportService importExportService, IContentDefinitionManager contentDefinitionManager, - ICustomExportStep customExportStep + ICustomExportStep customExportStep, + IRecipeJournal recipeJournal ) { _importExportService = importExportService; _contentDefinitionManager = contentDefinitionManager; _customExportStep = customExportStep; + _recipeJournal = recipeJournal; Services = services; T = NullLocalizer.Instance; } @@ -49,11 +53,17 @@ namespace Orchard.ImportExport.Controllers { } if (ModelState.IsValid) { - _importExportService.Import(new StreamReader(Request.Files["RecipeFile"].InputStream).ReadToEnd()); + var executionId = _importExportService.Import(new StreamReader(Request.Files["RecipeFile"].InputStream).ReadToEnd()); Services.Notifier.Information(T("Your recipe has been imported.")); - } - return RedirectToAction("Import"); + return RedirectToAction("ImportResult", new { ExecutionId = executionId }); + } + return View(new ImportViewModel()); + } + + public ActionResult ImportResult(string executionId) { + var journal = _recipeJournal.GetRecipeJournal(executionId); + return View(journal); } public ActionResult Export() { diff --git a/src/Orchard.Web/Modules/Orchard.ImportExport/Models/ExportOptions.cs b/src/Orchard.Web/Modules/Orchard.ImportExport/Models/ExportOptions.cs index 8a1aa71fe..4de00335c 100644 --- a/src/Orchard.Web/Modules/Orchard.ImportExport/Models/ExportOptions.cs +++ b/src/Orchard.Web/Modules/Orchard.ImportExport/Models/ExportOptions.cs @@ -4,6 +4,7 @@ namespace Orchard.ImportExport.Models { public class ExportOptions { public bool ExportMetadata { get; set; } public bool ExportData { get; set; } + public int? ImportBatchSize { get; set; } public VersionHistoryOptions VersionHistoryOptions { get; set; } public bool ExportSiteSettings { get; set; } public IEnumerable CustomSteps { get; set; } diff --git a/src/Orchard.Web/Modules/Orchard.ImportExport/Orchard.ImportExport.csproj b/src/Orchard.Web/Modules/Orchard.ImportExport/Orchard.ImportExport.csproj index 81405675f..b5f68b600 100644 --- a/src/Orchard.Web/Modules/Orchard.ImportExport/Orchard.ImportExport.csproj +++ b/src/Orchard.Web/Modules/Orchard.ImportExport/Orchard.ImportExport.csproj @@ -20,6 +20,10 @@ 4.0 + + + + true @@ -101,6 +105,9 @@ + + + 10.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) diff --git a/src/Orchard.Web/Modules/Orchard.ImportExport/Services/IImportExportService.cs b/src/Orchard.Web/Modules/Orchard.ImportExport/Services/IImportExportService.cs index cac717a81..de19de1fa 100644 --- a/src/Orchard.Web/Modules/Orchard.ImportExport/Services/IImportExportService.cs +++ b/src/Orchard.Web/Modules/Orchard.ImportExport/Services/IImportExportService.cs @@ -1,9 +1,11 @@ using System.Collections.Generic; +using Orchard.ContentManagement; using Orchard.ImportExport.Models; namespace Orchard.ImportExport.Services { public interface IImportExportService : IDependency { - void Import(string recipeText); + string Import(string recipeText); + string Export(IEnumerable contentTypes, IEnumerable contentItems, ExportOptions exportOptions); string Export(IEnumerable contentTypes, ExportOptions exportOptions); } } diff --git a/src/Orchard.Web/Modules/Orchard.ImportExport/Services/ImportExportService.cs b/src/Orchard.Web/Modules/Orchard.ImportExport/Services/ImportExportService.cs index c90b2e466..9d1672bd9 100644 --- a/src/Orchard.Web/Modules/Orchard.ImportExport/Services/ImportExportService.cs +++ b/src/Orchard.Web/Modules/Orchard.ImportExport/Services/ImportExportService.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Xml; using System.Xml.Linq; using JetBrains.Annotations; using Orchard.ContentManagement; @@ -10,7 +11,9 @@ using Orchard.FileSystems.AppData; using Orchard.ImportExport.Models; using Orchard.Localization; using Orchard.Logging; +using Orchard.Recipes.Models; using Orchard.Recipes.Services; +using Orchard.Services; using VersionOptions = Orchard.ContentManagement.VersionOptions; namespace Orchard.ImportExport.Services { @@ -23,6 +26,7 @@ namespace Orchard.ImportExport.Services { private readonly IRecipeParser _recipeParser; private readonly IRecipeManager _recipeManager; private readonly IShellDescriptorManager _shellDescriptorManager; + private readonly IClock _clock; private readonly IEnumerable _exportEventHandlers; private const string ExportsDirectory = "Exports"; @@ -32,8 +36,9 @@ namespace Orchard.ImportExport.Services { IContentDefinitionWriter contentDefinitionWriter, IAppDataFolder appDataFolder, IRecipeParser recipeParser, - IRecipeManager recipeManager, + IRecipeManager recipeManager, IShellDescriptorManager shellDescriptorManager, + IClock clock, IEnumerable exportEventHandlers) { _orchardServices = orchardServices; _contentDefinitionManager = contentDefinitionManager; @@ -42,6 +47,7 @@ namespace Orchard.ImportExport.Services { _recipeParser = recipeParser; _recipeManager = recipeManager; _shellDescriptorManager = shellDescriptorManager; + _clock = clock; _exportEventHandlers = exportEventHandlers; Logger = NullLogger.Instance; T = NullLocalizer.Instance; @@ -50,13 +56,24 @@ namespace Orchard.ImportExport.Services { public Localizer T { get; set; } public ILogger Logger { get; set; } - public void Import(string recipeText) { + public string Import(string recipeText) { var recipe = _recipeParser.ParseRecipe(recipeText); - _recipeManager.Execute(recipe); + var executionId = _recipeManager.Execute(recipe); UpdateShell(); + return executionId; } public string Export(IEnumerable contentTypes, ExportOptions exportOptions) { + //items need to be retrieved + IEnumerable contentItems = null; + if (exportOptions.ExportData) { + contentItems = _orchardServices.ContentManager.Query(GetContentExportVersionOptions(exportOptions.VersionHistoryOptions), contentTypes.ToArray()).List(); + } + + return Export(contentTypes, contentItems, exportOptions); + } + + public string Export(IEnumerable contentTypes, IEnumerable contentItems, ExportOptions exportOptions) { var exportDocument = CreateExportRoot(); var context = new ExportContext { @@ -67,7 +84,7 @@ namespace Orchard.ImportExport.Services { _exportEventHandlers.Invoke(x => x.Exporting(context), Logger); - if (exportOptions.ExportMetadata) { + if (exportOptions.ExportMetadata && (!exportOptions.ExportData || contentItems.Any())) { exportDocument.Element("Orchard").Add(ExportMetadata(contentTypes)); } @@ -75,8 +92,8 @@ namespace Orchard.ImportExport.Services { exportDocument.Element("Orchard").Add(ExportSiteSettings()); } - if (exportOptions.ExportData) { - exportDocument.Element("Orchard").Add(ExportData(contentTypes, exportOptions.VersionHistoryOptions)); + if (exportOptions.ExportData && contentItems.Any()) { + exportDocument.Element("Orchard").Add(ExportData(contentTypes, contentItems, exportOptions.ImportBatchSize)); } _exportEventHandlers.Invoke(x => x.Exported(context), Logger); @@ -91,7 +108,8 @@ namespace Orchard.ImportExport.Services { new XElement("Orchard", new XElement("Recipe", new XElement("Name", "Generated by Orchard.ImportExport"), - new XElement("Author", _orchardServices.WorkContext.CurrentUser.UserName) + new XElement("Author", _orchardServices.WorkContext.CurrentUser.UserName), + new XElement("ExportUtc", XmlConvert.ToString(_clock.UtcNow, XmlDateTimeSerializationMode.Utc)) ) ) ); @@ -148,18 +166,18 @@ namespace Orchard.ImportExport.Services { return settings; } - private XElement ExportData(IEnumerable contentTypes, VersionHistoryOptions versionHistoryOptions) { + private XElement ExportData(IEnumerable contentTypes, IEnumerable contentItems, int? batchSize) { var data = new XElement("Data"); - var options = GetContentExportVersionOptions(versionHistoryOptions); - var contentItems = _orchardServices.ContentManager.Query(options).List(); + if (batchSize.HasValue && batchSize.Value > 0) + data.SetAttributeValue("BatchSize", batchSize); foreach (var contentType in contentTypes) { var type = contentType; var items = contentItems.Where(i => i.ContentType == type); foreach (var contentItem in items) { var contentItemElement = ExportContentItem(contentItem); - if (contentItemElement != null) + if (contentItemElement != null) data.Add(contentItemElement); } } diff --git a/src/Orchard.Web/Modules/Orchard.ImportExport/Views/Admin/ImportResult.cshtml b/src/Orchard.Web/Modules/Orchard.ImportExport/Views/Admin/ImportResult.cshtml new file mode 100644 index 000000000..0c12dac8f --- /dev/null +++ b/src/Orchard.Web/Modules/Orchard.ImportExport/Views/Admin/ImportResult.cshtml @@ -0,0 +1,30 @@ +@model Orchard.Recipes.Models.RecipeJournal +@{ + Layout.Title = T("Import Result").ToString(); +} +
+ + @Model.Status +
+
+ + + + + + + + + + @foreach (var message in Model.Messages) + { + + + + } +
@T("Message")
+ @message.Message +
+
+ +@Html.ActionLink(T("Close").ToString(), "Import", null, new { @class = "button" }) \ No newline at end of file diff --git a/src/Orchard.Web/Modules/Orchard.Recipes/Orchard.Recipes.csproj b/src/Orchard.Web/Modules/Orchard.Recipes/Orchard.Recipes.csproj index aa3a9c06f..a58ddba13 100644 --- a/src/Orchard.Web/Modules/Orchard.Recipes/Orchard.Recipes.csproj +++ b/src/Orchard.Web/Modules/Orchard.Recipes/Orchard.Recipes.csproj @@ -20,6 +20,10 @@ 4.0 + + + + true diff --git a/src/Orchard.Web/Modules/Orchard.Recipes/RecipeHandlers/DataRecipeHandler.cs b/src/Orchard.Web/Modules/Orchard.Recipes/RecipeHandlers/DataRecipeHandler.cs index 0203102c8..66a8400aa 100644 --- a/src/Orchard.Web/Modules/Orchard.Recipes/RecipeHandlers/DataRecipeHandler.cs +++ b/src/Orchard.Web/Modules/Orchard.Recipes/RecipeHandlers/DataRecipeHandler.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; +using System.Xml.Linq; using Orchard.ContentManagement; +using Orchard.Data; using Orchard.Localization; using Orchard.Logging; using Orchard.Recipes.Models; @@ -8,9 +11,11 @@ using Orchard.Recipes.Services; namespace Orchard.Recipes.RecipeHandlers { public class DataRecipeHandler : IRecipeHandler { private readonly IOrchardServices _orchardServices; + private readonly ITransactionManager _transactionManager; - public DataRecipeHandler(IOrchardServices orchardServices) { + public DataRecipeHandler(IOrchardServices orchardServices, ITransactionManager transactionManager) { _orchardServices = orchardServices; + _transactionManager = transactionManager; Logger = NullLogger.Instance; T = NullLocalizer.Instance; } @@ -25,27 +30,72 @@ namespace Orchard.Recipes.RecipeHandlers { return; } - // First pass to resolve content items from content identities for all content items, new and old. var importContentSession = new ImportContentSession(_orchardServices.ContentManager); - foreach (var element in recipeContext.RecipeStep.Step.Elements()) { - var elementId = element.Attribute("Id"); - if (elementId == null) - continue; - var identity = elementId.Value; - var status = element.Attribute("Status"); - - importContentSession.Set(identity, element.Name.LocalName); - - var item = importContentSession.Get(identity); + // Populate local dictionary with elements and their ids + var elementDictionary = CreateElementDictionary(recipeContext.RecipeStep.Step); + + //Populate import session with all identities to be imported + foreach (var identity in elementDictionary.Keys) { + importContentSession.Set(identity.ToString(), elementDictionary[identity].Name.LocalName); } - // Second pass to import the content items. - foreach (var element in recipeContext.RecipeStep.Step.Elements()) { - _orchardServices.ContentManager.Import(element, importContentSession); + //Determine if the import is to be batched in multiple transactions + var startIndex = 0; + int batchSize = GetBatchSizeForDataStep(recipeContext.RecipeStep.Step); + + //Run the import + ContentIdentity nextIdentity = null; + try { + while (startIndex < elementDictionary.Count) { + importContentSession.InitializeBatch(startIndex, batchSize); + + //the session determines which items are included in the current batch + //so that dependencies can be managed within the same transaction + nextIdentity = importContentSession.GetNextInBatch(); + while (nextIdentity != null) { + _orchardServices.ContentManager.Import(elementDictionary[nextIdentity], importContentSession); + nextIdentity = importContentSession.GetNextInBatch(); + } + + startIndex += batchSize; + + //Create a new transaction for each batch + if (startIndex < elementDictionary.Count) { + _orchardServices.ContentManager.Clear(); + _transactionManager.RequireNew(); + } + } + } + catch (Exception) { + //Ensure a failed batch is rolled back + _transactionManager.Cancel(); + throw; } recipeContext.Executed = true; } + + private Dictionary CreateElementDictionary(XElement step) { + var elementDictionary = new Dictionary(new ContentIdentity.ContentIdentityEqualityComparer()); + foreach (var element in step.Elements()) { + if (element.Attribute("Id") == null || string.IsNullOrEmpty(element.Attribute("Id").Value)) + continue; + + var identity = new ContentIdentity(element.Attribute("Id").Value); + elementDictionary[identity] = element; + } + return elementDictionary; + } + + private int GetBatchSizeForDataStep(XElement step) { + int batchSize; + if (step.Attribute("BatchSize") == null || + !int.TryParse(step.Attribute("BatchSize").Value, out batchSize) || + batchSize <= 0) { + batchSize = int.MaxValue; + } + return batchSize; + } } } diff --git a/src/Orchard.Web/Modules/Orchard.Recipes/Services/RecipeManager.cs b/src/Orchard.Web/Modules/Orchard.Recipes/Services/RecipeManager.cs index 5c307a6d3..876486eda 100644 --- a/src/Orchard.Web/Modules/Orchard.Recipes/Services/RecipeManager.cs +++ b/src/Orchard.Web/Modules/Orchard.Recipes/Services/RecipeManager.cs @@ -1,6 +1,7 @@ using System; using Orchard.Localization; using Orchard.Logging; +using Orchard.Recipes.Events; using Orchard.Recipes.Models; namespace Orchard.Recipes.Services { @@ -8,11 +9,14 @@ namespace Orchard.Recipes.Services { private readonly IRecipeStepQueue _recipeStepQueue; private readonly IRecipeScheduler _recipeScheduler; private readonly IRecipeJournal _recipeJournal; + private readonly IRecipeExecuteEventHandler _recipeExecuteEventHandler; - public RecipeManager(IRecipeStepQueue recipeStepQueue, IRecipeScheduler recipeScheduler, IRecipeJournal recipeJournal) { + public RecipeManager(IRecipeStepQueue recipeStepQueue, IRecipeScheduler recipeScheduler, IRecipeJournal recipeJournal, + IRecipeExecuteEventHandler recipeExecuteEventHandler) { _recipeStepQueue = recipeStepQueue; _recipeScheduler = recipeScheduler; _recipeJournal = recipeJournal; + _recipeExecuteEventHandler = recipeExecuteEventHandler; Logger = NullLogger.Instance; T = NullLocalizer.Instance; @@ -27,6 +31,7 @@ namespace Orchard.Recipes.Services { var executionId = Guid.NewGuid().ToString("n"); _recipeJournal.ExecutionStart(executionId); + _recipeExecuteEventHandler.ExecutionStart(executionId, recipe); foreach (var recipeStep in recipe.RecipeSteps) { _recipeStepQueue.Enqueue(executionId, recipeStep); diff --git a/src/Orchard.Web/Modules/Orchard.Recipes/Services/RecipeParser.cs b/src/Orchard.Web/Modules/Orchard.Recipes/Services/RecipeParser.cs index 75b2cbfb3..46cb2bbb5 100644 --- a/src/Orchard.Web/Modules/Orchard.Recipes/Services/RecipeParser.cs +++ b/src/Orchard.Web/Modules/Orchard.Recipes/Services/RecipeParser.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Xml; using System.Xml.Linq; using Orchard.Localization; using Orchard.Logging; @@ -45,6 +46,9 @@ namespace Orchard.Recipes.Services { case "Version": recipe.Version = metadataElement.Value; break; + case "ExportUtc": + recipe.ExportUtc = !string.IsNullOrEmpty(metadataElement.Value) ? (DateTime?)XmlConvert.ToDateTime(metadataElement.Value, XmlDateTimeSerializationMode.Utc) : null; + break; case "Tags": recipe.Tags = metadataElement.Value; break; diff --git a/src/Orchard.Web/Modules/Orchard.Recipes/Services/RecipeStepExecutor.cs b/src/Orchard.Web/Modules/Orchard.Recipes/Services/RecipeStepExecutor.cs index 05d058826..7e58c3a53 100644 --- a/src/Orchard.Web/Modules/Orchard.Recipes/Services/RecipeStepExecutor.cs +++ b/src/Orchard.Web/Modules/Orchard.Recipes/Services/RecipeStepExecutor.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using Orchard.Localization; using Orchard.Logging; +using Orchard.Recipes.Events; using Orchard.Recipes.Models; namespace Orchard.Recipes.Services { @@ -9,11 +10,14 @@ namespace Orchard.Recipes.Services { private readonly IRecipeStepQueue _recipeStepQueue; private readonly IRecipeJournal _recipeJournal; private readonly IEnumerable _recipeHandlers; + private readonly IRecipeExecuteEventHandler _recipeExecuteEventHandler; - public RecipeStepExecutor(IRecipeStepQueue recipeStepQueue, IRecipeJournal recipeJournal, IEnumerable recipeHandlers) { + public RecipeStepExecutor(IRecipeStepQueue recipeStepQueue, IRecipeJournal recipeJournal, + IEnumerable recipeHandlers, IRecipeExecuteEventHandler recipeExecuteEventHandler) { _recipeStepQueue = recipeStepQueue; _recipeJournal = recipeJournal; _recipeHandlers = recipeHandlers; + _recipeExecuteEventHandler = recipeExecuteEventHandler; Logger = NullLogger.Instance; T = NullLocalizer.Instance; @@ -26,29 +30,38 @@ namespace Orchard.Recipes.Services { var nextRecipeStep= _recipeStepQueue.Dequeue(executionId); if (nextRecipeStep == null) { _recipeJournal.ExecutionComplete(executionId); + _recipeExecuteEventHandler.ExecutionComplete(executionId); return false; } _recipeJournal.WriteJournalEntry(executionId, string.Format("Executing step {0}.", nextRecipeStep.Name)); var recipeContext = new RecipeContext { RecipeStep = nextRecipeStep, Executed = false }; try { + _recipeExecuteEventHandler.RecipeStepExecuting(executionId, recipeContext); foreach (var recipeHandler in _recipeHandlers) { recipeHandler.ExecuteRecipeStep(recipeContext); } + _recipeExecuteEventHandler.RecipeStepExecuted(executionId, recipeContext); } catch(Exception exception) { Logger.Error(exception, "Recipe execution {0} was cancelled because a step failed to execute", executionId); while (_recipeStepQueue.Dequeue(executionId) != null) ; _recipeJournal.ExecutionFailed(executionId); - throw new OrchardCoreException(T("Recipe execution with id {0} was cancelled because the \"{1}\" step failed to execute. The following exception was thrown: {2}. Refer to the recipe journal for more information.", - executionId, nextRecipeStep.Name, exception.Message)); + var message = T("Recipe execution with id {0} was cancelled because the \"{1}\" step failed to execute. The following exception was thrown: {2}. Refer to the error logs for more information.", + executionId, nextRecipeStep.Name, exception.Message); + _recipeJournal.WriteJournalEntry(executionId, message.ToString()); + + throw new OrchardCoreException(message); } if (!recipeContext.Executed) { Logger.Error("Could not execute recipe step '{0}' because the recipe handler was not found.", recipeContext.RecipeStep.Name); while (_recipeStepQueue.Dequeue(executionId) != null) ; _recipeJournal.ExecutionFailed(executionId); - throw new OrchardCoreException(T("Recipe execution with id {0} was cancelled because the recipe handler for step \"{1}\" was not found. Refer to the recipe journal for more information.", - executionId, nextRecipeStep.Name)); + var message = T("Recipe execution with id {0} was cancelled because the recipe handler for step \"{1}\" was not found. Refer to the error logs for more information.", + executionId, nextRecipeStep.Name); + _recipeJournal.WriteJournalEntry(executionId, message.ToString()); + + throw new OrchardCoreException(message); } return true; diff --git a/src/Orchard/ContentManagement/ContentIdentity.cs b/src/Orchard/ContentManagement/ContentIdentity.cs index 3c7308404..2275440d8 100644 --- a/src/Orchard/ContentManagement/ContentIdentity.cs +++ b/src/Orchard/ContentManagement/ContentIdentity.cs @@ -6,6 +6,8 @@ using System.Text; namespace Orchard.ContentManagement { public class ContentIdentity { private readonly Dictionary _dictionary; + private int _currentIdentityPriority = int.MinValue; //initialise to lowest possible priority + private string _encodedIdentity = null; public ContentIdentity() { _dictionary = new Dictionary(); @@ -18,19 +20,33 @@ namespace Orchard.ContentManagement { foreach (var identityEntry in identityEntries) { var keyValuePair = GetIdentityKeyValue(identityEntry); if (keyValuePair != null) { - _dictionary.Add(keyValuePair.Value.Key, UnencodeIdentityValue(keyValuePair.Value.Value)); + Add(keyValuePair.Value.Key, UnencodeIdentityValue(keyValuePair.Value.Value)); } } } } public void Add(string name, string value) { + Add(name, value, 0/*default priority*/); + } + + public void Add(string name, string value, int priority) { + if (priority < _currentIdentityPriority) + return; //lower priority, so ignore + if (priority > _currentIdentityPriority) + _dictionary.Clear(); //higher, so override and delete existing + + //save the current highest priority + _currentIdentityPriority = priority; + + //if equal or higher priority add to identity collection if (_dictionary.ContainsKey(name)) { _dictionary[name] = value; } - else { - _dictionary.Add(name, value); + else { + _dictionary.Add(name, value); } + _encodedIdentity = null; } public string Get(string name) { @@ -38,12 +54,16 @@ namespace Orchard.ContentManagement { } public override string ToString() { + if (_encodedIdentity != null) + return _encodedIdentity; + var stringBuilder = new StringBuilder(); - foreach (var key in _dictionary.Keys) { + foreach (var key in _dictionary.Keys.OrderBy(key => key)) { var escapedIdentity = EncodeIdentityValue(_dictionary[key]); stringBuilder.Append("/" + key + "=" + escapedIdentity); } - return stringBuilder.ToString(); + _encodedIdentity = stringBuilder.ToString(); + return _encodedIdentity; } private static string EncodeIdentityValue(string identityValue) { @@ -141,14 +161,11 @@ namespace Orchard.ContentManagement { public class ContentIdentityEqualityComparer : IEqualityComparer { public bool Equals(ContentIdentity contentIdentity1, ContentIdentity contentIdentity2) { - if (contentIdentity1._dictionary.Keys.Count != contentIdentity2._dictionary.Keys.Count) - return false; - - return contentIdentity1._dictionary.OrderBy(kvp => kvp.Key).SequenceEqual(contentIdentity2._dictionary.OrderBy(kvp => kvp.Key)); + return contentIdentity1.ToString().Equals(contentIdentity2.ToString()); } public int GetHashCode(ContentIdentity contentIdentity) { - return contentIdentity._dictionary.OrderBy(kvp => kvp.Key).ToString().GetHashCode(); + return contentIdentity.ToString().GetHashCode(); } } diff --git a/src/Orchard/ContentManagement/DefaultContentManager.cs b/src/Orchard/ContentManagement/DefaultContentManager.cs index 6d8e860f4..905df48a5 100644 --- a/src/Orchard/ContentManagement/DefaultContentManager.cs +++ b/src/Orchard/ContentManagement/DefaultContentManager.cs @@ -514,6 +514,18 @@ namespace Orchard.ContentManagement { } } + public bool HasResolverForIdentity(ContentIdentity contentIdentity) { + var context = new RegisterIdentityResolverContext(); + Handlers.Invoke(handler => handler.RegisterIdentityResolver(context), Logger); + return context.HasResolverForIdentity(contentIdentity); + } + + public ContentItem ResolveIdentity(ContentIdentity contentIdentity) { + var context = new RegisterIdentityResolverContext(); + Handlers.Invoke(handler => handler.RegisterIdentityResolver(context), Logger); + return context.ResolveIdentity(contentIdentity); + } + public ContentItemMetadata GetItemMetadata(IContent content) { var context = new GetContentItemMetadataContext { ContentItem = content.ContentItem, @@ -593,7 +605,7 @@ namespace Orchard.ContentManagement { var identity = elementId.Value; var status = element.Attribute("Status"); - var item = importContentSession.Get(identity, XmlConvert.DecodeName(element.Name.LocalName)); + var item = importContentSession.Get(identity, VersionOptions.DraftRequired, XmlConvert.DecodeName(element.Name.LocalName)); if (item == null) { item = New(XmlConvert.DecodeName(element.Name.LocalName)); if (status != null && status.Value == "Draft") { @@ -625,7 +637,7 @@ namespace Orchard.ContentManagement { contentHandler.Imported(context); } - var savedItem = Get(item.Id, VersionOptions.DraftRequired); + var savedItem = Get(item.Id, VersionOptions.Latest); // the item has been pre-created in the first pass of the import, create it in db if(savedItem == null) { diff --git a/src/Orchard/ContentManagement/Handlers/ContentHandler.cs b/src/Orchard/ContentManagement/Handlers/ContentHandler.cs index eae8e82c2..212ba94fe 100644 --- a/src/Orchard/ContentManagement/Handlers/ContentHandler.cs +++ b/src/Orchard/ContentManagement/Handlers/ContentHandler.cs @@ -327,6 +327,10 @@ namespace Orchard.ContentManagement.Handlers { Exported(context); } + void IContentHandler.RegisterIdentityResolver(RegisterIdentityResolverContext context) { + RegisterIdentityResolver(context); + } + void IContentHandler.GetContentItemMetadata(GetContentItemMetadataContext context) { foreach (var filter in Filters.OfType()) filter.GetContentItemMetadata(context); @@ -382,6 +386,7 @@ namespace Orchard.ContentManagement.Handlers { protected virtual void Exporting(ExportContentContext context) { } protected virtual void Exported(ExportContentContext context) { } + protected virtual void RegisterIdentityResolver(RegisterIdentityResolverContext context) { } protected virtual void GetItemMetadata(GetContentItemMetadataContext context) { } protected virtual void BuildDisplayShape(BuildDisplayContext context) { } protected virtual void BuildEditorShape(BuildEditorContext context) { } diff --git a/src/Orchard/ContentManagement/Handlers/ContentHandlerBase.cs b/src/Orchard/ContentManagement/Handlers/ContentHandlerBase.cs index a5b3c344c..5a6290ea8 100644 --- a/src/Orchard/ContentManagement/Handlers/ContentHandlerBase.cs +++ b/src/Orchard/ContentManagement/Handlers/ContentHandlerBase.cs @@ -24,6 +24,7 @@ public virtual void Exporting(ExportContentContext context) {} public virtual void Exported(ExportContentContext context) {} + public virtual void RegisterIdentityResolver(RegisterIdentityResolverContext context) { } public virtual void GetContentItemMetadata(GetContentItemMetadataContext context) {} public virtual void BuildDisplay(BuildDisplayContext context) {} public virtual void BuildEditor(BuildEditorContext context) {} diff --git a/src/Orchard/ContentManagement/Handlers/IContentHandler.cs b/src/Orchard/ContentManagement/Handlers/IContentHandler.cs index 86489de92..18e34b015 100644 --- a/src/Orchard/ContentManagement/Handlers/IContentHandler.cs +++ b/src/Orchard/ContentManagement/Handlers/IContentHandler.cs @@ -24,6 +24,7 @@ void Exporting(ExportContentContext context); void Exported(ExportContentContext context); + void RegisterIdentityResolver(RegisterIdentityResolverContext context); void GetContentItemMetadata(GetContentItemMetadataContext context); void BuildDisplay(BuildDisplayContext context); void BuildEditor(BuildEditorContext context); diff --git a/src/Orchard/ContentManagement/Handlers/RegisterIdentityResolverContext.cs b/src/Orchard/ContentManagement/Handlers/RegisterIdentityResolverContext.cs new file mode 100644 index 000000000..a14a745c3 --- /dev/null +++ b/src/Orchard/ContentManagement/Handlers/RegisterIdentityResolverContext.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Orchard.ContentManagement.Handlers { + public class RegisterIdentityResolverContext { + private readonly IList, Func>> _resolvers; + + public RegisterIdentityResolverContext() { + _resolvers = new List, Func>>(); + } + + public void Register(Func isResolverForIdentity, Func resolveIdentity) { + _resolvers.Add(new Tuple, Func>( + isResolverForIdentity, resolveIdentity)); + } + + public bool HasResolverForIdentity(ContentIdentity identity) { + return _resolvers.Any(r => r.Item1(identity)); + } + + public ContentItem ResolveIdentity(ContentIdentity identity) { + return _resolvers.Where(r => r.Item1(identity)) + .Select(r => r.Item2(identity)) + .FirstOrDefault(r => r != null); + } + } +} diff --git a/src/Orchard/ContentManagement/IContentManager.cs b/src/Orchard/ContentManagement/IContentManager.cs index 9867f2070..4989d1cce 100644 --- a/src/Orchard/ContentManagement/IContentManager.cs +++ b/src/Orchard/ContentManagement/IContentManager.cs @@ -89,6 +89,8 @@ namespace Orchard.ContentManagement { GroupInfo GetEditorGroupInfo(IContent contentItem, string groupInfoId); GroupInfo GetDisplayGroupInfo(IContent contentItem, string groupInfoId); + bool HasResolverForIdentity(ContentIdentity contentIdentity); + ContentItem ResolveIdentity(ContentIdentity contentIdentity); /// /// Builds the display shape of the specified content item diff --git a/src/Orchard/ContentManagement/ImportContentSession.cs b/src/Orchard/ContentManagement/ImportContentSession.cs index 6aa3993cb..095d8ba3e 100644 --- a/src/Orchard/ContentManagement/ImportContentSession.cs +++ b/src/Orchard/ContentManagement/ImportContentSession.cs @@ -6,30 +6,102 @@ namespace Orchard.ContentManagement { // Maps content identities to content items on the importer. public class ImportContentSession { private readonly IContentManager _contentManager; - private const int BulkPage = 128; - private int _lastIndex = 0; + private readonly ContentIdentity.ContentIdentityEqualityComparer _identityComparer; private readonly Dictionary _identities; - private readonly Dictionary _contentItemIds; private readonly Dictionary _contentTypes; + private readonly Dictionary _draftVersionRecordIds; + + //for batching + private readonly List _allIdentitiesForImport; //List to maintain order + private readonly Dictionary _allIdentitiesForImportStatus; //For fast lookup of status + private readonly Queue _dependencyIdentities; + private int _startIndex; + private int _batchSize = int.MaxValue; + private int _currentIndex; + + //for identity prefetch + private const int BulkPage = 128; + private bool _firstRequest = true; + private bool _allIdentitiesPrefetched = false; public ImportContentSession(IContentManager contentManager) { + _identityComparer = new ContentIdentity.ContentIdentityEqualityComparer(); _contentManager = contentManager; - _identities = new Dictionary(new ContentIdentity.ContentIdentityEqualityComparer()); - _contentItemIds = new Dictionary(); - _contentTypes = new Dictionary(new ContentIdentity.ContentIdentityEqualityComparer()); + + _identities = new Dictionary(_identityComparer); + _contentTypes = new Dictionary(_identityComparer); + _draftVersionRecordIds = new Dictionary(); + + _allIdentitiesForImport = new List(); + _allIdentitiesForImportStatus = new Dictionary(_identityComparer); + _dependencyIdentities = new Queue(); } + public void Set(string id, string contentType) { var contentIdentity = new ContentIdentity(id); _contentTypes[contentIdentity] = contentType; + _allIdentitiesForImport.Add(contentIdentity); + _allIdentitiesForImportStatus[contentIdentity] = false; + } + + public void InitializeBatch(int startIndex, int batchSize) { + _currentIndex = _startIndex = startIndex; + _batchSize = batchSize; + } + + public ContentIdentity GetNextInBatch() { + ContentIdentity nextIdentity; + + //always process identified dependencies regardless of batch size + //so that they are within the same transaction + if (_dependencyIdentities.Any()) { + nextIdentity = _dependencyIdentities.Dequeue(); + _allIdentitiesForImportStatus[nextIdentity] = true; + return nextIdentity; + } + + //check if the item has already been imported (e.g. as a dependency) + while (_currentIndex < _allIdentitiesForImport.Count && + _allIdentitiesForImportStatus[_allIdentitiesForImport[_currentIndex]]) { + _currentIndex++; + } + + if (_currentIndex < _startIndex + _batchSize && //within batch + _currentIndex < _allIdentitiesForImport.Count) //still items to import + { + nextIdentity = _allIdentitiesForImport[_currentIndex]; + _allIdentitiesForImportStatus[nextIdentity] = true; + _currentIndex++; + return nextIdentity; + } + + return null; } public ContentItem Get(string id, string contentTypeHint = null) { + return Get(id, VersionOptions.Latest, contentTypeHint); + } + + public ContentItem Get(string id, VersionOptions versionOptions, string contentTypeHint = null) { var contentIdentity = new ContentIdentity(id); + if (_firstRequest) { + _firstRequest = false; + //If we know we have identities without a resolver + //just load all up-front + if (HasIdentitiesSetWithoutResolver()) { + PrefetchAllIdentities(true); + } + } + // lookup in local cache if (_identities.ContainsKey(contentIdentity)) { - var result = _contentManager.Get(_identities[contentIdentity], VersionOptions.DraftRequired); + if (_draftVersionRecordIds.ContainsKey(_identities[contentIdentity])) { + //draft was previously created. Recall. + versionOptions = VersionOptions.VersionRecord(_draftVersionRecordIds[_identities[contentIdentity]]); + } + var result = _contentManager.Get(_identities[contentIdentity], versionOptions); // if two identities are conflicting, then ensure that there types are the same // e.g., importing a blog as home page (alias=) and the current home page is a page, the blog @@ -39,57 +111,92 @@ namespace Orchard.ContentManagement { } } - // no result ? then check if there are some more content items to load from the db - if(_lastIndex != int.MaxValue) { - - var equalityComparer = new ContentIdentity.ContentIdentityEqualityComparer(); - IEnumerable block; - - // load identities in blocks - while ((block = _contentManager.HqlQuery() - .ForVersion(VersionOptions.Latest) - .OrderBy(x => x.ContentItemVersion(), x => x.Asc("Id")) - .Slice(_lastIndex, BulkPage)).Any()) { + if (!_allIdentitiesPrefetched) { + ContentItem existingItem = null; - foreach (var item in block) { - _lastIndex++; + //try retrieve the item using handlers to resolve, otherwise fall back to full scan + if (_contentManager.HasResolverForIdentity(contentIdentity)) { + existingItem = _contentManager.ResolveIdentity(contentIdentity); - // ignore content item if it has already been imported - if (_contentItemIds.ContainsKey(item.Id)) { - continue; - } + //ensure we have the correct version + if (existingItem != null) + existingItem = _contentManager.Get(existingItem.Id, versionOptions); + } + else { + //may be a contentidentity without a resolver and all identities have not + //been prefetched yet e.g. is a dependency without resolver. + PrefetchAllIdentities(false); + if (_identities.ContainsKey(contentIdentity)) + existingItem = _contentManager.Get(_identities[contentIdentity], versionOptions); + } - var identity = _contentManager.GetItemMetadata(item).Identity; + if (existingItem != null) { + _identities[contentIdentity] = existingItem.Id; + if (versionOptions.IsDraftRequired) { + _draftVersionRecordIds[existingItem.Id] = existingItem.VersionRecord.Id; + } + return existingItem; + } + } - // ignore content item if the same identity is already present - if (_identities.ContainsKey(identity)) { - continue; - } + //create item if not found and draft was requested, or it is found later in the import queue + if (versionOptions.IsDraftRequired || _allIdentitiesForImportStatus.ContainsKey(contentIdentity)) { + var contentType = _contentTypes.ContainsKey(contentIdentity) ? _contentTypes[contentIdentity] : contentTypeHint; - _identities.Add(identity, item.Id); - _contentItemIds.Add(item.Id, identity); - - if (equalityComparer.Equals(identity, contentIdentity)) { - return _contentManager.Get(item.Id, VersionOptions.DraftRequired); - } - } + if (!_contentTypes.ContainsKey(contentIdentity)) { + throw new ArgumentException("Unknown content type for " + id); + } + var contentItem = _contentManager.Create(contentType, VersionOptions.Draft); + + _identities[contentIdentity] = contentItem.Id; + + //store versionrecordid in case a draft is requested again + _draftVersionRecordIds[contentItem.Id] = contentItem.VersionRecord.Id; + + //add the requested item as a dependency if it is not the currently running item + if (_allIdentitiesForImportStatus.ContainsKey(contentIdentity) && + !_allIdentitiesForImportStatus[contentIdentity]) { + _dependencyIdentities.Enqueue(contentIdentity); + } + + return contentItem; + } + + return null; + } + + private bool HasIdentitiesSetWithoutResolver() { + return _allIdentitiesForImport.Any(id => !_contentManager.HasResolverForIdentity(id)); + } + + private void PrefetchAllIdentities(bool clearContentManager) { + if (_allIdentitiesPrefetched) + return; + + IEnumerable block; + int lastIndex = 0; + + while ((block = _contentManager.HqlQuery() + .ForVersion(VersionOptions.Latest) + .OrderBy(x => x.ContentItemVersion(), x => x.Asc("Id")) + .Slice(lastIndex, BulkPage)).Any()) { + foreach (var item in block) { + lastIndex++; + + var identity = _contentManager.GetItemMetadata(item).Identity; + + // store mapping for later + _identities[identity] = item.Id; + } + + //Clearing the ContentManger after import has started can cause errors + if (clearContentManager) { _contentManager.Clear(); } } - _lastIndex = int.MaxValue; - - if(!_contentTypes.ContainsKey(contentIdentity)) { - throw new ArgumentException("Unknown content type for " + id); - - } - - var contentItem = _contentManager.Create(_contentTypes[contentIdentity], VersionOptions.Draft); - _identities[contentIdentity] = contentItem.Id; - _contentItemIds[contentItem.Id] = contentIdentity; - - return contentItem; + _allIdentitiesPrefetched = true; } } } diff --git a/src/Orchard/Orchard.Framework.csproj b/src/Orchard/Orchard.Framework.csproj index 78c928569..e61ffbadc 100644 --- a/src/Orchard/Orchard.Framework.csproj +++ b/src/Orchard/Orchard.Framework.csproj @@ -153,6 +153,7 @@ + @@ -264,6 +265,7 @@ + diff --git a/src/Orchard/Recipes/Events/IRecipeExecuteEventHandler.cs b/src/Orchard/Recipes/Events/IRecipeExecuteEventHandler.cs new file mode 100644 index 000000000..c1d33a3a8 --- /dev/null +++ b/src/Orchard/Recipes/Events/IRecipeExecuteEventHandler.cs @@ -0,0 +1,12 @@ +using Orchard.Events; +using Orchard.Recipes.Models; + +namespace Orchard.Recipes.Events { + public interface IRecipeExecuteEventHandler : IEventHandler { + void ExecutionStart(string executionId, Recipe recipe); + void RecipeStepExecuting(string executionId, RecipeContext context); + void RecipeStepExecuted(string executionId, RecipeContext context); + void ExecutionComplete(string executionId); + void ExecutionFailed(string executionId); + } +} \ No newline at end of file diff --git a/src/Orchard/Recipes/Models/Recipe.cs b/src/Orchard/Recipes/Models/Recipe.cs index 476089c40..37b1f24d6 100644 --- a/src/Orchard/Recipes/Models/Recipe.cs +++ b/src/Orchard/Recipes/Models/Recipe.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; namespace Orchard.Recipes.Models { public class Recipe { @@ -7,6 +8,7 @@ namespace Orchard.Recipes.Models { public string Author { get; set; } public string WebSite { get; set; } public string Version { get; set; } + public DateTime? ExportUtc { get; set; } public string Tags { get; set; } public IEnumerable RecipeSteps { get; set; } }