#20213: Implementing retry logic

Work Item: 20213

src/Orchard.Web/Modules/Orchard.Azure/Orchard.Azure.csproj

@@ -130,7 +130,6 @@
     <Compile Include="Services\Caching\Database\AzureCacheProvider.cs" />
     <Compile Include="Services\Caching\Database\Models\AzureCacheRecord.cs" />
     <Compile Include="Services\Caching\Output\AzureOutputCacheStorageProvider.cs" />
-    <Compile Include="Services\Caching\Output\IAzureOutputCacheHolder.cs" />
     <Compile Include="Services\Environment\Configuration\AzureBlobShellSettingsManager.cs" />
     <Compile Include="Services\FileSystems\AzureFileSystem.cs" />
     <Compile Include="Services\FileSystems\CloudBlobContainerExtensions.cs" />

src/Orchard.Web/Modules/Orchard.Azure/Services/Caching/Output/AzureOutputCacheStorageProvider.cs

@@ -4,6 +4,7 @@ using System.Globalization;
 using System.Linq;
 using Microsoft.ApplicationServer.Caching;
 using Orchard.Azure.Services.Environment.Configuration;
+using Orchard.Caching;
 using Orchard.Environment.Configuration;
 using Orchard.Environment.Extensions;
 using Orchard.Logging;
@@ -15,11 +16,25 @@ namespace Orchard.Azure.Services.Caching.Output {
     [OrchardFeature(Constants.OutputCacheFeatureName)]
     [OrchardSuppressDependency("Orchard.OutputCache.Services.DefaultCacheStorageProvider")]
     public class AzureOutputCacheStorageProvider : Component, IOutputCacheStorageProvider {
+        public const string DataCacheKey = "DataCache";
+        public const string ClientConfigurationKey = "CacheClientConfiguration";
+        public const int Retries = 2;
 
-        private readonly DataCache _cache;
         private readonly string _regionAlphaNumeric;
+        private readonly ICacheManager _cacheManager;
+        private readonly ShellSettings _shellSettings;
+        private readonly IPlatformConfigurationAccessor _pca;
+        private readonly ISignals _signals;
 
-        public AzureOutputCacheStorageProvider(ShellSettings shellSettings, IAzureOutputCacheHolder cacheHolder, IPlatformConfigurationAccessor pca) {
+        public AzureOutputCacheStorageProvider(
+            ShellSettings shellSettings, 
+            IPlatformConfigurationAccessor pca, 
+            ICacheManager cacheManager,
+            ISignals signals) {
+            _cacheManager = cacheManager;
+            _shellSettings = shellSettings;
+            _pca = pca;
+            _signals = signals;
 
             var region = shellSettings.Name;
 
@@ -29,64 +44,117 @@ namespace Orchard.Azure.Services.Caching.Output {
             // of two distinct original region strings yielding the same transformed region string.
             _regionAlphaNumeric = new String(Array.FindAll(region.ToCharArray(), Char.IsLetterOrDigit)) + region.GetHashCode().ToString(CultureInfo.InvariantCulture);
 
-            _cache = cacheHolder.TryGetDataCache(() => {
-                CacheClientConfiguration cacheConfig;
-
-                try {
-                    cacheConfig = CacheClientConfiguration.FromPlatformConfiguration(shellSettings.Name, Constants.OutputCacheSettingNamePrefix, pca);
-                    cacheConfig.Validate();
-                }
-                catch (Exception ex) {
-                    throw new Exception(String.Format("The {0} configuration settings are missing or invalid.", Constants.OutputCacheFeatureName), ex);
-                }
-
-                var cache = cacheConfig.CreateCache();
-                cache.CreateRegion(_regionAlphaNumeric);
-
-                return cache;
-            });
+            Logger = NullLogger.Instance;
         }
 
+        public ILogger Logger { get; set; }
+
+        public CacheClientConfiguration CacheConfiguration {
+            get {
+                return _cacheManager.Get(ClientConfigurationKey, ctx => {
+                    CacheClientConfiguration cacheConfig ;
+                    try {
+                        cacheConfig = CacheClientConfiguration.FromPlatformConfiguration(
+                            _shellSettings.Name, 
+                            Constants.OutputCacheSettingNamePrefix, 
+                            _pca);
+
+                        cacheConfig.Validate();
+                        return cacheConfig;
+                    }
+                    catch (Exception ex) {
+                        throw new Exception(String.Format("The {0} configuration settings are missing or invalid.", Constants.OutputCacheFeatureName), ex);
+                    } 
+                });
+            }
+        }
+
+        public DataCache Cache {
+            get {
+                return _cacheManager.Get(DataCacheKey, ctx => {
+                    ctx.Monitor(_signals.When(DataCacheKey));
+                    var cache = CacheConfiguration.CreateCache();
+                    cache.CreateRegion(_regionAlphaNumeric);
+
+                    return cache;
+                });
+            }
+        }
+
+        public T SafeCall<T>(Func<T> function) {
+            return Retry(function, Retries);
+        }
+
+        public void SafeCall(Action function) {
+            Retry<object>(() => {
+                function();
+                return null;
+            }, Retries);
+        }
+
+        public T Retry<T>(Func<T> function, int times) {
+            if (times == 0) {
+                Logger.Error("Too many retries in cache resolution.");
+                return default(T);
+            }
+
+            try {
+                return function.Invoke();
+            }
+            catch (DataCacheException) {
+                _signals.Trigger(DataCacheKey);
+                return Retry(function, times--);
+            }
+        } 
+
         public void Set(string key, CacheItem cacheItem) {
             if (cacheItem.ValidFor <= 0) {
                 return;
             }
 
             Logger.Debug("Set() invoked with key='{0}' in region '{1}'.", key, _regionAlphaNumeric);
-            _cache.Put(key, cacheItem, TimeSpan.FromSeconds(cacheItem.ValidFor), _regionAlphaNumeric);
+            SafeCall(() => Cache.Put(key, cacheItem, TimeSpan.FromSeconds(cacheItem.ValidFor), _regionAlphaNumeric));
         }
 
         public void Remove(string key) {
             Logger.Debug("Remove() invoked with key='{0}' in region '{1}'.", key, _regionAlphaNumeric);
-            _cache.Remove(key, _regionAlphaNumeric);
+            SafeCall(() => Cache.Remove(key, _regionAlphaNumeric));
         }
 
         public void RemoveAll() {
             Logger.Debug("RemoveAll() invoked in region '{0}'.", _regionAlphaNumeric);
-            _cache.ClearRegion(_regionAlphaNumeric);
+            SafeCall<object>(() => {
+                Cache.ClearRegion(_regionAlphaNumeric); 
+                return null;
+            });
         }
 
         public CacheItem GetCacheItem(string key) {
             Logger.Debug("GetCacheItem() invoked with key='{0}' in region '{1}'.", key, _regionAlphaNumeric);
-            return _cache.Get(key, _regionAlphaNumeric) as CacheItem;
+            return SafeCall(() => Cache.Get(key, _regionAlphaNumeric)) as CacheItem;
         }
 
         public IEnumerable<CacheItem> GetCacheItems(int skip, int count) {
             Logger.Debug("GetCacheItems() invoked in region '{0}'.", _regionAlphaNumeric);
-            return _cache.GetObjectsInRegion(_regionAlphaNumeric).AsParallel()
+            return SafeCall(() => 
+                Cache.GetObjectsInRegion(_regionAlphaNumeric)
+                .AsParallel()
                 .Select(x => x.Value)
                 .OfType<CacheItem>()
                 .Skip(skip)
                 .Take(count)
-                .ToArray();
+                .ToArray()
+                );
         }
 
         public int GetCacheItemsCount() {
             Logger.Debug("GetCacheItemsCount() invoked in region '{0}'.", _regionAlphaNumeric);
-            return _cache.GetObjectsInRegion(_regionAlphaNumeric).AsParallel()
+            return SafeCall(() => 
+                Cache.GetObjectsInRegion(_regionAlphaNumeric).AsParallel()
                 .Select(x => x.Value)
                 .OfType<CacheItem>()
-                .Count();
+                .Count()
+                );
         }
     }
 }

src/Orchard.Web/Modules/Orchard.Azure/Services/Caching/Output/IAzureOutputCacheHolder.cs

@@ -1,23 +0,0 @@
-using System;
-using Microsoft.ApplicationServer.Caching;
-
-namespace Orchard.Azure.Services.Caching.Output {
-    public interface IAzureOutputCacheHolder : ISingletonDependency {
-        DataCache TryGetDataCache(Func<DataCache> builder);
-    }
-
-    public class AzureOutputCacheHolder : IAzureOutputCacheHolder {
-        private readonly object _synLock = new object();
-        private DataCache _dataCache;
-
-        public DataCache TryGetDataCache(Func<DataCache> builder) {
-            lock (_synLock) {
-                if (_dataCache != null) {
-                    return _dataCache;
-                }
-
-                return _dataCache = builder();
-            }
-        }
-    }
-}