Fixing the FileSystemOutputCache feature (#7913)

Fixes #8004 #6115 
- Caching keys for filenames to prevent too long paths
- Separating metadata from content storage to optimize some scenarios

src/Orchard.Web/Modules/Orchard.OutputCache/Services/FileSystemOutputCacheBackgroundTask.cs

@@ -1,4 +1,5 @@
-using System.Linq;
+using System.IO;
+using System.Linq;
 using Orchard.Caching;
 using Orchard.Environment.Configuration;
 using Orchard.Environment.Extensions;
@@ -18,7 +19,8 @@ namespace Orchard.OutputCache.Services {
         private readonly IClock _clock;
         private readonly ISignals _signals;
 
-        private string _root;
+        private string _content;
+        private string _metadata;
 
         public FileSystemOutputCacheBackgroundTask(
             IAppDataFolder appDataFolder, 
@@ -32,22 +34,26 @@ namespace Orchard.OutputCache.Services {
             _clock = clock;
             _signals = signals;
 
-            _root = _appDataFolder.Combine("OutputCache", _shellSettings.Name);
+            _metadata = FileSystemOutputCacheStorageProvider.GetMetadataPath(appDataFolder, _shellSettings.Name);
+            _content = FileSystemOutputCacheStorageProvider.GetContentPath(appDataFolder, _shellSettings.Name);
         }
 
         public void Sweep() {
-            foreach(var filename in _appDataFolder.ListFiles(_root).ToArray()) {
+            foreach(var filename in _appDataFolder.ListFiles(_metadata).ToArray()) {
-                var validUntilUtc = _cacheManager.Get(filename, context => {
+                var hash = Path.GetFileName(filename);
-                    _signals.When(filename);
+
+                var validUntilUtc = _cacheManager.Get(hash, context => {
+                    _signals.When(hash);
 
                     using (var stream = _appDataFolder.OpenFile(filename)) {
-                        var cacheItem = FileSystemOutputCacheStorageProvider.Deserialize(stream);
+                        var cacheItem = FileSystemOutputCacheStorageProvider.DeserializeMetadata(stream);
                         return cacheItem.ValidUntilUtc;
                     }
                 });
 
                 if (_clock.UtcNow > validUntilUtc) {
-                    _appDataFolder.DeleteFile(filename);
+                    _appDataFolder.DeleteFile(_appDataFolder.Combine(_metadata, hash));
+                    _appDataFolder.DeleteFile(_appDataFolder.Combine(_content, hash));
                     _signals.Trigger(filename);
                 }
             }

src/Orchard.Web/Modules/Orchard.OutputCache/Services/FileSystemOutputCacheProvider.cs

@@ -1,45 +1,51 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
-using Orchard.OutputCache.Models;
-using Orchard.Environment.Extensions;
-using Orchard.Logging;
-using Orchard.Services;
-using Orchard.FileSystems.AppData;
-using Orchard.Environment.Configuration;
-using System.Web;
 using System.IO;
+using System.Linq;
 using System.Runtime.Serialization.Formatters.Binary;
+using System.Security.Cryptography;
+using System.Text;
+using Orchard.Environment.Configuration;
+using Orchard.Environment.Extensions;
+using Orchard.FileSystems.AppData;
+using Orchard.Logging;
+using Orchard.OutputCache.Models;
+using Orchard.Services;
 
 namespace Orchard.OutputCache.Services {
     [OrchardFeature("Orchard.OutputCache.FileSystem")]
     [OrchardSuppressDependency("Orchard.OutputCache.Services.DefaultCacheStorageProvider")]
     /// <summary>
     /// This class provides an implementation of <see cref="IOutputCacheStorageProvider"/>
-    /// based on the local App_Data folder, inside <c>OuputCache/{tenant}</c>. It is not 
+    /// based on the local App_Data folder, inside <c>FileCache/{tenant}</c>. It is not 
-    /// recommended when used in a server farm.
+    /// recommended when used in a server farm, unless the file system is share (Azure App Services).
     /// The <see cref="CacheItem"/> instances are binary serialized.
     /// </summary>
     /// <remarks>
-    /// This provider doesn't implement quotas support yet.
+    /// This provider doesn't support quotas yet.
     /// </remarks>
     public class FileSystemOutputCacheStorageProvider : IOutputCacheStorageProvider {
         private readonly IClock _clock;
         private readonly IAppDataFolder _appDataFolder;
         private readonly ShellSettings _shellSettings;
-        private readonly string _root;
+        private readonly string _metadata;
+        private readonly string _content;
+
+        public static char[] InvalidPathChars = { '/', '\\', ':', '*', '?', '>', '<', '|' };
 
         public FileSystemOutputCacheStorageProvider(IClock clock, IAppDataFolder appDataFolder, ShellSettings shellSettings) {
             _appDataFolder = appDataFolder;
             _clock = clock;
             _shellSettings = shellSettings;
-            _root = _appDataFolder.Combine("OutputCache", _shellSettings.Name);
+
+            _metadata = GetMetadataPath(appDataFolder, _shellSettings.Name);
+            _content = GetContentPath(appDataFolder, _shellSettings.Name);
 
             Logger = NullLogger.Instance;
         }
 
         public ILogger Logger { get; set; }
- 
+
         public void Set(string key, CacheItem cacheItem) {
             Retry(() => {
                 if (cacheItem == null) {
@@ -50,82 +56,170 @@ namespace Orchard.OutputCache.Services {
                     return;
                 }
 
-                var filename = GetCacheItemFilename(key);
+                var hash = GetCacheItemFileHash(key);
 
-                using (var stream = Serialize(cacheItem)) {
+                lock (String.Intern(hash)) {
-                    using (var fileStream = _appDataFolder.CreateFile(filename)) {
+                    using (var stream = SerializeContent(cacheItem)) {
-                        stream.CopyTo(fileStream);
+                        var filename = _appDataFolder.Combine(_content, hash);
+                        using (var fileStream = _appDataFolder.CreateFile(filename)) {
+                            stream.CopyTo(fileStream);
+                        }
+                    }
+
+                    using (var stream = SerializeMetadata(cacheItem)) {
+                        var filename = _appDataFolder.Combine(_metadata, hash);
+                        using (var fileStream = _appDataFolder.CreateFile(filename)) {
+                            stream.CopyTo(fileStream);
+                        }
                     }
                 }
             });
         }
 
         public void Remove(string key) {
-            Retry(() => {
+            var hash = GetCacheItemFileHash(key);
-                var filename = GetCacheItemFilename(key);
+            lock (String.Intern(hash)) {
-                if (_appDataFolder.FileExists(filename)) {
+                Retry(() => {
-                    _appDataFolder.DeleteFile(filename);
+                    var filename = _appDataFolder.Combine(_metadata, hash);
-                }
+                    if (_appDataFolder.FileExists(filename)) {
-            });
+                        _appDataFolder.DeleteFile(filename);
+                    }
+                });
+
+                Retry(() => {
+                    var filename = _appDataFolder.Combine(_content, hash);
+                    if (_appDataFolder.FileExists(filename)) {
+                        _appDataFolder.DeleteFile(filename);
+                    }
+                });
+            }
         }
 
         public void RemoveAll() {
-            foreach(var filename in _appDataFolder.ListFiles(_root)) {
+            foreach (var folder in new[] { _metadata, _content }) {
-                if(_appDataFolder.FileExists(filename)) {
+                foreach (var filename in _appDataFolder.ListFiles(folder)) {
-                    _appDataFolder.DeleteFile(filename);
+                    var hash = Path.GetFileName(filename);
+                    lock (String.Intern(hash)) {
+                        try {
+                            if (_appDataFolder.FileExists(filename)) {
+                                _appDataFolder.DeleteFile(filename);
+                            }
+                        }
+                        catch (Exception e) {
+                            Logger.Warning(e, "An error occured while deleting the file: {0}", filename);
+                        }
+                    }
                 }
             }
         }
 
         public CacheItem GetCacheItem(string key) {
             return Retry(() => {
-                var filename = GetCacheItemFilename(key);
+                var hash = GetCacheItemFileHash(key);
+                lock (String.Intern(hash)) {
+                    var filename = _appDataFolder.Combine(_metadata, hash);
 
-                if (!_appDataFolder.FileExists(filename)) {
+                    if (!_appDataFolder.FileExists(filename)) {
-                    return null;
-                }
-
-                using (var stream = _appDataFolder.OpenFile(filename)) {
-
-                    if (stream == null) {
                         return null;
                     }
 
-                    return Deserialize(stream);
+                    CacheItem cacheItem = null;
+
+                    using (var stream = _appDataFolder.OpenFile(filename)) {
+                        if (stream == null) {
+                            return null;
+                        }
+
+                        cacheItem = DeserializeMetadata(stream);
+
+                        // We compare the requested key and the one stored in the metadata
+                        // as there could be key collisions with the hashed filenames.
+                        if (!cacheItem.CacheKey.Equals(key)) {
+                            return null;
+                        }
+                    }
+
+                    filename = _appDataFolder.Combine(_content, hash);
+                    using (var stream = _appDataFolder.OpenFile(filename)) {
+                        if (stream == null) {
+                            return null;
+                        }
+
+                        var content = DeserializeContent(stream);
+                        cacheItem.Output = content;
+                    }
+
+                    return cacheItem;
                 }
             });
         }
 
         public IEnumerable<CacheItem> GetCacheItems(int skip, int count) {
-            return _appDataFolder.ListFiles(_root)
+            return _appDataFolder.ListFiles(_metadata)
                 .OrderBy(x => x)
                 .Skip(skip)
                 .Take(count)
                 .Select(filename => {
                     using (var stream = _appDataFolder.OpenFile(filename)) {
-                        return Deserialize(stream);
+                        return DeserializeMetadata(stream);
                     }
                 })
                 .ToList();
         }
 
         public int GetCacheItemsCount() {
-            return _appDataFolder.ListFiles(_root).Count();
+            return _appDataFolder.ListFiles(_metadata).Count();
-        }
-        
-        private string GetCacheItemFilename(string key) {
-            return _appDataFolder.Combine(_root, HttpUtility.UrlEncode(key));
         }
 
-        internal static MemoryStream Serialize(CacheItem item) {
+        public static string GetMetadataPath(IAppDataFolder appDataFolder, string tenant) {
+            return appDataFolder.Combine("FileCache", tenant, "metadata");
+        }
+
+        public static string GetContentPath(IAppDataFolder appDataFolder, string tenant) {
+            return appDataFolder.Combine("FileCache", tenant, "content");
+        }
+
+        private string GetCacheItemFileHash(string key) {
+            // The key is typically too long to be useful, so we use a hash
+            using (var md5 = MD5.Create()) {
+                var keyBytes = Encoding.UTF8.GetBytes(key);
+                var hashedBytes = md5.ComputeHash(keyBytes);
+                var b64 = Convert.ToBase64String(hashedBytes);
+                return String.Join("-", b64.Split(InvalidPathChars, StringSplitOptions.RemoveEmptyEntries));
+            }
+        }
+
+        internal static MemoryStream SerializeContent(CacheItem item) {
             BinaryFormatter binaryFormatter = new BinaryFormatter();
             var memoryStream = new MemoryStream();
-            binaryFormatter.Serialize(memoryStream, item);
+            binaryFormatter.Serialize(memoryStream, item.Output);
             memoryStream.Seek(0, SeekOrigin.Begin);
             return memoryStream;
         }
 
-        internal static CacheItem Deserialize(Stream stream) {
+        internal static byte[] DeserializeContent(Stream stream) {
+            BinaryFormatter binaryFormatter = new BinaryFormatter();
+            var result = (byte[])binaryFormatter.Deserialize(stream);
+            return result;
+        }
+
+        internal static MemoryStream SerializeMetadata(CacheItem item) {
+            var output = item.Output;
+            item.Output = new byte[0];
+
+            try {
+                BinaryFormatter binaryFormatter = new BinaryFormatter();
+                var memoryStream = new MemoryStream();
+                binaryFormatter.Serialize(memoryStream, item);
+                memoryStream.Seek(0, SeekOrigin.Begin);
+                return memoryStream;
+            }
+            finally {
+                item.Output = output;
+            }            
+        }
+
+        internal static CacheItem DeserializeMetadata(Stream stream) {
             BinaryFormatter binaryFormatter = new BinaryFormatter();
             var result = (CacheItem)binaryFormatter.Deserialize(stream);
             return result;
@@ -138,7 +232,9 @@ namespace Orchard.OutputCache.Services {
                     var t = action();
                     return t;
                 }
-                catch {
+                catch(Exception e) {
+                    Logger.Warning("An unexpected error occured, attempt # {0}, i", e);
+
                     if (i == retries) {
                         throw;
                     }
@@ -153,9 +249,12 @@ namespace Orchard.OutputCache.Services {
             for(int i=1; i <= retries; i++) {
                 try {
                     action();
+                    return;
                 }
-                catch {
+                catch(Exception e) {
-                    if(i == retries) {
+                    Logger.Warning("An unexpected error occured, attempt # {0}, i", e);
+
+                    if (i == retries) {
                         throw;
                     }
                 }