lsm/Orchard
1
0
Fork 0
mirror of https://github.com/OrchardCMS/Orchard.git synced 2025-08-20 07:23:59 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Fixing the FileSystemOutputCache feature

Browse Source 
- Caching keys for filenames to prevent too long paths
- Separating metadata from content storage to optimize some scenarios
- Fixing retry logic
This commit is contained in:
Sebastien Ros 2016-03-10 17:11:33 -08:00
parent 712ff2f487
commit d7d26a3215
2 changed files with 162 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/Orchard.Web/Modules/Orchard.OutputCache/Services/FileSystemOutputCacheBackgroundTask.cs
View File

@ -1,4 +1,5 @@
using System.Linq;
using System.IO;
using System.Linq;
using Orchard.Caching;
using Orchard.Environment.Configuration;
using Orchard.Environment.Extensions;
@ -18,7 +19,8 @@ namespace Orchard.OutputCache.Services {
private readonly IClock _clock;
private readonly ISignals _signals;
private string _root;
private string _content;
private string _metadata;
public FileSystemOutputCacheBackgroundTask(
IAppDataFolder appDataFolder,
@ -32,22 +34,26 @@ namespace Orchard.OutputCache.Services {
_clock = clock;
_signals = signals;
_root = _appDataFolder.Combine("OutputCache", _shellSettings.Name);
_metadata = FileSystemOutputCacheStorageProvider.GetMetadataPath(appDataFolder, _shellSettings.Name);
_content = FileSystemOutputCacheStorageProvider.GetContentPath(appDataFolder, _shellSettings.Name);
}
public void Sweep() {
foreach(var filename in _appDataFolder.ListFiles(_root).ToArray()) {
var validUntilUtc = _cacheManager.Get(filename, context => {
_signals.When(filename);
foreach(var filename in _appDataFolder.ListFiles(_metadata).ToArray()) {
var hash = Path.GetFileName(filename);
var validUntilUtc = _cacheManager.Get(hash, context => {
_signals.When(hash);
using (var stream = _appDataFolder.OpenFile(filename)) {
var cacheItem = FileSystemOutputCacheStorageProvider.Deserialize(stream);
var cacheItem = FileSystemOutputCacheStorageProvider.DeserializeMetadata(stream);
return cacheItem.ValidUntilUtc;
}
});
if (_clock.UtcNow > validUntilUtc) {
_appDataFolder.DeleteFile(filename);
_appDataFolder.DeleteFile(_appDataFolder.Combine(_metadata, hash));
_appDataFolder.DeleteFile(_appDataFolder.Combine(_content, hash));
_signals.Trigger(filename);
}
}

197
src/Orchard.Web/Modules/Orchard.OutputCache/Services/FileSystemOutputCacheProvider.cs
View File

@ -1,45 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Orchard.OutputCache.Models;
using Orchard.Environment.Extensions;
using Orchard.Logging;
using Orchard.Services;
using Orchard.FileSystems.AppData;
using Orchard.Environment.Configuration;
using System.Web;
using System.IO;
using System.Linq;
using System.Runtime.Serialization.Formatters.Binary;
using System.Security.Cryptography;
using System.Text;
using Orchard.Environment.Configuration;
using Orchard.Environment.Extensions;
using Orchard.FileSystems.AppData;
using Orchard.Logging;
using Orchard.OutputCache.Models;
using Orchard.Services;
namespace Orchard.OutputCache.Services {
[OrchardFeature("Orchard.OutputCache.FileSystem")]
[OrchardSuppressDependency("Orchard.OutputCache.Services.DefaultCacheStorageProvider")]
/// <summary>
/// This class provides an implementation of <see cref="IOutputCacheStorageProvider"/>
/// based on the local App_Data folder, inside <c>OuputCache/{tenant}</c>. It is not
/// recommended when used in a server farm.
/// based on the local App_Data folder, inside <c>FileCache/{tenant}</c>. It is not
/// recommended when used in a server farm, unless the file system is share (Azure App Services).
/// The <see cref="CacheItem"/> instances are binary serialized.
/// </summary>
/// <remarks>
/// This provider doesn't implement quotas support yet.
/// This provider doesn't support quotas yet.
/// </remarks>
public class FileSystemOutputCacheStorageProvider : IOutputCacheStorageProvider {
private readonly IClock _clock;
private readonly IAppDataFolder _appDataFolder;
private readonly ShellSettings _shellSettings;
private readonly string _root;
private readonly string _metadata;
private readonly string _content;
public static char[] InvalidPathChars = { '/', '\\', ':', '*', '?', '>', '<', '|' };
public FileSystemOutputCacheStorageProvider(IClock clock, IAppDataFolder appDataFolder, ShellSettings shellSettings) {
_appDataFolder = appDataFolder;
_clock = clock;
_shellSettings = shellSettings;
_root = _appDataFolder.Combine("OutputCache", _shellSettings.Name);
_metadata = GetMetadataPath(appDataFolder, _shellSettings.Name);
_content = GetContentPath(appDataFolder, _shellSettings.Name);
Logger = NullLogger.Instance;
}
public ILogger Logger { get; set; }
public void Set(string key, CacheItem cacheItem) {
Retry(() => {
if (cacheItem == null) {
@ -50,82 +56,170 @@ namespace Orchard.OutputCache.Services {
return;
}
var filename = GetCacheItemFilename(key);
var hash = GetCacheItemFileHash(key);
using (var stream = Serialize(cacheItem)) {
using (var fileStream = _appDataFolder.CreateFile(filename)) {
stream.CopyTo(fileStream);
lock (String.Intern(hash)) {
using (var stream = SerializeContent(cacheItem)) {
var filename = _appDataFolder.Combine(_content, hash);
using (var fileStream = _appDataFolder.CreateFile(filename)) {
stream.CopyTo(fileStream);
}
}
using (var stream = SerializeMetadata(cacheItem)) {
var filename = _appDataFolder.Combine(_metadata, hash);
using (var fileStream = _appDataFolder.CreateFile(filename)) {
stream.CopyTo(fileStream);
}
}
}
});
}
public void Remove(string key) {
Retry(() => {
var filename = GetCacheItemFilename(key);
if (_appDataFolder.FileExists(filename)) {
_appDataFolder.DeleteFile(filename);
}
});
var hash = GetCacheItemFileHash(key);
lock (String.Intern(hash)) {
Retry(() => {
var filename = _appDataFolder.Combine(_metadata, hash);
if (_appDataFolder.FileExists(filename)) {
_appDataFolder.DeleteFile(filename);
}
});
Retry(() => {
var filename = _appDataFolder.Combine(_content, hash);
if (_appDataFolder.FileExists(filename)) {
_appDataFolder.DeleteFile(filename);
}
});
}
}
public void RemoveAll() {
foreach(var filename in _appDataFolder.ListFiles(_root)) {
if(_appDataFolder.FileExists(filename)) {
_appDataFolder.DeleteFile(filename);
foreach (var folder in new[] { _metadata, _content }) {
foreach (var filename in _appDataFolder.ListFiles(folder)) {
var hash = Path.GetFileName(filename);
lock (String.Intern(hash)) {
try {
if (_appDataFolder.FileExists(filename)) {
_appDataFolder.DeleteFile(filename);
}
}
catch (Exception e) {
Logger.Warning(e, "An error occured while deleting the file: {0}", filename);
}
}
}
}
}
public CacheItem GetCacheItem(string key) {
return Retry(() => {
var filename = GetCacheItemFilename(key);
var hash = GetCacheItemFileHash(key);
lock (String.Intern(hash)) {
var filename = _appDataFolder.Combine(_metadata, hash);
if (!_appDataFolder.FileExists(filename)) {
return null;
}
using (var stream = _appDataFolder.OpenFile(filename)) {
if (stream == null) {
if (!_appDataFolder.FileExists(filename)) {
return null;
}
return Deserialize(stream);
CacheItem cacheItem = null;
using (var stream = _appDataFolder.OpenFile(filename)) {
if (stream == null) {
return null;
}
cacheItem = DeserializeMetadata(stream);
// We compare the requested key and the one stored in the metadata
// as there could be key collisions with the hashed filenames.
if (!cacheItem.CacheKey.Equals(key)) {
return null;
}
}
filename = _appDataFolder.Combine(_content, hash);
using (var stream = _appDataFolder.OpenFile(filename)) {
if (stream == null) {
return null;
}
var content = DeserializeContent(stream);
cacheItem.Output = content;
}
return cacheItem;
}
});
}
public IEnumerable<CacheItem> GetCacheItems(int skip, int count) {
return _appDataFolder.ListFiles(_root)
return _appDataFolder.ListFiles(_metadata)
.OrderBy(x => x)
.Skip(skip)
.Take(count)
.Select(filename => {
using (var stream = _appDataFolder.OpenFile(filename)) {
return Deserialize(stream);
return DeserializeMetadata(stream);
}
})
.ToList();
}
public int GetCacheItemsCount() {
return _appDataFolder.ListFiles(_root).Count();
}
private string GetCacheItemFilename(string key) {
return _appDataFolder.Combine(_root, HttpUtility.UrlEncode(key));
return _appDataFolder.ListFiles(_metadata).Count();
}
internal static MemoryStream Serialize(CacheItem item) {
public static string GetMetadataPath(IAppDataFolder appDataFolder, string tenant) {
return appDataFolder.Combine("FileCache", tenant, "metadata");
}
public static string GetContentPath(IAppDataFolder appDataFolder, string tenant) {
return appDataFolder.Combine("FileCache", tenant, "content");
}
private string GetCacheItemFileHash(string key) {
// The key is typically too long to be useful, so we use a hash
using (var md5 = MD5.Create()) {
var keyBytes = Encoding.UTF8.GetBytes(key);
var hashedBytes = md5.ComputeHash(keyBytes);
var b64 = Convert.ToBase64String(hashedBytes);
return String.Join("-", b64.Split(InvalidPathChars, StringSplitOptions.RemoveEmptyEntries));
}
}
internal static MemoryStream SerializeContent(CacheItem item) {
BinaryFormatter binaryFormatter = new BinaryFormatter();
var memoryStream = new MemoryStream();
binaryFormatter.Serialize(memoryStream, item);
binaryFormatter.Serialize(memoryStream, item.Output);
memoryStream.Seek(0, SeekOrigin.Begin);
return memoryStream;
}
internal static CacheItem Deserialize(Stream stream) {
internal static byte[] DeserializeContent(Stream stream) {
BinaryFormatter binaryFormatter = new BinaryFormatter();
var result = (byte[])binaryFormatter.Deserialize(stream);
return result;
}
internal static MemoryStream SerializeMetadata(CacheItem item) {
var output = item.Output;
item.Output = new byte[0];
try {
BinaryFormatter binaryFormatter = new BinaryFormatter();
var memoryStream = new MemoryStream();
binaryFormatter.Serialize(memoryStream, item);
memoryStream.Seek(0, SeekOrigin.Begin);
return memoryStream;
}
finally {
item.Output = output;
}
}
internal static CacheItem DeserializeMetadata(Stream stream) {
BinaryFormatter binaryFormatter = new BinaryFormatter();
var result = (CacheItem)binaryFormatter.Deserialize(stream);
return result;
@ -138,7 +232,9 @@ namespace Orchard.OutputCache.Services {
var t = action();
return t;
}
catch {
catch(Exception e) {
Logger.Warning("An unexpected error occured, attempt # {0}, i", e);
if (i == retries) {
throw;
}
@ -153,9 +249,12 @@ namespace Orchard.OutputCache.Services {
for(int i=1; i <= retries; i++) {
try {
action();
return;
}
catch {
if(i == retries) {
catch(Exception e) {
Logger.Warning("An unexpected error occured, attempt # {0}, i", e);
if (i == retries) {
throw;
}
}