mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-07-17 12:13:52 +08:00
tiered caching
1/4 for small less than 1MB files. 1/4 for 1~4MB files, 1/2 for bigger than 4MB files
This commit is contained in:
parent
f282ed444b
commit
2b5c4fbbf3
@ -84,7 +84,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
if option.CacheSizeMB > 0 {
|
if option.CacheSizeMB > 0 {
|
||||||
wfs.chunkCache = chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB, 4)
|
wfs.chunkCache = chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB)
|
||||||
util.OnInterrupt(func() {
|
util.OnInterrupt(func() {
|
||||||
wfs.chunkCache.Shutdown()
|
wfs.chunkCache.Shutdown()
|
||||||
})
|
})
|
||||||
|
@ -99,7 +99,7 @@ type WebDavFile struct {
|
|||||||
|
|
||||||
func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) {
|
func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) {
|
||||||
|
|
||||||
chunkCache := chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB, 4)
|
chunkCache := chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB)
|
||||||
util.OnInterrupt(func() {
|
util.OnInterrupt(func() {
|
||||||
chunkCache.Shutdown()
|
chunkCache.Shutdown()
|
||||||
})
|
})
|
||||||
|
@ -8,27 +8,27 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
memCacheSizeLimit = 1024 * 1024
|
memCacheSizeLimit = 1024 * 1024
|
||||||
|
onDiskCacheSizeLimit0 = memCacheSizeLimit
|
||||||
|
onDiskCacheSizeLimit1 = 4 * memCacheSizeLimit
|
||||||
)
|
)
|
||||||
|
|
||||||
// a global cache for recently accessed file chunks
|
// a global cache for recently accessed file chunks
|
||||||
type ChunkCache struct {
|
type ChunkCache struct {
|
||||||
memCache *ChunkCacheInMemory
|
memCache *ChunkCacheInMemory
|
||||||
diskCache *OnDiskCacheLayer
|
diskCaches []*OnDiskCacheLayer
|
||||||
sync.RWMutex
|
sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewChunkCache(maxEntries int64, dir string, diskSizeMB int64, segmentCount int) *ChunkCache {
|
func NewChunkCache(maxEntries int64, dir string, diskSizeMB int64) *ChunkCache {
|
||||||
|
|
||||||
volumeCount, volumeSize := int(diskSizeMB/30000), int64(30000)
|
|
||||||
if volumeCount < segmentCount {
|
|
||||||
volumeCount, volumeSize = segmentCount, diskSizeMB/int64(segmentCount)
|
|
||||||
}
|
|
||||||
|
|
||||||
c := &ChunkCache{
|
c := &ChunkCache{
|
||||||
memCache: NewChunkCacheInMemory(maxEntries),
|
memCache: NewChunkCacheInMemory(maxEntries),
|
||||||
diskCache: NewOnDiskCacheLayer(dir, "cache", volumeCount, volumeSize),
|
|
||||||
}
|
}
|
||||||
|
c.diskCaches = make([]*OnDiskCacheLayer, 3)
|
||||||
|
c.diskCaches[0] = NewOnDiskCacheLayer(dir, "c0_1", diskSizeMB/4, 4)
|
||||||
|
c.diskCaches[1] = NewOnDiskCacheLayer(dir, "c1_4", diskSizeMB/4, 4)
|
||||||
|
c.diskCaches[2] = NewOnDiskCacheLayer(dir, "cache", diskSizeMB/2, 4)
|
||||||
|
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
@ -58,7 +58,14 @@ func (c *ChunkCache) doGetChunk(fileId string, chunkSize uint64) (data []byte) {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.diskCache.getChunk(fid.Key)
|
for _, diskCache := range c.diskCaches {
|
||||||
|
data := diskCache.getChunk(fid.Key)
|
||||||
|
if len(data) != 0 {
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -84,7 +91,13 @@ func (c *ChunkCache) doSetChunk(fileId string, data []byte) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
c.diskCache.setChunk(fid.Key, data)
|
if len(data) < onDiskCacheSizeLimit0 {
|
||||||
|
c.diskCaches[0].setChunk(fid.Key, data)
|
||||||
|
} else if len(data) < onDiskCacheSizeLimit1 {
|
||||||
|
c.diskCaches[1].setChunk(fid.Key, data)
|
||||||
|
} else {
|
||||||
|
c.diskCaches[2].setChunk(fid.Key, data)
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -94,5 +107,7 @@ func (c *ChunkCache) Shutdown() {
|
|||||||
}
|
}
|
||||||
c.Lock()
|
c.Lock()
|
||||||
defer c.Unlock()
|
defer c.Unlock()
|
||||||
c.diskCache.shutdown()
|
for _, diskCache := range c.diskCaches {
|
||||||
|
diskCache.shutdown()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -14,10 +14,9 @@ func TestOnDisk(t *testing.T) {
|
|||||||
tmpDir, _ := ioutil.TempDir("", "c")
|
tmpDir, _ := ioutil.TempDir("", "c")
|
||||||
defer os.RemoveAll(tmpDir)
|
defer os.RemoveAll(tmpDir)
|
||||||
|
|
||||||
totalDiskSizeMb := int64(6)
|
totalDiskSizeMb := int64(32)
|
||||||
segmentCount := 2
|
|
||||||
|
|
||||||
cache := NewChunkCache(0, tmpDir, totalDiskSizeMb, segmentCount)
|
cache := NewChunkCache(0, tmpDir, totalDiskSizeMb)
|
||||||
|
|
||||||
writeCount := 5
|
writeCount := 5
|
||||||
type test_data struct {
|
type test_data struct {
|
||||||
@ -46,7 +45,7 @@ func TestOnDisk(t *testing.T) {
|
|||||||
|
|
||||||
cache.Shutdown()
|
cache.Shutdown()
|
||||||
|
|
||||||
cache = NewChunkCache(0, tmpDir, totalDiskSizeMb, segmentCount)
|
cache = NewChunkCache(0, tmpDir, totalDiskSizeMb)
|
||||||
|
|
||||||
for i := 0; i < writeCount; i++ {
|
for i := 0; i < writeCount; i++ {
|
||||||
data := cache.GetChunk(testData[i].fileId, testData[i].size)
|
data := cache.GetChunk(testData[i].fileId, testData[i].size)
|
||||||
|
@ -14,7 +14,13 @@ type OnDiskCacheLayer struct {
|
|||||||
diskCaches []*ChunkCacheVolume
|
diskCaches []*ChunkCacheVolume
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewOnDiskCacheLayer(dir, namePrefix string, volumeCount int, volumeSize int64) *OnDiskCacheLayer{
|
func NewOnDiskCacheLayer(dir, namePrefix string, diskSizeMB int64, segmentCount int) *OnDiskCacheLayer{
|
||||||
|
|
||||||
|
volumeCount, volumeSize := int(diskSizeMB/30000), int64(30000)
|
||||||
|
if volumeCount < segmentCount {
|
||||||
|
volumeCount, volumeSize = segmentCount, diskSizeMB/int64(segmentCount)
|
||||||
|
}
|
||||||
|
|
||||||
c := &OnDiskCacheLayer{}
|
c := &OnDiskCacheLayer{}
|
||||||
for i := 0; i < volumeCount; i++ {
|
for i := 0; i < volumeCount; i++ {
|
||||||
fileName := path.Join(dir, fmt.Sprintf("%s_%d", namePrefix, i))
|
fileName := path.Join(dir, fmt.Sprintf("%s_%d", namePrefix, i))
|
||||||
|
Loading…
Reference in New Issue
Block a user