lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-07-18 07:30:22 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

tiered caching

Browse Source 
1/4 for small less than 1MB files. 1/4 for 1~4MB files, 1/2 for bigger than 4MB files
This commit is contained in:
Chris Lu 2020-04-13 22:19:27 -07:00
parent f282ed444b
commit 2b5c4fbbf3
5 changed files with 41 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
weed/filesys/wfs.go
View File

@ -84,7 +84,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
}, },
} }
if option.CacheSizeMB > 0 { if option.CacheSizeMB > 0 {
wfs.chunkCache = chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB, 4) wfs.chunkCache = chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB)
util.OnInterrupt(func() { util.OnInterrupt(func() {
wfs.chunkCache.Shutdown() wfs.chunkCache.Shutdown()
}) })

2
weed/server/webdav_server.go
View File

@ -99,7 +99,7 @@ type WebDavFile struct {
func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) { func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) {
chunkCache := chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB, 4) chunkCache := chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB)
util.OnInterrupt(func() { util.OnInterrupt(func() {
chunkCache.Shutdown() chunkCache.Shutdown()
}) })

43
weed/util/chunk_cache/chunk_cache.go
View File

@ -8,27 +8,27 @@ import (
) )
const ( const (
memCacheSizeLimit = 1024 * 1024 memCacheSizeLimit = 1024 * 1024
onDiskCacheSizeLimit0 = memCacheSizeLimit
onDiskCacheSizeLimit1 = 4 * memCacheSizeLimit
) )
// a global cache for recently accessed file chunks // a global cache for recently accessed file chunks
type ChunkCache struct { type ChunkCache struct {
memCache *ChunkCacheInMemory memCache *ChunkCacheInMemory
diskCache *OnDiskCacheLayer diskCaches []*OnDiskCacheLayer
sync.RWMutex sync.RWMutex
} }
func NewChunkCache(maxEntries int64, dir string, diskSizeMB int64, segmentCount int) *ChunkCache { func NewChunkCache(maxEntries int64, dir string, diskSizeMB int64) *ChunkCache {
volumeCount, volumeSize := int(diskSizeMB/30000), int64(30000)
if volumeCount < segmentCount {
volumeCount, volumeSize = segmentCount, diskSizeMB/int64(segmentCount)
}
c := &ChunkCache{ c := &ChunkCache{
memCache: NewChunkCacheInMemory(maxEntries), memCache: NewChunkCacheInMemory(maxEntries),
diskCache: NewOnDiskCacheLayer(dir, "cache", volumeCount, volumeSize),
} }
c.diskCaches = make([]*OnDiskCacheLayer, 3)
c.diskCaches[0] = NewOnDiskCacheLayer(dir, "c0_1", diskSizeMB/4, 4)
c.diskCaches[1] = NewOnDiskCacheLayer(dir, "c1_4", diskSizeMB/4, 4)
c.diskCaches[2] = NewOnDiskCacheLayer(dir, "cache", diskSizeMB/2, 4)
return c return c
} }
@ -58,7 +58,14 @@ func (c *ChunkCache) doGetChunk(fileId string, chunkSize uint64) (data []byte) {
return nil return nil
} }
return c.diskCache.getChunk(fid.Key) for _, diskCache := range c.diskCaches {
data := diskCache.getChunk(fid.Key)
if len(data) != 0 {
return data
}
}
return nil
} }
@ -84,7 +91,13 @@ func (c *ChunkCache) doSetChunk(fileId string, data []byte) {
return return
} }
c.diskCache.setChunk(fid.Key, data) if len(data) < onDiskCacheSizeLimit0 {
c.diskCaches[0].setChunk(fid.Key, data)
} else if len(data) < onDiskCacheSizeLimit1 {
c.diskCaches[1].setChunk(fid.Key, data)
} else {
c.diskCaches[2].setChunk(fid.Key, data)
}
} }
@ -94,5 +107,7 @@ func (c *ChunkCache) Shutdown() {
} }
c.Lock() c.Lock()
defer c.Unlock() defer c.Unlock()
c.diskCache.shutdown() for _, diskCache := range c.diskCaches {
diskCache.shutdown()
}
} }

7
weed/util/chunk_cache/chunk_cache_on_disk_test.go
View File

@ -14,10 +14,9 @@ func TestOnDisk(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "c") tmpDir, _ := ioutil.TempDir("", "c")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
totalDiskSizeMb := int64(6) totalDiskSizeMb := int64(32)
segmentCount := 2
cache := NewChunkCache(0, tmpDir, totalDiskSizeMb, segmentCount) cache := NewChunkCache(0, tmpDir, totalDiskSizeMb)
writeCount := 5 writeCount := 5
type test_data struct { type test_data struct {
@ -46,7 +45,7 @@ func TestOnDisk(t *testing.T) {
cache.Shutdown() cache.Shutdown()
cache = NewChunkCache(0, tmpDir, totalDiskSizeMb, segmentCount) cache = NewChunkCache(0, tmpDir, totalDiskSizeMb)
for i := 0; i < writeCount; i++ { for i := 0; i < writeCount; i++ {
data := cache.GetChunk(testData[i].fileId, testData[i].size) data := cache.GetChunk(testData[i].fileId, testData[i].size)

8
weed/util/chunk_cache/on_disk_cache_layer.go
View File

@ -14,7 +14,13 @@ type OnDiskCacheLayer struct {
diskCaches []*ChunkCacheVolume diskCaches []*ChunkCacheVolume
} }
func NewOnDiskCacheLayer(dir, namePrefix string, volumeCount int, volumeSize int64) *OnDiskCacheLayer{ func NewOnDiskCacheLayer(dir, namePrefix string, diskSizeMB int64, segmentCount int) *OnDiskCacheLayer{
volumeCount, volumeSize := int(diskSizeMB/30000), int64(30000)
if volumeCount < segmentCount {
volumeCount, volumeSize = segmentCount, diskSizeMB/int64(segmentCount)
}
c := &OnDiskCacheLayer{} c := &OnDiskCacheLayer{}
for i := 0; i < volumeCount; i++ { for i := 0; i < volumeCount; i++ {
fileName := path.Join(dir, fmt.Sprintf("%s_%d", namePrefix, i)) fileName := path.Join(dir, fmt.Sprintf("%s_%d", namePrefix, i))