lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-09-22 22:43:33 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

weed mount, weed dav add option to force cache

Browse Source
This commit is contained in:
chrislu
2024-09-04 01:19:14 -07:00
parent 66ac82bb8f
commit 7367b976b0
9 changed files with 22 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/command/mount.go
View File

@@ -20,6 +20,7 @@ type MountOptions struct {
cacheDirForRead *string
cacheDirForWrite *string
cacheSizeMBForRead *int64
forceCache *bool
dataCenter *string
allowOthers *bool
umaskString *string
@@ -58,6 +59,7 @@ func init() {
mountOptions.cacheDirForRead = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
mountOptions.cacheSizeMBForRead = cmdMount.Flag.Int64("cacheCapacityMB", 0, "file chunk read cache capacity in MB")
mountOptions.cacheDirForWrite = cmdMount.Flag.String("cacheDirWrite", "", "buffer writes mostly for large files")
mountOptions.forceCache = cmdMount.Flag.Bool("forceCache", true, "force to cache all reads")
mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")

1
weed/command/mount_std.go
View File

@@ -235,6 +235,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
CacheDirForRead: *option.cacheDirForRead,
CacheSizeMBForRead: *option.cacheSizeMBForRead,
CacheDirForWrite: cacheDirForWrite,
ForceCache: *option.forceCache,
DataCenter: *option.dataCenter,
Quota: int64(*option.collectionQuota) * 1024 * 1024,
MountUid: uid,

3
weed/command/webdav.go
View File

@@ -32,6 +32,7 @@ type WebDavOption struct {
tlsCertificate *string
cacheDir *string
cacheSizeMB *int64
forceCache *bool
maxMB *int
}
@@ -46,6 +47,7 @@ func init() {
webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file")
webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 0, "local cache capacity in MB")
webDavStandaloneOptions.forceCache = cmdWebDav.Flag.Bool("forceCache", false, "force to cache reads to local disk")
webDavStandaloneOptions.maxMB = cmdWebDav.Flag.Int("maxMB", 4, "split files larger than the limit")
webDavStandaloneOptions.filerRootPath = cmdWebDav.Flag.String("filer.path", "/", "use this remote path from filer server")
}
@@ -118,6 +120,7 @@ func (wo *WebDavOption) startWebDav() bool {
Cipher: cipher,
CacheDir: util.ResolvePath(*wo.cacheDir),
CacheSizeMB: *wo.cacheSizeMB,
ForceCache: *wo.forceCache,
MaxMB: *wo.maxMB,
})
if webdavServer_err != nil {

5
weed/filer/reader_at.go
View File

@@ -19,6 +19,7 @@ type ChunkReadAt struct {
fileSize int64
readerCache *ReaderCache
readerPattern *ReaderPattern
forceCache bool
lastChunkFid string
}
@@ -196,7 +197,9 @@ func (c *ChunkReadAt) readChunkSliceAt(buffer []byte, chunkView *ChunkView, next
if n > 0 {
return n, err
}
return fetchChunkRange(buffer, c.readerCache.lookupFileIdFn, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset))
if !c.forceCache {
return fetchChunkRange(buffer, c.readerCache.lookupFileIdFn, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset))
}
}
n, err = c.readerCache.ReadChunkAt(buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), chunkView.ViewOffset == 0)

4
weed/filer/reader_cache.go
View File

@@ -7,9 +7,9 @@ import (
"time"
"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
"github.com/seaweedfs/seaweedfs/weed/util/mem"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
type ReaderCache struct {
@@ -69,7 +69,7 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView]) {
// glog.V(4).Infof("prefetch %s offset %d", chunkView.FileId, chunkView.ViewOffset)
// cache this chunk if not yet
cacher := newSingleChunkCacher(rc, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int(chunkView.ChunkSize), false)
cacher := newSingleChunkCacher(rc, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int(chunkView.ChunkSize), chunkView.ViewOffset == 0)
go cacher.startCaching()
<-cacher.cacheStartedCh
rc.downloaders[chunkView.FileId] = cacher

3
weed/mount/weedfs.go
View File

@@ -41,6 +41,7 @@ type Option struct {
CacheDirForRead string
CacheSizeMBForRead int64
CacheDirForWrite string
ForceCache bool
DataCenter string
Umask os.FileMode
Quota int64
@@ -95,7 +96,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
wfs.option.filerIndex = int32(rand.Intn(len(option.FilerAddresses)))
wfs.option.setupUniqueCacheDirectory()
if option.CacheSizeMBForRead > 0 {
wfs.chunkCache = chunk_cache.NewTieredChunkCache(256, option.getUniqueCacheDirForRead(), option.CacheSizeMBForRead, 1024*1024)
wfs.chunkCache = chunk_cache.NewTieredChunkCache(256, option.getUniqueCacheDirForRead(), option.CacheSizeMBForRead, 1024*1024, option.ForceCache)
}
wfs.metaCache = meta_cache.NewMetaCache(path.Join(option.getUniqueCacheDirForRead(), "meta"), option.UidGidMapper,

3
weed/server/webdav_server.go
View File

@@ -38,6 +38,7 @@ type WebDavOption struct {
Cipher bool
CacheDir string
CacheSizeMB int64
ForceCache bool
MaxMB int
}
@@ -133,7 +134,7 @@ func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) {
cacheDir := path.Join(option.CacheDir, cacheUniqueId)
os.MkdirAll(cacheDir, os.FileMode(0755))
chunkCache := chunk_cache.NewTieredChunkCache(256, cacheDir, option.CacheSizeMB, 1024*1024)
chunkCache := chunk_cache.NewTieredChunkCache(256, cacheDir, option.CacheSizeMB, 1024*1024, option.ForceCache)
t := &WebDavFileSystem{
option: option,
chunkCache: chunkCache,

6
weed/util/chunk_cache/chunk_cache.go
View File

@@ -23,14 +23,16 @@ type TieredChunkCache struct {
onDiskCacheSizeLimit0 uint64
onDiskCacheSizeLimit1 uint64
onDiskCacheSizeLimit2 uint64
forceCache bool
}
var _ ChunkCache = &TieredChunkCache{}
func NewTieredChunkCache(maxEntries int64, dir string, diskSizeInUnit int64, unitSize int64) *TieredChunkCache {
func NewTieredChunkCache(maxEntries int64, dir string, diskSizeInUnit int64, unitSize int64, forceCache bool) *TieredChunkCache {
c := &TieredChunkCache{
memCache: NewChunkCacheInMemory(maxEntries),
memCache: NewChunkCacheInMemory(maxEntries),
forceCache: forceCache,
}
c.diskCaches = make([]*OnDiskCacheLayer, 3)
c.onDiskCacheSizeLimit0 = uint64(unitSize)

4
weed/util/chunk_cache/chunk_cache_on_disk_test.go
View File

@@ -13,7 +13,7 @@ func TestOnDisk(t *testing.T) {
totalDiskSizeInKB := int64(32)
cache := NewTieredChunkCache(2, tmpDir, totalDiskSizeInKB, 1024)
cache := NewTieredChunkCache(2, tmpDir, totalDiskSizeInKB, 1024, false)
writeCount := 5
type test_data struct {
@@ -61,7 +61,7 @@ func TestOnDisk(t *testing.T) {
cache.Shutdown()
cache = NewTieredChunkCache(2, tmpDir, totalDiskSizeInKB, 1024)
cache = NewTieredChunkCache(2, tmpDir, totalDiskSizeInKB, 1024, false)
for i := 0; i < 2; i++ {
data := mem.Allocate(testData[i].size)