lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-11-08 16:56:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

remote.cache and uncache: more flexible options to select files to cache or uncache

Browse Source
This commit is contained in:
Chris Lu
2021-08-14 15:11:52 -07:00
parent 0c671885bf
commit 708debca14
2 changed files with 89 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
weed/shell/command_remote_cache.go
View File

@@ -32,6 +32,12 @@ func (c *commandRemoteCache) Help() string {
# after mount, run one of these command to cache the content of the files
remote.cache -dir=xxx
remote.cache -dir=xxx/some/sub/dir
remote.cache -dir=xxx/some/sub/dir -include=*.pdf
This is designed to run regularly. So you can add it to some cronjob.
If a file is already synchronized with the remote copy, the file will be skipped to avoid unnecessary copy.
The actual data copying goes through volume severs.
`
}
@@ -41,6 +47,7 @@ func (c *commandRemoteCache) Do(args []string, commandEnv *CommandEnv, writer io
remoteMountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
dir := remoteMountCommand.String("dir", "", "a directory in filer")
fileFiler := newFileFilter(remoteMountCommand)
if err = remoteMountCommand.Parse(args); err != nil {
return nil
@@ -76,7 +83,7 @@ func (c *commandRemoteCache) Do(args []string, commandEnv *CommandEnv, writer io
}
// pull content from remote
if err = c.cacheContentData(commandEnv, writer, util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(*dir), remoteStorageConf); err != nil {
if err = c.cacheContentData(commandEnv, writer, util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(*dir), fileFiler, remoteStorageConf); err != nil {
return fmt.Errorf("cache content data: %v", err)
}
@@ -122,7 +129,7 @@ func mayHaveCachedToLocal(entry *filer_pb.Entry) bool {
return false
}
if entry.RemoteEntry == nil {
return false
return false // should not uncache an entry that is not in remote
}
if entry.RemoteEntry.LocalMtime > 0 && len(entry.Chunks) > 0 {
return true
@@ -130,13 +137,17 @@ func mayHaveCachedToLocal(entry *filer_pb.Entry) bool {
return false
}
func (c *commandRemoteCache) cacheContentData(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *filer_pb.RemoteStorageLocation, dirToCache util.FullPath, remoteConf *filer_pb.RemoteConf) error {
func (c *commandRemoteCache) cacheContentData(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *filer_pb.RemoteStorageLocation, dirToCache util.FullPath, fileFilter *FileFilter, remoteConf *filer_pb.RemoteConf) error {
return recursivelyTraverseDirectory(commandEnv, dirToCache, func(dir util.FullPath, entry *filer_pb.Entry) bool {
if !shouldCacheToLocal(entry) {
return true // true means recursive traversal should continue
}
if fileFilter.matches(entry) {
return true
}
println(dir, entry.Name)
remoteLocation := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, dir.Child(entry.Name))