lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-08-20 07:32:44 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

ensure metadata follow a specific folder

Browse Source 
fix https://github.com/seaweedfs/seaweedfs/issues/5774
This commit is contained in:
chrislu 2024-07-12 11:17:28 -07:00
parent ec9e7493b3
commit 3a82f5ffad
7 changed files with 40 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go
View File

@ -10,6 +10,7 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"strconv"
"strings"
"time"
)
@ -79,12 +80,17 @@ func startGenerateMetadata() {
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
prefix := *dir
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "tail",
ClientId: 0,
ClientEpoch: 0,
SelfSignature: 0,
PathPrefix: *dir,
PathPrefix: prefix,
AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil,
StartTsNs: 0,

8
weed/command/filer_backup.go
View File

@ -9,6 +9,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"regexp"
"strings"
"time"
)
@ -148,12 +149,17 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
}()
}
prefix := sourcePath
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "backup_" + dataSink.GetName(),
ClientId: clientId,
ClientEpoch: clientEpoch,
SelfSignature: 0,
PathPrefix: sourcePath,
PathPrefix: prefix,
AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil,
StartTsNs: startFrom.UnixNano(),

7
weed/command/filer_meta_backup.go
View File

@ -8,6 +8,7 @@ import (
"github.com/spf13/viper"
"google.golang.org/grpc"
"reflect"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
@ -197,12 +198,16 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
metaBackup.clientEpoch++
prefix := *metaBackup.filerDirectory
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "meta_backup",
ClientId: metaBackup.clientId,
ClientEpoch: metaBackup.clientEpoch,
SelfSignature: 0,
PathPrefix: *metaBackup.filerDirectory,
PathPrefix: prefix,
AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil,
StartTsNs: startTime.UnixNano(),

2
weed/command/filer_remote_gateway_buckets.go
View File

@ -55,7 +55,7 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo
ClientId: option.clientId,
ClientEpoch: option.clientEpoch,
SelfSignature: 0,
PathPrefix: option.bucketsDir,
PathPrefix: option.bucketsDir + "/",
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
DirectoriesToWatch: nil,
StartTsNs: lastOffsetTs.UnixNano(),

7
weed/command/filer_remote_sync_dir.go
View File

@ -64,12 +64,17 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
option.clientEpoch++
prefix := mountedDir
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "filer.remote.sync",
ClientId: option.clientId,
ClientEpoch: option.clientEpoch,
SelfSignature: 0,
PathPrefix: mountedDir,
PathPrefix: prefix,
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
DirectoriesToWatch: nil,
StartTsNs: lastOffsetTs.UnixNano(),

7
weed/command/filer_sync.go
View File

@ -296,12 +296,17 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
})
prefix := sourcePath
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{
ClientName: clientName,
ClientId: clientId,
ClientEpoch: clientEpoch,
SelfSignature: targetFilerSignature,
PathPrefix: sourcePath,
PathPrefix: prefix,
AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil,
StartTsNs: sourceFilerOffsetTsNs,

8
weed/mount/meta_cache/meta_cache_subscribe.go
View File

@ -7,6 +7,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"strings"
)
func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64) error {
@ -57,12 +58,17 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
}
prefix := dir
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "mount",
ClientId: selfSignature,
ClientEpoch: 1,
SelfSignature: selfSignature,
PathPrefix: dir,
PathPrefix: prefix,
AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil,
StartTsNs: lastTsNs,