lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-05-03 04:37:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor

Browse Source
This commit is contained in:
Chris Lu 2021-09-02 07:07:16 -07:00
parent e281f0fa82
commit 9a73b0e3c9
1 changed files with 23 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
weed/command/filer_remote_sync.go
View File

@ -107,11 +107,27 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
lastOffsetTs := collectLastSyncOffset(option, mountedDir) lastOffsetTs := collectLastSyncOffset(option, mountedDir)
client, err := remote_storage.GetRemoteStorage(remoteStorage) eachEntryFunc, err := makeEventProcessor(remoteStorage, mountedDir, remoteStorageMountLocation, filerSource)
if err != nil { if err != nil {
return err return err
} }
processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
lastTime := time.Unix(0, lastTsNs)
glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
return remote_storage.SetSyncOffset(option.grpcDialOption, *option.filerAddress, mountedDir, lastTsNs)
})
return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, "filer.remote.sync",
mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
}
func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
client, err := remote_storage.GetRemoteStorage(remoteStorage)
if err != nil {
return nil, err
}
handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error { handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
message := resp.EventNotification message := resp.EventNotification
if message.NewEntry == nil { if message.NewEntry == nil {
@ -137,8 +153,11 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err) return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
} }
remoteStorage = conf remoteStorage = conf
client, err = remote_storage.GetRemoteStorage(remoteStorage) if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
return err client = newClient
} else {
return err
}
} }
return nil return nil
@ -217,15 +236,7 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
return nil return nil
} }
return eachEntryFunc, nil
processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
lastTime := time.Unix(0, lastTsNs)
glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
return remote_storage.SetSyncOffset(option.grpcDialOption, *option.filerAddress, mountedDir, lastTsNs)
})
return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, "filer.remote.sync",
mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
} }
func collectLastSyncOffset(option *RemoteSyncOptions, mountedDir string) (time.Time) { func collectLastSyncOffset(option *RemoteSyncOptions, mountedDir string) (time.Time) {