lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-07-15 21:04:08 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallelize remote content fetching

Browse Source
This commit is contained in:
Chris Lu 2021-08-14 15:41:13 -07:00
parent 00ffbb4c9a
commit f365af81c2
1 changed files with 63 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
weed/server/filer_grpc_server_remote.go
View File

@ -78,25 +78,32 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo
dest := util.FullPath(remoteStorageMountedLocation.Path).Child(string(util.FullPath(req.Directory).Child(req.Name))[len(localMountedDir):])
var chunks []*filer_pb.FileChunk
var fetchAndWriteErr error
// FIXME limit on parallel
limitedConcurrentExecutor := util.NewLimitedConcurrentExecutor(8)
for offset := int64(0); offset < entry.Remote.RemoteSize; offset += chunkSize {
localOffset := offset
limitedConcurrentExecutor.Execute(func() {
size := chunkSize
if offset+chunkSize > entry.Remote.RemoteSize {
size = entry.Remote.RemoteSize - offset
if localOffset+chunkSize > entry.Remote.RemoteSize {
size = entry.Remote.RemoteSize - localOffset
}
// assign one volume server
assignResult, err := operation.Assign(fs.filer.GetMaster, fs.grpcDialOption, assignRequest, altRequest)
if err != nil {
return resp, err
fetchAndWriteErr = err
return
}
if assignResult.Error != "" {
return resp, fmt.Errorf("assign: %v", assignResult.Error)
fetchAndWriteErr = fmt.Errorf("assign: %v", assignResult.Error)
return
}
fileId, parseErr := needle.ParseFileIdFromString(assignResult.Fid)
if assignResult.Error != "" {
return resp, fmt.Errorf("unrecognized file id %s: %v", assignResult.Fid, parseErr)
fetchAndWriteErr = fmt.Errorf("unrecognized file id %s: %v", assignResult.Fid, parseErr)
return
}
// tell filer to tell volume server to download into needles
@ -105,7 +112,7 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo
VolumeId: uint32(fileId.VolumeId),
NeedleId: uint64(fileId.Key),
Cookie: uint32(fileId.Cookie),
Offset: offset,
Offset: localOffset,
Size: size,
RemoteType: storageConf.Type,
RemoteName: storageConf.Name,
@ -123,12 +130,13 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo
})
if err != nil {
return nil, err
fetchAndWriteErr = err
return
}
chunks = append(chunks, &filer_pb.FileChunk{
FileId: assignResult.Fid,
Offset: offset,
Offset: localOffset,
Size: uint64(size),
Mtime: time.Now().Unix(),
Fid: &filer_pb.FileId{
@ -137,7 +145,7 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo
Cookie: uint32(fileId.Cookie),
},
})
})
}
garbage := entry.Chunks