lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-02-09 09:17:28 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

randomly pick one replica to write

Browse Source
This commit is contained in:
chrislu
2024-08-27 09:02:39 -07:00
parent 62102d1afc
commit bebc385607
1 changed files with 24 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
weed/operation/submit.go
View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"io" "io"
"math/rand"
"mime" "mime"
"net/url" "net/url"
"os" "os"
@@ -179,11 +180,8 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
id += "_" + strconv.FormatInt(i, 10) id += "_" + strconv.FormatInt(i, 10)
} }
} }
fileUrl := "http://" + ret.Url + "/" + id fileUrl := genFileUrl(ret, id, usePublicUrl)
if usePublicUrl { count, e := uploadOneChunk(
fileUrl = "http://" + ret.PublicUrl + "/" + id
}
count, e := upload_one_chunk(
baseName+"-"+strconv.FormatInt(i+1, 10), baseName+"-"+strconv.FormatInt(i+1, 10),
io.LimitReader(fi.Reader, chunkSize), io.LimitReader(fi.Reader, chunkSize),
masterFn, fileUrl, masterFn, fileUrl,
@@ -202,7 +200,7 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
) )
retSize += count retSize += count
} }
err = upload_chunked_file_manifest(fileUrl, &cm, jwt) err = uploadChunkedFileManifest(fileUrl, &cm, jwt)
if err != nil { if err != nil {
// delete all uploaded chunks // delete all uploaded chunks
cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption) cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
@@ -217,7 +215,7 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
PairMap: nil, PairMap: nil,
Jwt: jwt, Jwt: jwt,
} }
uploader, e := NewUploader() uploader, e := NewUploader()
if e != nil { if e != nil {
return 0, e return 0, e
@@ -232,7 +230,23 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
return return
} }
func upload_one_chunk(filename string, reader io.Reader, masterFn GetMasterFn, func genFileUrl(ret *AssignResult, id string, usePublicUrl bool) string {
fileUrl := "http://" + ret.Url + "/" + id
if usePublicUrl {
fileUrl = "http://" + ret.PublicUrl + "/" + id
}
for _, replica := range ret.Replicas {
if rand.Intn(len(ret.Replicas)+1) == 0 {
fileUrl = "http://" + replica.Url + "/" + id
if usePublicUrl {
fileUrl = "http://" + replica.PublicUrl + "/" + id
}
}
}
return fileUrl
}
func uploadOneChunk(filename string, reader io.Reader, masterFn GetMasterFn,
fileUrl string, jwt security.EncodedJwt, fileUrl string, jwt security.EncodedJwt,
) (size uint32, e error) { ) (size uint32, e error) {
glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...") glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
@@ -258,7 +272,7 @@ func upload_one_chunk(filename string, reader io.Reader, masterFn GetMasterFn,
return uploadResult.Size, nil return uploadResult.Size, nil
} }
func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt security.EncodedJwt) error { func uploadChunkedFileManifest(fileUrl string, manifest *ChunkManifest, jwt security.EncodedJwt) error {
buf, e := manifest.Marshal() buf, e := manifest.Marshal()
if e != nil { if e != nil {
return e return e
@@ -277,7 +291,7 @@ func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt s
PairMap: nil, PairMap: nil,
Jwt: jwt, Jwt: jwt,
} }
uploader, e := NewUploader() uploader, e := NewUploader()
if e != nil { if e != nil {
return e return e