lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-15 20:06:19 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

del checks

Browse Source 
url err logging
stats chunk fetch
This commit is contained in:
Konstantin Lebedev
2021-05-21 15:59:12 +05:00
parent 42f631f549
commit 26a4f34a57
6 changed files with 17 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/filer/filer_on_meta_event.go
View File

@@ -52,7 +52,7 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR
func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
var buf bytes.Buffer
err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64, false)
err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64)
if err != nil {
return nil, err
}

2
weed/filer/read_write.go
View File

@@ -27,7 +27,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed
return err
}
return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
}

22
weed/filer/stream.go
View File

@@ -7,14 +7,16 @@ import (
"io"
"math"
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/wdclient"
)
func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64, isCheck bool) error {
func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks)
chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
@@ -34,31 +36,23 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
fileId2Url[chunkView.FileId] = urlStrings
}
if isCheck {
// Pre-check all chunkViews urls
gErr := new(errgroup.Group)
CheckAllChunkViews(chunkViews, &fileId2Url, gErr)
if err := gErr.Wait(); err != nil {
glog.Errorf("check all chunks: %v", err)
return fmt.Errorf("check all chunks: %v", err)
}
return nil
}
for _, chunkView := range chunkViews {
urlStrings := fileId2Url[chunkView.FileId]
start := time.Now()
data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
if err != nil {
glog.Errorf("read chunk: %v", err)
stats.FilerRequestCounter.WithLabelValues("chunkDownloadError").Inc()
return fmt.Errorf("read chunk: %v", err)
}
_, err = w.Write(data)
if err != nil {
glog.Errorf("write chunk: %v", err)
stats.FilerRequestCounter.WithLabelValues("chunkDownloadedError").Inc()
return fmt.Errorf("write chunk: %v", err)
}
stats.FilerRequestCounter.WithLabelValues("chunkDownload").Inc()
}
return nil