lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-22 03:31:49 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

speed up leveldb bolddb loading

Browse Source
This commit is contained in:
Chris Lu
2018-07-24 22:17:56 -07:00
parent ea8ecf35f4
commit 57ded4fd80
1 changed files with 23 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
weed/storage/needle_map_metric.go
View File

@@ -2,7 +2,6 @@ package storage
import ( import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
. "github.com/chrislusf/seaweedfs/weed/storage/types" . "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/willf/bloom" "github.com/willf/bloom"
"os" "os"
@@ -91,16 +90,32 @@ func reverseWalkIndexFile(r *os.File, initFn func(entryCount int64), fn func(key
return fmt.Errorf("unexpected file %s size: %d", r.Name(), fileSize) return fmt.Errorf("unexpected file %s size: %d", r.Name(), fileSize)
} }
initFn(fileSize / NeedleEntrySize) entryCount := fileSize / NeedleEntrySize
initFn(entryCount)
bytes := make([]byte, NeedleEntrySize) batchSize := int64(1024 * 4)
for readerOffset := fileSize - NeedleEntrySize; readerOffset >= 0; readerOffset -= NeedleEntrySize {
count, e := r.ReadAt(bytes, readerOffset) bytes := make([]byte, NeedleEntrySize*batchSize)
glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e) nextBatchSize := entryCount % batchSize
key, offset, size := IdxFileEntry(bytes) if nextBatchSize == 0 {
if e = fn(key, offset, size); e != nil { nextBatchSize = batchSize
}
remainingCount := entryCount - nextBatchSize
for remainingCount >= 0 {
_, e := r.ReadAt(bytes[:NeedleEntrySize*nextBatchSize], NeedleEntrySize*remainingCount)
// glog.V(0).Infoln("file", r.Name(), "readerOffset", NeedleEntrySize*remainingCount, "count", count, "e", e)
if e != nil {
return e return e
} }
for i := int(nextBatchSize) - 1; i >= 0; i-- {
key, offset, size := IdxFileEntry(bytes[i*NeedleEntrySize:i*NeedleEntrySize+NeedleEntrySize])
if e = fn(key, offset, size); e != nil {
return e
}
}
nextBatchSize = batchSize
remainingCount -= nextBatchSize
} }
return nil return nil
} }