lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-21 07:17:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactoring to typed Size

Browse Source 
Go is amazing with refactoring!
This commit is contained in:
Chris Lu
2020-08-18 17:04:28 -07:00
parent 618b2f6829
commit 6a92f0bc7a
34 changed files with 118 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
weed/storage/needle_map/compact_map.go
View File

@@ -18,7 +18,7 @@ const SectionalNeedleIdLimit = 1<<32 - 1
type SectionalNeedleValue struct {
Key SectionalNeedleId
OffsetLower OffsetLower `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
Size uint32 `comment:"Size of the data portion"`
Size Size `comment:"Size of the data portion"`
}
type SectionalNeedleValueExtra struct {
@@ -50,7 +50,7 @@ func NewCompactSection(start NeedleId) *CompactSection {
}
//return old entry size
func (cs *CompactSection) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
func (cs *CompactSection) Set(key NeedleId, offset Offset, size Size) (oldOffset Offset, oldSize Size) {
cs.Lock()
if key > cs.end {
cs.end = key
@@ -80,7 +80,7 @@ func (cs *CompactSection) Set(key NeedleId, offset Offset, size uint32) (oldOffs
return
}
func (cs *CompactSection) setOverflowEntry(skey SectionalNeedleId, offset Offset, size uint32) {
func (cs *CompactSection) setOverflowEntry(skey SectionalNeedleId, offset Offset, size Size) {
needleValue := SectionalNeedleValue{Key: skey, OffsetLower: offset.OffsetLower, Size: size}
needleValueExtra := SectionalNeedleValueExtra{OffsetHigher: offset.OffsetHigher}
insertCandidate := sort.Search(len(cs.overflow), func(i int) bool {
@@ -125,10 +125,10 @@ func (cs *CompactSection) deleteOverflowEntry(key SectionalNeedleId) {
}
//return old entry size
func (cs *CompactSection) Delete(key NeedleId) uint32 {
func (cs *CompactSection) Delete(key NeedleId) Size {
skey := SectionalNeedleId(key - cs.start)
cs.Lock()
ret := uint32(0)
ret := Size(0)
if i := cs.binarySearchValues(skey); i >= 0 {
if cs.values[i].Size > 0 && cs.values[i].Size != TombstoneFileSize {
ret = cs.values[i].Size
@@ -181,7 +181,7 @@ func NewCompactMap() *CompactMap {
return &CompactMap{}
}
func (cm *CompactMap) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
func (cm *CompactMap) Set(key NeedleId, offset Offset, size Size) (oldOffset Offset, oldSize Size) {
x := cm.binarySearchCompactSection(key)
if x < 0 || (key-cm.list[x].start) > SectionalNeedleIdLimit {
// println(x, "adding to existing", len(cm.list), "sections, starting", key)
@@ -204,10 +204,10 @@ func (cm *CompactMap) Set(key NeedleId, offset Offset, size uint32) (oldOffset O
// println(key, "set to section[", x, "].start", cm.list[x].start)
return cm.list[x].Set(key, offset, size)
}
func (cm *CompactMap) Delete(key NeedleId) uint32 {
func (cm *CompactMap) Delete(key NeedleId) Size {
x := cm.binarySearchCompactSection(key)
if x < 0 {
return uint32(0)
return Size(0)
}
return cm.list[x].Delete(key)
}

3
weed/storage/needle_map/compact_map_perf_test.go
View File

@@ -9,7 +9,6 @@ import (
"time"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
/*
@@ -60,7 +59,7 @@ func loadNewNeedleMap(file *os.File) (*CompactMap, uint64) {
rowCount++
key := BytesToNeedleId(bytes[i : i+NeedleIdSize])
offset := BytesToOffset(bytes[i+NeedleIdSize : i+NeedleIdSize+OffsetSize])
size := util.BytesToUint32(bytes[i+NeedleIdSize+OffsetSize : i+NeedleIdSize+OffsetSize+SizeSize])
size := BytesToSize(bytes[i+NeedleIdSize+OffsetSize : i+NeedleIdSize+OffsetSize+SizeSize])
if !offset.IsZero() {
m.Set(NeedleId(key), offset, size)

10
weed/storage/needle_map/compact_map_test.go
View File

@@ -49,7 +49,7 @@ func TestIssue52(t *testing.T) {
func TestCompactMap(t *testing.T) {
m := NewCompactMap()
for i := uint32(0); i < 100*batch; i += 2 {
m.Set(NeedleId(i), ToOffset(int64(i)), i)
m.Set(NeedleId(i), ToOffset(int64(i)), Size(i))
}
for i := uint32(0); i < 100*batch; i += 37 {
@@ -57,7 +57,7 @@ func TestCompactMap(t *testing.T) {
}
for i := uint32(0); i < 10*batch; i += 3 {
m.Set(NeedleId(i), ToOffset(int64(i+11)), i+5)
m.Set(NeedleId(i), ToOffset(int64(i+11)), Size(i+5))
}
// for i := uint32(0); i < 100; i++ {
@@ -72,7 +72,7 @@ func TestCompactMap(t *testing.T) {
if !ok {
t.Fatal("key", i, "missing!")
}
if v.Size != i+5 {
if v.Size != Size(i+5) {
t.Fatal("key", i, "size", v.Size)
}
} else if i%37 == 0 {
@@ -80,7 +80,7 @@ func TestCompactMap(t *testing.T) {
t.Fatal("key", i, "should have been deleted needle value", v)
}
} else if i%2 == 0 {
if v.Size != i {
if v.Size != Size(i) {
t.Fatal("key", i, "size", v.Size)
}
}
@@ -96,7 +96,7 @@ func TestCompactMap(t *testing.T) {
if v == nil {
t.Fatal("key", i, "missing")
}
if v.Size != i {
if v.Size != Size(i) {
t.Fatal("key", i, "size", v.Size)
}
}

9
weed/storage/needle_map/memdb.go
View File

@@ -11,7 +11,6 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/idx"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
//This map uses in memory level db
@@ -32,7 +31,7 @@ func NewMemDb() *MemDb {
return t
}
func (cm *MemDb) Set(key NeedleId, offset Offset, size uint32) error {
func (cm *MemDb) Set(key NeedleId, offset Offset, size Size) error {
bytes := ToBytes(key, offset, size)
@@ -56,7 +55,7 @@ func (cm *MemDb) Get(key NeedleId) (*NeedleValue, bool) {
return nil, false
}
offset := BytesToOffset(data[0:OffsetSize])
size := util.BytesToUint32(data[OffsetSize : OffsetSize+SizeSize])
size := BytesToSize(data[OffsetSize : OffsetSize+SizeSize])
return &NeedleValue{Key: key, Offset: offset, Size: size}, true
}
@@ -67,7 +66,7 @@ func (cm *MemDb) AscendingVisit(visit func(NeedleValue) error) (ret error) {
key := BytesToNeedleId(iter.Key())
data := iter.Value()
offset := BytesToOffset(data[0:OffsetSize])
size := util.BytesToUint32(data[OffsetSize : OffsetSize+SizeSize])
size := BytesToSize(data[OffsetSize : OffsetSize+SizeSize])
needle := NeedleValue{Key: key, Offset: offset, Size: size}
ret = visit(needle)
@@ -105,7 +104,7 @@ func (cm *MemDb) LoadFromIdx(idxName string) (ret error) {
}
defer idxFile.Close()
return idx.WalkIndexFile(idxFile, func(key NeedleId, offset Offset, size uint32) error {
return idx.WalkIndexFile(idxFile, func(key NeedleId, offset Offset, size Size) error {
if offset.IsZero() || size == TombstoneFileSize {
return cm.Delete(key)
}

6
weed/storage/needle_map/needle_value.go
View File

@@ -9,7 +9,7 @@ import (
type NeedleValue struct {
Key NeedleId
Offset Offset `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
Size uint32 `comment:"Size of the data portion"`
Size Size `comment:"Size of the data portion"`
}
func (this NeedleValue) Less(than btree.Item) bool {
@@ -21,10 +21,10 @@ func (nv NeedleValue) ToBytes() []byte {
return ToBytes(nv.Key, nv.Offset, nv.Size)
}
func ToBytes(key NeedleId, offset Offset, size uint32) []byte {
func ToBytes(key NeedleId, offset Offset, size Size) []byte {
bytes := make([]byte, NeedleIdSize+OffsetSize+SizeSize)
NeedleIdToBytes(bytes[0:NeedleIdSize], key)
OffsetToBytes(bytes[NeedleIdSize:NeedleIdSize+OffsetSize], offset)
util.Uint32toBytes(bytes[NeedleIdSize+OffsetSize:NeedleIdSize+OffsetSize+SizeSize], size)
util.Uint32toBytes(bytes[NeedleIdSize+OffsetSize:NeedleIdSize+OffsetSize+SizeSize], uint32(size))
return bytes
}

4
weed/storage/needle_map/needle_value_map.go
View File

@@ -5,8 +5,8 @@ import (
)
type NeedleValueMap interface {
Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32)
Delete(key NeedleId) uint32
Set(key NeedleId, offset Offset, size Size) (oldOffset Offset, oldSize Size)
Delete(key NeedleId) Size
Get(key NeedleId) (*NeedleValue, bool)
AscendingVisit(visit func(NeedleValue) error) error
}