lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-21 10:57:24 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactoring

Browse Source
This commit is contained in:
Chris Lu
2019-04-18 21:43:36 -07:00
parent 33c92b819a
commit e5506152c0
72 changed files with 384 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
weed/storage/disk_location.go
View File

@@ -9,22 +9,23 @@ import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
type DiskLocation struct {
Directory string
MaxVolumeCount int
volumes map[VolumeId]*Volume
volumes map[needle.VolumeId]*Volume
sync.RWMutex
}
func NewDiskLocation(dir string, maxVolumeCount int) *DiskLocation {
location := &DiskLocation{Directory: dir, MaxVolumeCount: maxVolumeCount}
location.volumes = make(map[VolumeId]*Volume)
location.volumes = make(map[needle.VolumeId]*Volume)
return location
}
func (l *DiskLocation) volumeIdFromPath(dir os.FileInfo) (VolumeId, string, error) {
func (l *DiskLocation) volumeIdFromPath(dir os.FileInfo) (needle.VolumeId, string, error) {
name := dir.Name()
if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
collection := ""
@@ -33,7 +34,7 @@ func (l *DiskLocation) volumeIdFromPath(dir os.FileInfo) (VolumeId, string, erro
if i > 0 {
collection, base = base[0:i], base[i+1:]
}
vol, err := NewVolumeId(base)
vol, err := needle.NewVolumeId(base)
return vol, collection, err
}
@@ -114,7 +115,7 @@ func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e er
return
}
func (l *DiskLocation) deleteVolumeById(vid VolumeId) (e error) {
func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (e error) {
v, ok := l.volumes[vid]
if !ok {
return
@@ -127,7 +128,7 @@ func (l *DiskLocation) deleteVolumeById(vid VolumeId) (e error) {
return
}
func (l *DiskLocation) LoadVolume(vid VolumeId, needleMapKind NeedleMapType) bool {
func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapType) bool {
if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
for _, dir := range dirs {
volId, _, err := l.volumeIdFromPath(dir)
@@ -142,7 +143,7 @@ func (l *DiskLocation) LoadVolume(vid VolumeId, needleMapKind NeedleMapType) boo
return false
}
func (l *DiskLocation) DeleteVolume(vid VolumeId) error {
func (l *DiskLocation) DeleteVolume(vid needle.VolumeId) error {
l.Lock()
defer l.Unlock()
@@ -153,7 +154,7 @@ func (l *DiskLocation) DeleteVolume(vid VolumeId) error {
return l.deleteVolumeById(vid)
}
func (l *DiskLocation) UnloadVolume(vid VolumeId) error {
func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error {
l.Lock()
defer l.Unlock()
@@ -166,14 +167,14 @@ func (l *DiskLocation) UnloadVolume(vid VolumeId) error {
return nil
}
func (l *DiskLocation) SetVolume(vid VolumeId, volume *Volume) {
func (l *DiskLocation) SetVolume(vid needle.VolumeId, volume *Volume) {
l.Lock()
defer l.Unlock()
l.volumes[vid] = volume
}
func (l *DiskLocation) FindVolume(vid VolumeId) (*Volume, bool) {
func (l *DiskLocation) FindVolume(vid needle.VolumeId) (*Volume, bool) {
l.RLock()
defer l.RUnlock()

2
weed/storage/crc.go → weed/storage/needle/crc.go
View File

@@ -1,4 +1,4 @@
package storage
package needle
import (
"crypto/md5"

3
weed/storage/file_id.go → weed/storage/needle/file_id.go
View File

@@ -1,7 +1,8 @@
package storage
package needle
import (
"encoding/hex"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
)

6
weed/storage/needle.go → weed/storage/needle/needle.go
View File

@@ -1,4 +1,4 @@
package storage
package needle
import (
"encoding/json"
@@ -8,9 +8,10 @@ import (
"strings"
"time"
"io/ioutil"
"github.com/chrislusf/seaweedfs/weed/images"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"io/ioutil"
)
const (
@@ -187,3 +188,4 @@ func ParseNeedleIdCookie(key_hash_string string) (NeedleId, Cookie, error) {
func (n *Needle) LastModifiedString() string {
return time.Unix(int64(n.LastModified), 0).Format("2006-01-02T15:04:05")
}

11
weed/storage/needle_parse_multipart.go → weed/storage/needle/needle_parse_multipart.go
View File

@@ -1,8 +1,9 @@
package storage
package needle
import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/util"
"io"
"io/ioutil"
"mime"
@@ -90,12 +91,12 @@ func parseMultipart(r *http.Request) (
}
if part.Header.Get("Content-Encoding") == "gzip" {
if unzipped, e := operation.UnGzipData(data); e == nil {
if unzipped, e := util.UnGzipData(data); e == nil {
originalDataSize = len(unzipped)
}
isGzipped = true
} else if operation.IsGzippable(ext, mtype, data) {
if compressedData, err := operation.GzipData(data); err == nil {
} else if util.IsGzippable(ext, mtype, data) {
if compressedData, err := util.GzipData(data); err == nil {
if len(data) > len(compressedData) {
data = compressedData
isGzipped = true

10
weed/storage/needle_read_write.go → weed/storage/needle/needle_read_write.go
View File

@@ -1,4 +1,4 @@
package storage
package needle
import (
"errors"
@@ -6,10 +6,11 @@ import (
"io"
"os"
"math"
"github.com/chrislusf/seaweedfs/weed/glog"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
"math"
)
const (
@@ -383,3 +384,8 @@ func (n *Needle) HasPairs() bool {
func (n *Needle) SetHasPairs() {
n.Flags = n.Flags | FlagHasPairs
}
func getActualSize(size uint32, version Version) int64 {
return NeedleEntrySize + NeedleBodyLength(size, version)
}

5
weed/storage/needle_read_write_test.go → weed/storage/needle/needle_read_write_test.go
View File

@@ -1,10 +1,11 @@
package storage
package needle
import (
"github.com/chrislusf/seaweedfs/weed/storage/types"
"io/ioutil"
"os"
"testing"
"github.com/chrislusf/seaweedfs/weed/storage/types"
)
func TestAppend(t *testing.T) {

5
weed/storage/needle_test.go → weed/storage/needle/needle_test.go
View File

@@ -1,8 +1,9 @@
package storage
package needle
import (
"github.com/chrislusf/seaweedfs/weed/storage/types"
"testing"
"github.com/chrislusf/seaweedfs/weed/storage/types"
)
func TestParseKeyHash(t *testing.T) {

2
weed/storage/volume_id.go → weed/storage/needle/volume_id.go
View File

@@ -1,4 +1,4 @@
package storage
package needle
import (
"strconv"

2
weed/storage/volume_ttl.go → weed/storage/needle/volume_ttl.go
View File

@@ -1,4 +1,4 @@
package storage
package needle
import (
"strconv"

2
weed/storage/volume_ttl_test.go → weed/storage/needle/volume_ttl_test.go
View File

@@ -1,4 +1,4 @@
package storage
package needle
import (
"testing"

2
weed/storage/volume_version.go → weed/storage/needle/volume_version.go
View File

@@ -1,4 +1,4 @@
package storage
package needle
type Version uint8

11
weed/storage/needle_byte_cache.go
View File

@@ -1,11 +0,0 @@
package storage
import (
"os"
)
func getBytesForFileBlock(r *os.File, offset int64, readSize int) (dataSlice []byte, err error) {
dataSlice = make([]byte, readSize)
_, err = r.ReadAt(dataSlice, offset)
return dataSlice, err
}

4
weed/storage/needle_map.go
View File

@@ -6,7 +6,7 @@ import (
"os"
"sync"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
@@ -22,7 +22,7 @@ const (
type NeedleMapper interface {
Put(key NeedleId, offset Offset, size uint32) error
Get(key NeedleId) (element *needle.NeedleValue, ok bool)
Get(key NeedleId) (element *needle_map.NeedleValue, ok bool)
Delete(key NeedleId, offset Offset) error
Close()
Destroy() error

2
weed/storage/needle/btree_map.go → weed/storage/needle_map/btree_map.go
View File

@@ -1,4 +1,4 @@
package needle
package needle_map
import (
. "github.com/chrislusf/seaweedfs/weed/storage/types"

2
weed/storage/needle/compact_map.go → weed/storage/needle_map/compact_map.go
View File

@@ -1,4 +1,4 @@
package needle
package needle_map
import (
. "github.com/chrislusf/seaweedfs/weed/storage/types"

2
weed/storage/needle/compact_map_perf_test.go → weed/storage/needle_map/compact_map_perf_test.go
View File

@@ -1,4 +1,4 @@
package needle
package needle_map
import (
"fmt"

2
weed/storage/needle/compact_map_test.go → weed/storage/needle_map/compact_map_test.go
View File

@@ -1,4 +1,4 @@
package needle
package needle_map
import (
"fmt"

2
weed/storage/needle/needle_value.go → weed/storage/needle_map/needle_value.go
View File

@@ -1,4 +1,4 @@
package needle
package needle_map
import (
. "github.com/chrislusf/seaweedfs/weed/storage/types"

2
weed/storage/needle/needle_value_map.go → weed/storage/needle_map/needle_value_map.go
View File

@@ -1,4 +1,4 @@
package needle
package needle_map
import (
. "github.com/chrislusf/seaweedfs/weed/storage/types"

6
weed/storage/needle_map_leveldb.go
View File

@@ -7,7 +7,7 @@ import (
"path/filepath"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/syndtr/goleveldb/leveldb"
@@ -74,7 +74,7 @@ func generateLevelDbFile(dbFileName string, indexFile *os.File) error {
})
}
func (m *LevelDbNeedleMap) Get(key NeedleId) (element *needle.NeedleValue, ok bool) {
func (m *LevelDbNeedleMap) Get(key NeedleId) (element *needle_map.NeedleValue, ok bool) {
bytes := make([]byte, NeedleIdSize)
NeedleIdToBytes(bytes[0:NeedleIdSize], key)
data, err := m.db.Get(bytes, nil)
@@ -83,7 +83,7 @@ func (m *LevelDbNeedleMap) Get(key NeedleId) (element *needle.NeedleValue, ok bo
}
offset := BytesToOffset(data[0:OffsetSize])
size := util.BytesToUint32(data[OffsetSize : OffsetSize+SizeSize])
return &needle.NeedleValue{Key: NeedleId(key), Offset: offset, Size: size}, true
return &needle_map.NeedleValue{Key: NeedleId(key), Offset: offset, Size: size}, true
}
func (m *LevelDbNeedleMap) Put(key NeedleId, offset Offset, size uint32) error {

10
weed/storage/needle_map_memory.go
View File

@@ -5,19 +5,19 @@ import (
"os"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
)
type NeedleMap struct {
m needle.NeedleValueMap
m needle_map.NeedleValueMap
baseNeedleMapper
}
func NewCompactNeedleMap(file *os.File) *NeedleMap {
nm := &NeedleMap{
m: needle.NewCompactMap(),
m: needle_map.NewCompactMap(),
}
nm.indexFile = file
return nm
@@ -25,7 +25,7 @@ func NewCompactNeedleMap(file *os.File) *NeedleMap {
func NewBtreeNeedleMap(file *os.File) *NeedleMap {
nm := &NeedleMap{
m: needle.NewBtreeMap(),
m: needle_map.NewBtreeMap(),
}
nm.indexFile = file
return nm
@@ -106,7 +106,7 @@ func (nm *NeedleMap) Put(key NeedleId, offset Offset, size uint32) error {
nm.logPut(key, oldSize, size)
return nm.appendToIndexFile(key, offset, size)
}
func (nm *NeedleMap) Get(key NeedleId) (element *needle.NeedleValue, ok bool) {
func (nm *NeedleMap) Get(key NeedleId) (element *needle_map.NeedleValue, ok bool) {
element, ok = nm.m.Get(NeedleId(key))
return
}

41
weed/storage/store.go
View File

@@ -6,6 +6,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
)
@@ -27,8 +28,8 @@ type Store struct {
volumeSizeLimit uint64 //read from the master
Client master_pb.Seaweed_SendHeartbeatClient
NeedleMapType NeedleMapType
NewVolumeIdChan chan VolumeId
DeletedVolumeIdChan chan VolumeId
NewVolumeIdChan chan needle.VolumeId
DeletedVolumeIdChan chan needle.VolumeId
}
func (s *Store) String() (str string) {
@@ -44,16 +45,16 @@ func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts
location.loadExistingVolumes(needleMapKind)
s.Locations = append(s.Locations, location)
}
s.NewVolumeIdChan = make(chan VolumeId, 3)
s.DeletedVolumeIdChan = make(chan VolumeId, 3)
s.NewVolumeIdChan = make(chan needle.VolumeId, 3)
s.DeletedVolumeIdChan = make(chan needle.VolumeId, 3)
return
}
func (s *Store) AddVolume(volumeId VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64) error {
func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64) error {
rt, e := NewReplicaPlacementFromString(replicaPlacement)
if e != nil {
return e
}
ttl, e := ReadTTL(ttlString)
ttl, e := needle.ReadTTL(ttlString)
if e != nil {
return e
}
@@ -71,7 +72,7 @@ func (s *Store) DeleteCollection(collection string) (e error) {
return
}
func (s *Store) findVolume(vid VolumeId) *Volume {
func (s *Store) findVolume(vid needle.VolumeId) *Volume {
for _, location := range s.Locations {
if v, found := location.FindVolume(vid); found {
return v
@@ -90,7 +91,7 @@ func (s *Store) FindFreeLocation() (ret *DiskLocation) {
}
return ret
}
func (s *Store) addVolume(vid VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL, preallocate int64) error {
func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64) error {
if s.findVolume(vid) != nil {
return fmt.Errorf("Volume Id %d already exists!", vid)
}
@@ -114,7 +115,7 @@ func (s *Store) Status() []*VolumeInfo {
location.RLock()
for k, v := range location.volumes {
s := &VolumeInfo{
Id: VolumeId(k),
Id: needle.VolumeId(k),
Size: v.ContentSize(),
Collection: v.Collection,
ReplicaPlacement: v.ReplicaPlacement,
@@ -184,7 +185,7 @@ func (s *Store) Close() {
}
}
func (s *Store) Write(i VolumeId, n *Needle) (size uint32, err error) {
func (s *Store) Write(i needle.VolumeId, n *needle.Needle) (size uint32, err error) {
if v := s.findVolume(i); v != nil {
if v.readOnly {
err = fmt.Errorf("Volume %d is read only", i)
@@ -203,32 +204,32 @@ func (s *Store) Write(i VolumeId, n *Needle) (size uint32, err error) {
return
}
func (s *Store) Delete(i VolumeId, n *Needle) (uint32, error) {
func (s *Store) Delete(i needle.VolumeId, n *needle.Needle) (uint32, error) {
if v := s.findVolume(i); v != nil && !v.readOnly {
return v.deleteNeedle(n)
}
return 0, nil
}
func (s *Store) ReadVolumeNeedle(i VolumeId, n *Needle) (int, error) {
func (s *Store) ReadVolumeNeedle(i needle.VolumeId, n *needle.Needle) (int, error) {
if v := s.findVolume(i); v != nil {
return v.readNeedle(n)
}
return 0, fmt.Errorf("Volume %d not found!", i)
}
func (s *Store) GetVolume(i VolumeId) *Volume {
func (s *Store) GetVolume(i needle.VolumeId) *Volume {
return s.findVolume(i)
}
func (s *Store) HasVolume(i VolumeId) bool {
func (s *Store) HasVolume(i needle.VolumeId) bool {
v := s.findVolume(i)
return v != nil
}
func (s *Store) MountVolume(i VolumeId) error {
func (s *Store) MountVolume(i needle.VolumeId) error {
for _, location := range s.Locations {
if found := location.LoadVolume(i, s.NeedleMapType); found == true {
s.NewVolumeIdChan <- VolumeId(i)
s.NewVolumeIdChan <- needle.VolumeId(i)
return nil
}
}
@@ -236,10 +237,10 @@ func (s *Store) MountVolume(i VolumeId) error {
return fmt.Errorf("Volume %d not found on disk", i)
}
func (s *Store) UnmountVolume(i VolumeId) error {
func (s *Store) UnmountVolume(i needle.VolumeId) error {
for _, location := range s.Locations {
if err := location.UnloadVolume(i); err == nil {
s.DeletedVolumeIdChan <- VolumeId(i)
s.DeletedVolumeIdChan <- needle.VolumeId(i)
return nil
}
}
@@ -247,10 +248,10 @@ func (s *Store) UnmountVolume(i VolumeId) error {
return fmt.Errorf("Volume %d not found on disk", i)
}
func (s *Store) DeleteVolume(i VolumeId) error {
func (s *Store) DeleteVolume(i needle.VolumeId) error {
for _, location := range s.Locations {
if error := location.deleteVolumeById(i); error == nil {
s.DeletedVolumeIdChan <- VolumeId(i)
s.DeletedVolumeIdChan <- needle.VolumeId(i)
return nil
}
}

10
weed/storage/store_vacuum.go
View File

@@ -2,29 +2,31 @@ package storage
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
func (s *Store) CheckCompactVolume(volumeId VolumeId) (float64, error) {
func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (float64, error) {
if v := s.findVolume(volumeId); v != nil {
glog.V(3).Infof("volumd %d garbage level: %f", volumeId, v.garbageLevel())
return v.garbageLevel(), nil
}
return 0, fmt.Errorf("volume id %d is not found during check compact", volumeId)
}
func (s *Store) CompactVolume(vid VolumeId, preallocate int64) error {
func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64) error {
if v := s.findVolume(vid); v != nil {
return v.Compact(preallocate)
}
return fmt.Errorf("volume id %d is not found during compact", vid)
}
func (s *Store) CommitCompactVolume(vid VolumeId) error {
func (s *Store) CommitCompactVolume(vid needle.VolumeId) error {
if v := s.findVolume(vid); v != nil {
return v.CommitCompact()
}
return fmt.Errorf("volume id %d is not found during commit compact", vid)
}
func (s *Store) CommitCleanupVolume(vid VolumeId) error {
func (s *Store) CommitCleanupVolume(vid needle.VolumeId) error {
if v := s.findVolume(vid); v != nil {
return v.cleanupCompact()
}

78
weed/storage/tail_volume.go.go
View File

@@ -1,78 +0,0 @@
package storage
import (
"context"
"fmt"
"io"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"google.golang.org/grpc"
)
func TailVolume(master string, grpcDialOption grpc.DialOption, vid VolumeId, sinceNs uint64, timeoutSeconds int, fn func(n *Needle) error) error {
// find volume location, replication, ttl info
lookup, err := operation.Lookup(master, vid.String())
if err != nil {
return fmt.Errorf("look up volume %d: %v", vid, err)
}
if len(lookup.Locations) == 0 {
return fmt.Errorf("unable to locate volume %d", vid)
}
volumeServer := lookup.Locations[0].Url
return operation.WithVolumeServerClient(volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
stream, err := client.VolumeTail(context.Background(), &volume_server_pb.VolumeTailRequest{
VolumeId: uint32(vid),
SinceNs: sinceNs,
DrainingSeconds: uint32(timeoutSeconds),
})
if err != nil {
return err
}
for {
resp, recvErr := stream.Recv()
if recvErr != nil {
if recvErr == io.EOF {
break
} else {
return recvErr
}
}
needleHeader := resp.NeedleHeader
needleBody := resp.NeedleBody
if len(needleHeader) == 0 {
continue
}
for !resp.IsLastChunk {
resp, recvErr = stream.Recv()
if recvErr != nil {
if recvErr == io.EOF {
break
} else {
return recvErr
}
}
needleBody = append(needleBody, resp.NeedleBody...)
}
n := new(Needle)
n.ParseNeedleHeader(needleHeader)
n.ReadNeedleBodyBytes(needleBody, CurrentVersion)
err = fn(n)
if err != nil {
return err
}
}
return nil
})
}

9
weed/storage/volume.go
View File

@@ -2,7 +2,10 @@ package storage
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"os"
"path"
"strconv"
@@ -13,7 +16,7 @@ import (
)
type Volume struct {
Id VolumeId
Id needle.VolumeId
dir string
Collection string
dataFile *os.File
@@ -31,7 +34,7 @@ type Volume struct {
lastCompactRevision uint16
}
func NewVolume(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL, preallocate int64) (v *Volume, e error) {
func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64) (v *Volume, e error) {
// if replicaPlacement is nil, the superblock will be loaded from disk
v = &Volume{dir: dirname, Collection: collection, Id: id}
v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}
@@ -59,7 +62,7 @@ func (v *Volume) DataFile() *os.File {
return v.dataFile
}
func (v *Volume) Version() Version {
func (v *Volume) Version() needle.Version {
return v.SuperBlock.Version()
}

14
weed/storage/volume_backup.go
View File

@@ -3,12 +3,14 @@ package storage
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"google.golang.org/grpc"
"io"
"os"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"google.golang.org/grpc"
)
func (v *Volume) GetVolumeSyncStatus() *volume_server_pb.VolumeSyncStatusResponse {
@@ -147,7 +149,7 @@ func (v *Volume) locateLastAppendEntry() (Offset, error) {
func (v *Volume) readAppendAtNs(offset Offset) (uint64, error) {
n, _, bodyLength, err := ReadNeedleHeader(v.dataFile, v.SuperBlock.version, offset.ToAcutalOffset())
n, _, bodyLength, err := needle.ReadNeedleHeader(v.dataFile, v.SuperBlock.version, offset.ToAcutalOffset())
if err != nil {
return 0, fmt.Errorf("ReadNeedleHeader: %v", err)
}
@@ -245,7 +247,7 @@ func (scanner *VolumeFileScanner4GenIdx) ReadNeedleBody() bool {
return false
}
func (scanner *VolumeFileScanner4GenIdx) VisitNeedle(n *Needle, offset int64) error {
func (scanner *VolumeFileScanner4GenIdx) VisitNeedle(n *needle.Needle, offset int64) error {
if n.Size > 0 && n.Size != TombstoneFileSize {
return scanner.v.nm.Put(n.Id, ToOffset(offset), n.Size)
}

9
weed/storage/volume_checking.go
View File

@@ -4,14 +4,11 @@ import (
"fmt"
"os"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
func getActualSize(size uint32, version Version) int64 {
return NeedleEntrySize + NeedleBodyLength(size, version)
}
func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) error {
var indexSize int64
var e error
@@ -55,8 +52,8 @@ func readIndexEntryAtOffset(indexFile *os.File, offset int64) (bytes []byte, err
return
}
func verifyNeedleIntegrity(datFile *os.File, v Version, offset int64, key NeedleId, size uint32) error {
n := new(Needle)
func verifyNeedleIntegrity(datFile *os.File, v needle.Version, offset int64, key NeedleId, size uint32) error {
n := new(needle.Needle)
err := n.ReadData(datFile, offset, size, v)
if err != nil {
return err

13
weed/storage/volume_info.go
View File

@@ -5,15 +5,16 @@ import (
"sort"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
type VolumeInfo struct {
Id VolumeId
Id needle.VolumeId
Size uint64
ReplicaPlacement *ReplicaPlacement
Ttl *TTL
Ttl *needle.TTL
Collection string
Version Version
Version needle.Version
FileCount int
DeleteCount int
DeletedByteCount uint64
@@ -23,14 +24,14 @@ type VolumeInfo struct {
func NewVolumeInfo(m *master_pb.VolumeInformationMessage) (vi VolumeInfo, err error) {
vi = VolumeInfo{
Id: VolumeId(m.Id),
Id: needle.VolumeId(m.Id),
Size: m.Size,
Collection: m.Collection,
FileCount: int(m.FileCount),
DeleteCount: int(m.DeleteCount),
DeletedByteCount: m.DeletedByteCount,
ReadOnly: m.ReadOnly,
Version: Version(m.Version),
Version: needle.Version(m.Version),
CompactRevision: m.CompactRevision,
}
rp, e := NewReplicaPlacementFromByte(byte(m.ReplicaPlacement))
@@ -38,7 +39,7 @@ func NewVolumeInfo(m *master_pb.VolumeInformationMessage) (vi VolumeInfo, err er
return vi, e
}
vi.ReplicaPlacement = rp
vi.Ttl = LoadTTLFromUint32(m.Ttl)
vi.Ttl = needle.LoadTTLFromUint32(m.Ttl)
return vi, nil
}

8
weed/storage/volume_info_test.go
View File

@@ -1,6 +1,10 @@
package storage
import "testing"
import (
"testing"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
func TestSortVolumeInfos(t *testing.T) {
vis := []*VolumeInfo{
@@ -16,7 +20,7 @@ func TestSortVolumeInfos(t *testing.T) {
}
sortVolumeInfos(vis)
for i := 0; i < len(vis); i++ {
if vis[i].Id != VolumeId(i+1) {
if vis[i].Id != needle.VolumeId(i+1) {
t.Fatal()
}
}

6
weed/storage/volume_loading.go
View File

@@ -2,14 +2,16 @@ package storage
import (
"fmt"
"github.com/syndtr/goleveldb/leveldb/opt"
"os"
"time"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/chrislusf/seaweedfs/weed/glog"
)
func loadVolumeWithoutIndex(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType) (v *Volume, e error) {
func loadVolumeWithoutIndex(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType) (v *Volume, e error) {
v = &Volume{dir: dirname, Collection: collection, Id: id}
v.SuperBlock = SuperBlock{}
v.needleMapKind = needleMapKind

31
weed/storage/volume_read_write.go
View File

@@ -9,6 +9,7 @@ import (
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
)
@@ -16,13 +17,13 @@ var ErrorNotFound = errors.New("not found")
// isFileUnchanged checks whether this needle to write is same as last one.
// It requires serialized access in the same volume.
func (v *Volume) isFileUnchanged(n *Needle) bool {
func (v *Volume) isFileUnchanged(n *needle.Needle) bool {
if v.Ttl.String() != "" {
return false
}
nv, ok := v.nm.Get(n.Id)
if ok && !nv.Offset.IsZero() && nv.Size != TombstoneFileSize {
oldNeedle := new(Needle)
oldNeedle := new(needle.Needle)
err := oldNeedle.ReadData(v.dataFile, nv.Offset.ToAcutalOffset(), nv.Size, v.Version())
if err != nil {
glog.V(0).Infof("Failed to check updated file at offset %d size %d: %v", nv.Offset.ToAcutalOffset(), nv.Size, err)
@@ -76,8 +77,8 @@ func (v *Volume) AppendBlob(b []byte) (offset int64, err error) {
return
}
func (v *Volume) writeNeedle(n *Needle) (offset uint64, size uint32, err error) {
glog.V(4).Infof("writing needle %s", NewFileIdFromNeedle(v.Id, n).String())
func (v *Volume) writeNeedle(n *needle.Needle) (offset uint64, size uint32, err error) {
glog.V(4).Infof("writing needle %s", needle.NewFileIdFromNeedle(v.Id, n).String())
if v.readOnly {
err = fmt.Errorf("%s is read-only", v.dataFile.Name())
return
@@ -107,8 +108,8 @@ func (v *Volume) writeNeedle(n *Needle) (offset uint64, size uint32, err error)
return
}
func (v *Volume) deleteNeedle(n *Needle) (uint32, error) {
glog.V(4).Infof("delete needle %s", NewFileIdFromNeedle(v.Id, n).String())
func (v *Volume) deleteNeedle(n *needle.Needle) (uint32, error) {
glog.V(4).Infof("delete needle %s", needle.NewFileIdFromNeedle(v.Id, n).String())
if v.readOnly {
return 0, fmt.Errorf("%s is read-only", v.dataFile.Name())
}
@@ -133,7 +134,7 @@ func (v *Volume) deleteNeedle(n *Needle) (uint32, error) {
}
// read fills in Needle content by looking up n.Id from NeedleMapper
func (v *Volume) readNeedle(n *Needle) (int, error) {
func (v *Volume) readNeedle(n *needle.Needle) (int, error) {
nv, ok := v.nm.Get(n.Id)
if !ok || nv.Offset.IsZero() {
v.compactingWg.Wait()
@@ -172,10 +173,10 @@ func (v *Volume) readNeedle(n *Needle) (int, error) {
type VolumeFileScanner interface {
VisitSuperBlock(SuperBlock) error
ReadNeedleBody() bool
VisitNeedle(n *Needle, offset int64) error
VisitNeedle(n *needle.Needle, offset int64) error
}
func ScanVolumeFile(dirname string, collection string, id VolumeId,
func ScanVolumeFile(dirname string, collection string, id needle.VolumeId,
needleMapKind NeedleMapType,
volumeFileScanner VolumeFileScanner) (err error) {
var v *Volume
@@ -194,8 +195,8 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
return ScanVolumeFileFrom(version, v.dataFile, offset, volumeFileScanner)
}
func ScanVolumeFileFrom(version Version, dataFile *os.File, offset int64, volumeFileScanner VolumeFileScanner) (err error) {
n, _, rest, e := ReadNeedleHeader(dataFile, version, offset)
func ScanVolumeFileFrom(version needle.Version, dataFile *os.File, offset int64, volumeFileScanner VolumeFileScanner) (err error) {
n, _, rest, e := needle.ReadNeedleHeader(dataFile, version, offset)
if e != nil {
if e == io.EOF {
return nil
@@ -219,7 +220,7 @@ func ScanVolumeFileFrom(version Version, dataFile *os.File, offset int64, volume
}
offset += NeedleEntrySize + rest
glog.V(4).Infof("==> new entry offset %d", offset)
if n, _, rest, err = ReadNeedleHeader(dataFile, version, offset); err != nil {
if n, _, rest, err = needle.ReadNeedleHeader(dataFile, version, offset); err != nil {
if err == io.EOF {
return nil
}
@@ -230,8 +231,8 @@ func ScanVolumeFileFrom(version Version, dataFile *os.File, offset int64, volume
return nil
}
func ScanVolumeFileNeedleFrom(version Version, dataFile *os.File, offset int64, fn func(needleHeader, needleBody []byte, needleAppendAtNs uint64) error) (err error) {
n, nh, rest, e := ReadNeedleHeader(dataFile, version, offset)
func ScanVolumeFileNeedleFrom(version needle.Version, dataFile *os.File, offset int64, fn func(needleHeader, needleBody []byte, needleAppendAtNs uint64) error) (err error) {
n, nh, rest, e := needle.ReadNeedleHeader(dataFile, version, offset)
if e != nil {
if e == io.EOF {
return nil
@@ -252,7 +253,7 @@ func ScanVolumeFileNeedleFrom(version Version, dataFile *os.File, offset int64,
}
offset += NeedleEntrySize + rest
glog.V(4).Infof("==> new entry offset %d", offset)
if n, nh, rest, err = ReadNeedleHeader(dataFile, version, offset); err != nil {
if n, nh, rest, err = needle.ReadNeedleHeader(dataFile, version, offset); err != nil {
if err == io.EOF {
return nil
}

15
weed/storage/volume_super_block.go
View File

@@ -6,6 +6,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/golang/protobuf/proto"
)
@@ -23,9 +24,9 @@ const (
* Rest bytes: Reserved
*/
type SuperBlock struct {
version Version
version needle.Version
ReplicaPlacement *ReplicaPlacement
Ttl *TTL
Ttl *needle.TTL
CompactRevision uint16
Extra *master_pb.SuperBlockExtra
extraSize uint16
@@ -33,13 +34,13 @@ type SuperBlock struct {
func (s *SuperBlock) BlockSize() int {
switch s.version {
case Version2, Version3:
case needle.Version2, needle.Version3:
return _SuperBlockSize + int(s.extraSize)
}
return _SuperBlockSize
}
func (s *SuperBlock) Version() Version {
func (s *SuperBlock) Version() needle.Version {
return s.version
}
func (s *SuperBlock) Bytes() []byte {
@@ -75,7 +76,7 @@ func (v *Volume) maybeWriteSuperBlock() error {
return e
}
if stat.Size() == 0 {
v.SuperBlock.version = CurrentVersion
v.SuperBlock.version = needle.CurrentVersion
_, e = v.dataFile.Write(v.SuperBlock.Bytes())
if e != nil && os.IsPermission(e) {
//read-only, but zero length - recreate it!
@@ -105,12 +106,12 @@ func ReadSuperBlock(dataFile *os.File) (superBlock SuperBlock, err error) {
err = fmt.Errorf("cannot read volume %s super block: %v", dataFile.Name(), e)
return
}
superBlock.version = Version(header[0])
superBlock.version = needle.Version(header[0])
if superBlock.ReplicaPlacement, err = NewReplicaPlacementFromByte(header[1]); err != nil {
err = fmt.Errorf("cannot read replica type: %s", err.Error())
return
}
superBlock.Ttl = LoadTTLFromBytes(header[2:4])
superBlock.Ttl = needle.LoadTTLFromBytes(header[2:4])
superBlock.CompactRevision = util.BytesToUint16(header[4:6])
superBlock.extraSize = util.BytesToUint16(header[6:8])

8
weed/storage/volume_super_block_test.go
View File

@@ -2,20 +2,22 @@ package storage
import (
"testing"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
func TestSuperBlockReadWrite(t *testing.T) {
rp, _ := NewReplicaPlacementFromByte(byte(001))
ttl, _ := ReadTTL("15d")
ttl, _ := needle.ReadTTL("15d")
s := &SuperBlock{
version: CurrentVersion,
version: needle.CurrentVersion,
ReplicaPlacement: rp,
Ttl: ttl,
}
bytes := s.Bytes()
if !(bytes[2] == 15 && bytes[3] == Day) {
if !(bytes[2] == 15 && bytes[3] == needle.Day) {
println("byte[2]:", bytes[2], "byte[3]:", bytes[3])
t.Fail()
}

11
weed/storage/volume_vacuum.go
View File

@@ -6,6 +6,7 @@ import (
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
@@ -205,7 +206,7 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
//even the needle cache in memory is hit, the need_bytes is correct
glog.V(4).Infof("file %d offset %d size %d", key, increIdxEntry.offset.ToAcutalOffset(), increIdxEntry.size)
var needleBytes []byte
needleBytes, err = ReadNeedleBlob(oldDatFile, increIdxEntry.offset.ToAcutalOffset(), increIdxEntry.size, v.Version())
needleBytes, err = needle.ReadNeedleBlob(oldDatFile, increIdxEntry.offset.ToAcutalOffset(), increIdxEntry.size, v.Version())
if err != nil {
return fmt.Errorf("ReadNeedleBlob %s key %d offset %d size %d failed: %v", oldDatFile.Name(), key, increIdxEntry.offset.ToAcutalOffset(), increIdxEntry.size, err)
}
@@ -213,7 +214,7 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
util.Uint32toBytes(idxEntryBytes[8:12], uint32(offset/NeedlePaddingSize))
} else { //deleted needle
//fakeDelNeedle 's default Data field is nil
fakeDelNeedle := new(Needle)
fakeDelNeedle := new(needle.Needle)
fakeDelNeedle.Id = key
fakeDelNeedle.Cookie = 0x12345678
fakeDelNeedle.AppendAtNs = uint64(time.Now().UnixNano())
@@ -235,7 +236,7 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
}
type VolumeFileScanner4Vacuum struct {
version Version
version needle.Version
v *Volume
dst *os.File
nm *NeedleMap
@@ -255,7 +256,7 @@ func (scanner *VolumeFileScanner4Vacuum) ReadNeedleBody() bool {
return true
}
func (scanner *VolumeFileScanner4Vacuum) VisitNeedle(n *Needle, offset int64) error {
func (scanner *VolumeFileScanner4Vacuum) VisitNeedle(n *needle.Needle, offset int64) error {
if n.HasTtl() && scanner.now >= n.LastModified+uint64(scanner.v.Ttl.Minutes()*60) {
return nil
}
@@ -334,7 +335,7 @@ func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) {
return nil
}
n := new(Needle)
n := new(needle.Needle)
err := n.ReadData(v.dataFile, offset.ToAcutalOffset(), size, v.Version())
if err != nil {
return nil

18
weed/storage/volume_vacuum_test.go
View File

@@ -1,11 +1,13 @@
package storage
import (
"github.com/chrislusf/seaweedfs/weed/storage/types"
"io/ioutil"
"math/rand"
"os"
"testing"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/types"
)
/*
@@ -65,7 +67,7 @@ func TestCompaction(t *testing.T) {
}
defer os.RemoveAll(dir) // clean up
v, err := NewVolume(dir, "", 1, NeedleMapInMemory, &ReplicaPlacement{}, &TTL{}, 0)
v, err := NewVolume(dir, "", 1, NeedleMapInMemory, &ReplicaPlacement{}, &needle.TTL{}, 0)
if err != nil {
t.Fatalf("volume creation: %v", err)
}
@@ -145,21 +147,21 @@ func doSomeWritesDeletes(i int, v *Volume, t *testing.T, infos []*needleInfo) {
type needleInfo struct {
size uint32
crc CRC
crc needle.CRC
}
func newRandomNeedle(id uint64) *Needle {
n := new(Needle)
func newRandomNeedle(id uint64) *needle.Needle {
n := new(needle.Needle)
n.Data = make([]byte, rand.Intn(1024))
rand.Read(n.Data)
n.Checksum = NewCRC(n.Data)
n.Checksum = needle.NewCRC(n.Data)
n.Id = types.Uint64ToNeedleId(id)
return n
}
func newEmptyNeedle(id uint64) *Needle {
n := new(Needle)
func newEmptyNeedle(id uint64) *needle.Needle {
n := new(needle.Needle)
n.Id = types.Uint64ToNeedleId(id)
return n
}