lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-21 13:48:51 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add VolumeStreamFollow, but not used yet

Browse Source
This commit is contained in:
Chris Lu
2019-04-18 00:18:29 -07:00
parent 13ad5c1966
commit b09e8dbb37
8 changed files with 351 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
weed/storage/needle_read_write.go
View File

@@ -267,14 +267,14 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
return nil
}
func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bodyLength int64, err error) {
func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bytes []byte, bodyLength int64, err error) {
n = new(Needle)
if version == Version1 || version == Version2 || version == Version3 {
bytes := make([]byte, NeedleEntrySize)
bytes = make([]byte, NeedleEntrySize)
var count int
count, err = r.ReadAt(bytes, offset)
if count <= 0 || err != nil {
return nil, 0, err
return nil, bytes, 0, err
}
n.ParseNeedleHeader(bytes)
bodyLength = NeedleBodyLength(n.Size, version)
@@ -299,21 +299,21 @@ func NeedleBodyLength(needleSize uint32, version Version) int64 {
//n should be a needle already read the header
//the input stream will read until next file entry
func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyLength int64) (err error) {
func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyLength int64) (bytes []byte, err error) {
if bodyLength <= 0 {
return nil
return nil, nil
}
switch version {
case Version1:
bytes := make([]byte, bodyLength)
bytes = make([]byte, bodyLength)
if _, err = r.ReadAt(bytes, offset); err != nil {
return
}
n.Data = bytes[:n.Size]
n.Checksum = NewCRC(n.Data)
case Version2, Version3:
bytes := make([]byte, bodyLength)
bytes = make([]byte, bodyLength)
if _, err = r.ReadAt(bytes, offset); err != nil {
return
}

6
weed/storage/volume_backup.go
View File

@@ -68,7 +68,7 @@ func (v *Volume) IncrementalBackup(volumeServer string, grpcDialOption grpc.Dial
stream, err := client.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{
VolumeId: uint32(v.Id),
Since: appendAtNs,
SinceNs: appendAtNs,
})
if err != nil {
return err
@@ -147,11 +147,11 @@ func (v *Volume) locateLastAppendEntry() (Offset, error) {
func (v *Volume) readAppendAtNs(offset Offset) (uint64, error) {
n, bodyLength, err := ReadNeedleHeader(v.dataFile, v.SuperBlock.version, offset.ToAcutalOffset())
n, _, bodyLength, err := ReadNeedleHeader(v.dataFile, v.SuperBlock.version, offset.ToAcutalOffset())
if err != nil {
return 0, fmt.Errorf("ReadNeedleHeader: %v", err)
}
err = n.ReadNeedleBody(v.dataFile, v.SuperBlock.version, offset.ToAcutalOffset()+int64(NeedleEntrySize), bodyLength)
_, err = n.ReadNeedleBody(v.dataFile, v.SuperBlock.version, offset.ToAcutalOffset()+int64(NeedleEntrySize), bodyLength)
if err != nil {
return 0, fmt.Errorf("ReadNeedleBody offset %d, bodyLength %d: %v", offset.ToAcutalOffset(), bodyLength, err)
}

39
weed/storage/volume_read_write.go
View File

@@ -195,7 +195,7 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
}
func ScanVolumeFileFrom(version Version, dataFile *os.File, offset int64, volumeFileScanner VolumeFileScanner) (err error) {
n, rest, e := ReadNeedleHeader(dataFile, version, offset)
n, _, rest, e := ReadNeedleHeader(dataFile, version, offset)
if e != nil {
if e == io.EOF {
return nil
@@ -204,7 +204,7 @@ func ScanVolumeFileFrom(version Version, dataFile *os.File, offset int64, volume
}
for n != nil {
if volumeFileScanner.ReadNeedleBody() {
if err = n.ReadNeedleBody(dataFile, version, offset+NeedleEntrySize, rest); err != nil {
if _, err = n.ReadNeedleBody(dataFile, version, offset+NeedleEntrySize, rest); err != nil {
glog.V(0).Infof("cannot read needle body: %v", err)
//err = fmt.Errorf("cannot read needle body: %v", err)
//return
@@ -219,7 +219,40 @@ func ScanVolumeFileFrom(version Version, dataFile *os.File, offset int64, volume
}
offset += NeedleEntrySize + rest
glog.V(4).Infof("==> new entry offset %d", offset)
if n, rest, err = ReadNeedleHeader(dataFile, version, offset); err != nil {
if n, _, rest, err = ReadNeedleHeader(dataFile, version, offset); err != nil {
if err == io.EOF {
return nil
}
return fmt.Errorf("cannot read needle header at offset %d: %v", offset, err)
}
glog.V(4).Infof("new entry needle size:%d rest:%d", n.Size, rest)
}
return nil
}
func ScanVolumeFileNeedleFrom(version Version, dataFile *os.File, offset int64, fn func(needleHeader, needleBody []byte, needleAppendAtNs uint64) error) (err error) {
n, nh, rest, e := ReadNeedleHeader(dataFile, version, offset)
if e != nil {
if e == io.EOF {
return nil
}
return fmt.Errorf("cannot read %s at offset %d: %v", dataFile.Name(), offset, e)
}
for n != nil {
var needleBody []byte
if needleBody, err = n.ReadNeedleBody(dataFile, version, offset+NeedleEntrySize, rest); err != nil {
glog.V(0).Infof("cannot read needle body: %v", err)
//err = fmt.Errorf("cannot read needle body: %v", err)
//return
}
err = fn(nh, needleBody, n.AppendAtNs)
if err != nil {
glog.V(0).Infof("visit needle error: %v", err)
return
}
offset += NeedleEntrySize + rest
glog.V(4).Infof("==> new entry offset %d", offset)
if n, nh, rest, err = ReadNeedleHeader(dataFile, version, offset); err != nil {
if err == io.EOF {
return nil
}