lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-02-09 09:17:28 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Include meta in ReadAllNeedles (#3991)

Browse Source 
This is useful for doing backups on the data so we can accurately store the
last modified time, the compression state, and verify the crc.

Previously we were doing VolumeNeedleStatus and then an HTTP request which
needlessly read from the dat file twice.
This commit is contained in:
James Hartig
2022-11-20 23:19:41 -05:00
committed by GitHub
parent 70a4c98b00
commit 4c85da7844
4 changed files with 746 additions and 708 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
unmaintained/stream_read_volume/stream_read_volume.go
View File

@@ -5,13 +5,14 @@ import (
"errors" "errors"
"flag" "flag"
"fmt" "fmt"
"io"
"github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc" "google.golang.org/grpc"
"io"
) )
var ( var (
@@ -29,7 +30,7 @@ func main() {
vid := uint32(*volumeId) vid := uint32(*volumeId)
eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error { eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error {
fmt.Printf("%d,%x%08x %d\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob)) fmt.Printf("%d,%x%08x %d %v %d %x\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob), resp.NeedleBlobCompressed, resp.LastModified, resp.Crc)
return nil return nil
} }

3
weed/pb/volume_server.proto
View File

@@ -317,6 +317,9 @@ message ReadAllNeedlesResponse {
uint64 needle_id = 2; uint64 needle_id = 2;
uint32 cookie = 3; uint32 cookie = 3;
bytes needle_blob = 5; bytes needle_blob = 5;
bool needle_blob_compressed = 6;
uint64 last_modified = 7;
uint32 crc = 8;
} }
message VolumeTailSenderRequest { message VolumeTailSenderRequest {

1435
weed/pb/volume_server_pb/volume_server.pb.go
View File

File diff suppressed because it is too large Load Diff

11
weed/storage/volume_read_all.go
View File

@@ -30,10 +30,13 @@ func (scanner *VolumeFileScanner4ReadAll) VisitNeedle(n *needle.Needle, offset i
} }
sendErr := scanner.Stream.Send(&volume_server_pb.ReadAllNeedlesResponse{ sendErr := scanner.Stream.Send(&volume_server_pb.ReadAllNeedlesResponse{
VolumeId: uint32(scanner.V.Id), VolumeId: uint32(scanner.V.Id),
NeedleId: uint64(n.Id), NeedleId: uint64(n.Id),
Cookie: uint32(n.Cookie), Cookie: uint32(n.Cookie),
NeedleBlob: n.Data, NeedleBlob: n.Data,
NeedleBlobCompressed: n.IsCompressed(),
LastModified: n.LastModified,
Crc: n.Checksum.Value(),
}) })
if sendErr != nil { if sendErr != nil {
return sendErr return sendErr