lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-12-17 09:42:29 +08:00
Code Issues Actions 32 Packages Projects Releases Wiki Activity

go code can read and write chunk manifest

Browse Source
This commit is contained in:
Chris Lu
2020-07-19 17:59:43 -07:00
parent f90d2c93c9
commit 97d97f3528
21 changed files with 511 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/replication/sink/azuresink/azure_sink.go
View File

@@ -96,7 +96,7 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error {
}
totalSize := filer2.TotalSize(entry.Chunks)
chunkViews := filer2.ViewFromChunks(entry.Chunks, 0, int64(totalSize))
chunkViews := filer2.ViewFromChunks(g.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize))
// Create a URL that references a to-be-created blob in your
// Azure Storage account's container.

2
weed/replication/sink/b2sink/b2_sink.go
View File

@@ -85,7 +85,7 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
}
totalSize := filer2.TotalSize(entry.Chunks)
chunkViews := filer2.ViewFromChunks(entry.Chunks, 0, int64(totalSize))
chunkViews := filer2.ViewFromChunks(g.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize))
bucket, err := g.client.Bucket(context.Background(), g.bucket)
if err != nil {

26
weed/replication/sink/filersink/filer_sink.go
View File

@@ -167,12 +167,15 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent
glog.V(0).Infof("already replicated %s", key)
} else {
// find out what changed
deletedChunks, newChunks := compareChunks(oldEntry, newEntry)
deletedChunks, newChunks, err := compareChunks(filer2.LookupFn(fs), oldEntry, newEntry)
if err != nil {
return true, fmt.Errorf("replicte %s compare chunks error: %v", key, err)
}
// delete the chunks that are deleted from the source
if deleteIncludeChunks {
// remove the deleted chunks. Actual data deletion happens in filer UpdateEntry FindUnusedFileChunks
existingEntry.Chunks = filer2.MinusChunks(existingEntry.Chunks, deletedChunks)
existingEntry.Chunks = filer2.DoMinusChunks(existingEntry.Chunks, deletedChunks)
}
// replicate the chunks that are new in the source
@@ -200,8 +203,21 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent
})
}
func compareChunks(oldEntry, newEntry *filer_pb.Entry) (deletedChunks, newChunks []*filer_pb.FileChunk) {
deletedChunks = filer2.MinusChunks(oldEntry.Chunks, newEntry.Chunks)
newChunks = filer2.MinusChunks(newEntry.Chunks, oldEntry.Chunks)
func compareChunks(lookupFileIdFn filer2.LookupFileIdFunctionType, oldEntry, newEntry *filer_pb.Entry) (deletedChunks, newChunks []*filer_pb.FileChunk, err error) {
aData, aMeta, aErr := filer2.ResolveChunkManifest(lookupFileIdFn, oldEntry.Chunks)
if aErr != nil {
return nil, nil, aErr
}
bData, bMeta, bErr := filer2.ResolveChunkManifest(lookupFileIdFn, newEntry.Chunks)
if bErr != nil {
return nil, nil, bErr
}
deletedChunks = append(deletedChunks, filer2.DoMinusChunks(aData, bData)...)
deletedChunks = append(deletedChunks, filer2.DoMinusChunks(aMeta, bMeta)...)
newChunks = append(newChunks, filer2.DoMinusChunks(bData, aData)...)
newChunks = append(newChunks, filer2.DoMinusChunks(bMeta, aMeta)...)
return
}

2
weed/replication/sink/gcssink/gcs_sink.go
View File

@@ -90,7 +90,7 @@ func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry) error {
}
totalSize := filer2.TotalSize(entry.Chunks)
chunkViews := filer2.ViewFromChunks(entry.Chunks, 0, int64(totalSize))
chunkViews := filer2.ViewFromChunks(g.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize))
wc := g.client.Bucket(g.bucket).Object(key).NewWriter(context.Background())

2
weed/replication/sink/s3sink/s3_sink.go
View File

@@ -108,7 +108,7 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
}
totalSize := filer2.TotalSize(entry.Chunks)
chunkViews := filer2.ViewFromChunks(entry.Chunks, 0, int64(totalSize))
chunkViews := filer2.ViewFromChunks(s3sink.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize))
parts := make([]*s3.CompletedPart, len(chunkViews))