lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-21 16:37:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

allocate ec shards to volume servers

Browse Source
This commit is contained in:
Chris Lu
2019-05-25 02:02:44 -07:00
parent 6f4b09b6a4
commit f0e6574d5e
6 changed files with 172 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
weed/storage/erasure_coding/ec_encoder.go
View File

@@ -15,6 +15,7 @@ import (
const (
DataShardsCount = 10
ParityShardsCount = 4
TotalShardsCount = DataShardsCount + ParityShardsCount
ErasureCodingLargeBlockSize = 1024 * 1024 * 1024 // 1GB
ErasureCodingSmallBlockSize = 1024 * 1024 // 1MB
)
@@ -93,7 +94,7 @@ func encodeData(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize i
}
func openEcFiles(baseFileName string, forRead bool) (files []*os.File, err error) {
for i := 0; i < DataShardsCount+ParityShardsCount; i++ {
for i := 0; i < TotalShardsCount; i++ {
fname := baseFileName + ToExt(i)
openOption := os.O_TRUNC | os.O_CREATE | os.O_WRONLY
if forRead {
@@ -138,7 +139,7 @@ func encodeDataOneBatch(file *os.File, enc reedsolomon.Encoder, startOffset, blo
return err
}
for i := 0; i < DataShardsCount+ParityShardsCount; i++ {
for i := 0; i < TotalShardsCount; i++ {
_, err := outputs[i].Write(buffers[i])
if err != nil {
return err
@@ -154,7 +155,7 @@ func encodeDatFile(remainingSize int64, err error, baseFileName string, bufferSi
if err != nil {
return fmt.Errorf("failed to create encoder: %v", err)
}
buffers := make([][]byte, DataShardsCount+ParityShardsCount)
buffers := make([][]byte, TotalShardsCount)
outputs, err := openEcFiles(baseFileName, false)
defer closeEcFiles(outputs)
if err != nil {

4
weed/storage/erasure_coding/ec_test.go
View File

@@ -153,9 +153,9 @@ func readFromOtherEcFiles(ecFiles []*os.File, ecFileIndex int, ecFileOffset int6
return nil, fmt.Errorf("failed to create encoder: %v", err)
}
bufs := make([][]byte, DataShardsCount+ParityShardsCount)
bufs := make([][]byte, TotalShardsCount)
for i := 0; i < DataShardsCount; {
n := int(rand.Int31n(DataShardsCount + ParityShardsCount))
n := int(rand.Int31n(TotalShardsCount))
if n == ecFileIndex || bufs[n] != nil {
continue
}

13
weed/storage/erasure_coding/ec_volume_info.go
View File

@@ -36,6 +36,10 @@ func (ecInfo *EcVolumeInfo) ShardIds() (ret []ShardId) {
return ecInfo.ShardBits.ShardIds()
}
func (ecInfo *EcVolumeInfo) ShardIdCount() (count int) {
return ecInfo.ShardBits.ShardIdCount()
}
func (ecInfo *EcVolumeInfo) Minus(other *EcVolumeInfo) (*EcVolumeInfo) {
ret := &EcVolumeInfo{
VolumeId: ecInfo.VolumeId,
@@ -69,7 +73,7 @@ func (b ShardBits) HasShardId(id ShardId) bool {
}
func (b ShardBits) ShardIds() (ret []ShardId) {
for i := ShardId(0); i < DataShardsCount+ParityShardsCount; i++ {
for i := ShardId(0); i < TotalShardsCount; i++ {
if b.HasShardId(i) {
ret = append(ret, i)
}
@@ -77,6 +81,13 @@ func (b ShardBits) ShardIds() (ret []ShardId) {
return
}
func (b ShardBits) ShardIdCount() (count int) {
for count = 0; b > 0; count++ {
b &= b - 1
}
return
}
func (b ShardBits) Minus(other ShardBits) (ShardBits) {
return b &^ other
}