lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-14 21:16:17 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add dataCenter option when assign file keys

Browse Source 
add dataCenter option when starting volume servers
some work related to freeze a volume. Not tested yet.
This commit is contained in:
Chris Lu
2013-06-19 18:10:38 -07:00
parent 715d327df0
commit 50269b74ce
16 changed files with 287 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
go/storage/needle_map.go
View File

@@ -52,37 +52,61 @@ const (
func LoadNeedleMap(file *os.File) (*NeedleMap, error) {
nm := NewNeedleMap(file)
bufferReader := bufio.NewReaderSize(nm.indexFile, 1024*1024)
bytes := make([]byte, 16*RowsToRead)
count, e := bufferReader.Read(bytes)
for count > 0 && e == nil {
for i := 0; i < count; i += 16 {
key := util.BytesToUint64(bytes[i : i+8])
offset := util.BytesToUint32(bytes[i+8 : i+12])
size := util.BytesToUint32(bytes[i+12 : i+16])
nm.FileCounter++
nm.FileByteCounter = nm.FileByteCounter + uint64(size)
if offset > 0 {
oldSize := nm.m.Set(Key(key), offset, size)
//log.Println("reading key", key, "offset", offset, "size", size, "oldSize", oldSize)
if oldSize > 0 {
nm.DeletionCounter++
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
}
} else {
oldSize := nm.m.Delete(Key(key))
//log.Println("removing key", key, "offset", offset, "size", size, "oldSize", oldSize)
e := walkIndexFile(file, func(key uint64, offset, size uint32) error {
nm.FileCounter++
nm.FileByteCounter = nm.FileByteCounter + uint64(size)
if offset > 0 {
oldSize := nm.m.Set(Key(key), offset, size)
//log.Println("reading key", key, "offset", offset, "size", size, "oldSize", oldSize)
if oldSize > 0 {
nm.DeletionCounter++
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
}
} else {
oldSize := nm.m.Delete(Key(key))
//log.Println("removing key", key, "offset", offset, "size", size, "oldSize", oldSize)
nm.DeletionCounter++
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
}
return nil
})
return nm, e
}
count, e = bufferReader.Read(bytes)
// walks through the index file, calls fn function with each key, offset, size
// stops with the error returned by the fn function
func walkIndexFile(r io.Reader, fn func(key uint64, offset, size uint32) error) error {
br := bufio.NewReaderSize(r, 1024*1024)
bytes := make([]byte, 16*RowsToRead)
count, e := br.Read(bytes)
var (
key uint64
offset, size uint32
i int
)
for count > 0 && e == nil {
for i = 0; i+16 <= count; i += 16 {
key = util.BytesToUint64(bytes[i : i+8])
offset = util.BytesToUint32(bytes[i+8 : i+12])
size = util.BytesToUint32(bytes[i+12 : i+16])
if e = fn(key, offset, size); e != nil {
return e
}
}
if count%16 != 0 {
copy(bytes[:count-i], bytes[i:count])
i = count - i
count, e = br.Read(bytes[i:])
count += i
} else {
count, e = br.Read(bytes)
}
}
if e == io.EOF {
e = nil
return nil
}
return nm, e
return e
}
func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) (int, error) {