lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-08-01 17:46:55 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Important Fix: Avoid overwriting the index file!!!

Browse Source
This commit is contained in:
Chris Lu 2014-05-29 19:19:03 -07:00
parent 5f58da8de7
commit faf497feb8
2 changed files with 37 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
go/storage/needle_map.go
View File

@ -51,7 +51,7 @@ const (
func LoadNeedleMap(file *os.File) (*NeedleMap, error) { func LoadNeedleMap(file *os.File) (*NeedleMap, error) {
nm := NewNeedleMap(file) nm := NewNeedleMap(file)
e := walkIndexFile(file, func(key uint64, offset, size uint32) error { e := WalkIndexFile(file, func(key uint64, offset, size uint32) error {
if key > nm.MaximumFileKey { if key > nm.MaximumFileKey {
nm.MaximumFileKey = key nm.MaximumFileKey = key
} }
@ -78,7 +78,7 @@ func LoadNeedleMap(file *os.File) (*NeedleMap, error) {
// walks through the index file, calls fn function with each key, offset, size // walks through the index file, calls fn function with each key, offset, size
// stops with the error returned by the fn function // stops with the error returned by the fn function
func walkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) error { func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) error {
var readerOffset int64 var readerOffset int64
bytes := make([]byte, 16*RowsToRead) bytes := make([]byte, 16*RowsToRead)
count, e := r.ReadAt(bytes, readerOffset) count, e := r.ReadAt(bytes, readerOffset)
@ -124,6 +124,9 @@ func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) (int, error) {
nm.DeletionCounter++ nm.DeletionCounter++
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize) nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
} }
if _, err := nm.indexFile.Seek(0, 2); err != nil {
return 0, fmt.Errorf("cannot go to the end of indexfile %s: %s", nm.indexFile.Name(), err.Error())
}
return nm.indexFile.Write(bytes) return nm.indexFile.Write(bytes)
} }
func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
@ -132,20 +135,15 @@ func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
} }
func (nm *NeedleMap) Delete(key uint64) error { func (nm *NeedleMap) Delete(key uint64) error {
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(nm.m.Delete(Key(key))) nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(nm.m.Delete(Key(key)))
offset, err := nm.indexFile.Seek(0, 1)
if err != nil {
return fmt.Errorf("cannot get position of indexfile: %s", err)
}
bytes := make([]byte, 16) bytes := make([]byte, 16)
util.Uint64toBytes(bytes[0:8], key) util.Uint64toBytes(bytes[0:8], key)
util.Uint32toBytes(bytes[8:12], 0) util.Uint32toBytes(bytes[8:12], 0)
util.Uint32toBytes(bytes[12:16], 0) util.Uint32toBytes(bytes[12:16], 0)
if _, err = nm.indexFile.Write(bytes); err != nil { if _, err := nm.indexFile.Seek(0, 2); err != nil {
plus := "" return fmt.Errorf("cannot go to the end of indexfile %s: %s", nm.indexFile.Name(), err.Error())
if e := nm.indexFile.Truncate(offset); e != nil { }
plus = "\ncouldn't truncate index file: " + e.Error() if _, err := nm.indexFile.Write(bytes); err != nil {
} return fmt.Errorf("error writing to indexfile %s: %s", nm.indexFile.Name(), err.Error())
return fmt.Errorf("error writing to indexfile %s: %s%s", nm.indexFile.Name(), err, plus)
} }
nm.DeletionCounter++ nm.DeletionCounter++
return nil return nil

27
go/tools/read_index.go Normal file
View File

@ -0,0 +1,27 @@
package main
import (
"code.google.com/p/weed-fs/go/storage"
"flag"
"fmt"
"log"
"os"
)
var (
indexFileName = flag.String("file", "", ".idx file to analyze")
)
func main() {
flag.Parse()
indexFile, err := os.OpenFile(*indexFileName, os.O_RDONLY, 0644)
if err != nil {
log.Fatalf("Create Volume Index [ERROR] %s\n", err)
}
defer indexFile.Close()
storage.WalkIndexFile(indexFile, func(key uint64, offset, size uint32) error {
fmt.Printf("key %d, offset %d, size %d, nextOffset %d\n", key, offset*8, size, offset*8+size)
return nil
})
}