lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-07-31 20:36:08 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor out volume vacuum.

Browse Source
This commit is contained in:
Chris Lu 2014-05-19 20:54:39 -07:00
parent e7aaa24da8
commit fe3f06435e
2 changed files with 40 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
go/storage/volume.go
View File

@ -346,44 +346,6 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
return return
} }
func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) {
var (
dst, idx *os.File
)
if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
return
}
defer dst.Close()
if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
return
}
defer idx.Close()
nm := NewNeedleMap(idx)
new_offset := int64(SuperBlockSize)
err = ScanVolumeFile(v.dir, v.Collection, v.Id, func(superBlock SuperBlock) error {
_, err = dst.Write(superBlock.Bytes())
return err
}, func(n *Needle, offset int64) error {
nv, ok := v.nm.Get(n.Id)
glog.V(3).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv)
if ok && int64(nv.Offset)*NeedlePaddingSize == offset && nv.Size > 0 {
if _, err = nm.Put(n.Id, uint32(new_offset/NeedlePaddingSize), n.Size); err != nil {
return fmt.Errorf("cannot put needle: %s", err)
}
if _, err = n.Append(dst, v.Version()); err != nil {
return fmt.Errorf("cannot append needle: %s", err)
}
new_offset += n.DiskSize()
glog.V(3).Infoln("saving key", n.Id, "volume offset", offset, "=>", new_offset, "data_size", n.Size)
}
return nil
})
return
}
func (v *Volume) ContentSize() uint64 { func (v *Volume) ContentSize() uint64 {
return v.nm.ContentSize() return v.nm.ContentSize()
} }

40
go/storage/volume_vacuum.go
View File

@ -2,6 +2,7 @@ package storage
import ( import (
"code.google.com/p/weed-fs/go/glog" "code.google.com/p/weed-fs/go/glog"
"fmt"
"os" "os"
) )
@ -33,3 +34,42 @@ func (v *Volume) commitCompact() error {
} }
return nil return nil
} }
func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) {
var (
dst, idx *os.File
)
if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
return
}
defer dst.Close()
if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
return
}
defer idx.Close()
nm := NewNeedleMap(idx)
new_offset := int64(SuperBlockSize)
err = ScanVolumeFile(v.dir, v.Collection, v.Id, func(superBlock SuperBlock) error {
_, err = dst.Write(superBlock.Bytes())
return err
}, func(n *Needle, offset int64) error {
nv, ok := v.nm.Get(n.Id)
glog.V(3).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv)
if ok && int64(nv.Offset)*NeedlePaddingSize == offset && nv.Size > 0 {
if _, err = nm.Put(n.Id, uint32(new_offset/NeedlePaddingSize), n.Size); err != nil {
return fmt.Errorf("cannot put needle: %s", err)
}
if _, err = n.Append(dst, v.Version()); err != nil {
return fmt.Errorf("cannot append needle: %s", err)
}
new_offset += n.DiskSize()
glog.V(3).Infoln("saving key", n.Id, "volume offset", offset, "=>", new_offset, "data_size", n.Size)
}
return nil
})
return
}