lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-21 21:38:51 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

support Fallocate on linux

Browse Source
This commit is contained in:
Chris Lu
2017-01-08 11:01:46 -08:00
parent 7b6837cbc2
commit ed44f12f6d
17 changed files with 104 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/storage/disk_location.go
View File

@@ -36,7 +36,7 @@ func (l *DiskLocation) loadExistingVolume(dir os.FileInfo, needleMapKind NeedleM
_, found := l.volumes[vid]
mutex.RUnlock()
if !found {
if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil); e == nil {
if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0); e == nil {
mutex.Lock()
l.volumes[vid] = v
mutex.Unlock()

9
weed/storage/needle_read_write.go
View File

@@ -146,18 +146,13 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize i
util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
_, err = w.Write(header[0 : NeedleChecksumSize+padding])
actualSize = NeedleHeaderSize + int64(n.Size) + NeedleChecksumSize + int64(padding)
return n.DataSize, actualSize, err
return n.DataSize, getActualSize(n.Size), err
}
return 0, 0, fmt.Errorf("Unsupported Version! (%d)", version)
}
func ReadNeedleBlob(r *os.File, offset int64, size uint32) (dataSlice []byte, block *Block, err error) {
NeedleWithoutPaddingSize := NeedleHeaderSize + size + NeedleChecksumSize
padding := NeedlePaddingSize - (NeedleWithoutPaddingSize % NeedlePaddingSize)
readSize := NeedleWithoutPaddingSize + padding
return getBytesForFileBlock(r, offset, int(readSize))
return getBytesForFileBlock(r, offset, int(getActualSize(size)))
}
func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) {

10
weed/storage/store.go
View File

@@ -95,7 +95,7 @@ func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts
}
return
}
func (s *Store) AddVolume(volumeListString string, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string) error {
func (s *Store) AddVolume(volumeListString string, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64) error {
rt, e := NewReplicaPlacementFromString(replicaPlacement)
if e != nil {
return e
@@ -111,7 +111,7 @@ func (s *Store) AddVolume(volumeListString string, collection string, needleMapK
if err != nil {
return fmt.Errorf("Volume Id %s is not a valid unsigned integer!", id_string)
}
e = s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl)
e = s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl, preallocate)
} else {
pair := strings.Split(range_string, "-")
start, start_err := strconv.ParseUint(pair[0], 10, 64)
@@ -123,7 +123,7 @@ func (s *Store) AddVolume(volumeListString string, collection string, needleMapK
return fmt.Errorf("Volume End Id %s is not a valid unsigned integer!", pair[1])
}
for id := start; id <= end; id++ {
if err := s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl); err != nil {
if err := s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl, preallocate); err != nil {
e = err
}
}
@@ -160,14 +160,14 @@ func (s *Store) findFreeLocation() (ret *DiskLocation) {
}
return ret
}
func (s *Store) addVolume(vid VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL) error {
func (s *Store) addVolume(vid VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL, preallocate int64) error {
if s.findVolume(vid) != nil {
return fmt.Errorf("Volume Id %d already exists!", vid)
}
if location := s.findFreeLocation(); location != nil {
glog.V(0).Infof("In dir %s adds volume:%v collection:%s replicaPlacement:%v ttl:%v",
location.Directory, vid, collection, replicaPlacement, ttl)
if volume, err := NewVolume(location.Directory, collection, vid, needleMapKind, replicaPlacement, ttl); err == nil {
if volume, err := NewVolume(location.Directory, collection, vid, needleMapKind, replicaPlacement, ttl, preallocate); err == nil {
location.SetVolume(vid, volume)
return nil
} else {

4
weed/storage/volume.go
View File

@@ -29,11 +29,11 @@ type Volume struct {
lastCompactRevision uint16
}
func NewVolume(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL) (v *Volume, e error) {
func NewVolume(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL, preallocate int64) (v *Volume, e error) {
v = &Volume{dir: dirname, Collection: collection, Id: id}
v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}
v.needleMapKind = needleMapKind
e = v.load(true, true, needleMapKind)
e = v.load(true, true, needleMapKind, preallocate)
return
}
func (v *Volume) String() string {

17
weed/storage/volume_create.go Normal file
View File

@@ -0,0 +1,17 @@
// +build !linux
package storage
import (
"os"
"github.com/chrislusf/seaweedfs/weed/glog"
)
func createVolumeFile(fileName string, preallocate int64) (file *os.File, e error) {
file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644)
if preallocate > 0 {
glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName)
}
return file, e
}

19
weed/storage/volume_create_linux.go Normal file
View File

@@ -0,0 +1,19 @@
// +build linux
package storage
import (
"os"
"syscall"
"github.com/chrislusf/seaweedfs/weed/glog"
)
func createVolumeFile(fileName string, preallocate int64) (file *os.File, e error) {
file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644)
if preallocate != 0 {
syscall.Fallocate(int(file.Fd()), 1, 0, preallocate)
glog.V(0).Infof("Preallocated %d bytes disk space for %s", preallocate, fileName)
}
return file, e
}

6
weed/storage/volume_loading.go
View File

@@ -12,11 +12,11 @@ func loadVolumeWithoutIndex(dirname string, collection string, id VolumeId, need
v = &Volume{dir: dirname, Collection: collection, Id: id}
v.SuperBlock = SuperBlock{}
v.needleMapKind = needleMapKind
e = v.load(false, false, needleMapKind)
e = v.load(false, false, needleMapKind, 0)
return
}
func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType) error {
func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType, preallocate int64) error {
var e error
fileName := v.FileName()
@@ -34,7 +34,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
}
} else {
if createDatIfMissing {
v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
v.dataFile, e = createVolumeFile(fileName+".dat", preallocate)
} else {
return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName)
}

8
weed/storage/volume_vacuum.go
View File

@@ -24,7 +24,7 @@ func (v *Volume) Compact() error {
v.lastCompactIndexOffset = v.nm.IndexFileSize()
v.lastCompactRevision = v.SuperBlock.CompactRevision
glog.V(3).Infof("creating copies for volume %d ,last offset %d...", v.Id, v.lastCompactIndexOffset)
return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx")
return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx", v.dataFileSize)
}
func (v *Volume) Compact2() error {
@@ -66,7 +66,7 @@ func (v *Volume) commitCompact() error {
//glog.V(3).Infof("Pretending to be vacuuming...")
//time.Sleep(20 * time.Second)
glog.V(3).Infof("Loading Commit file...")
if e = v.load(true, false, v.needleMapKind); e != nil {
if e = v.load(true, false, v.needleMapKind, 0); e != nil {
return e
}
return nil
@@ -207,11 +207,11 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
return nil
}
func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) {
func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, preallocate int64) (err error) {
var (
dst, idx *os.File
)
if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
if dst, err = createVolumeFile(dstName, preallocate); err != nil {
return
}
defer dst.Close()