lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-21 23:17:23 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Changed the InMemory bool to a uint32 so that it can be used to alter how much space to reserve

Browse Source
This commit is contained in:
Tom Maxwell
2019-09-04 15:27:14 +01:00
parent 6ee65356e3
commit 4a878c0006
20 changed files with 111 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/storage/disk_location.go
View File

@@ -60,7 +60,7 @@ func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind Ne
_, found := l.volumes[vid]
l.RUnlock()
if !found {
if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0, false); e == nil {
if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0, 0); e == nil {
l.Lock()
l.volumes[vid] = v
l.Unlock()

14
weed/storage/needle/volume_memory_map_max_size.go Normal file
View File

@@ -0,0 +1,14 @@
package needle
import "strconv"
func ReadMemoryMapMaxSizeMB(MemoryMapMaxSizeMBString string) (uint32, error) {
if MemoryMapMaxSizeMBString == "" {
return 0, nil
}
var MemoryMapMaxSize64 uint64
var err error
MemoryMapMaxSize64, err = strconv.ParseUint(MemoryMapMaxSizeMBString, 10, 32)
MemoryMapMaxSize := uint32(MemoryMapMaxSize64)
return MemoryMapMaxSize, err
}

10
weed/storage/needle/volume_memory_map_max_size_test.go Normal file
View File

@@ -0,0 +1,10 @@
package needle
import "testing"
func TestMemoryMapMaxSizeReadWrite(t *testing.T) {
memoryMapSize, _ := ReadMemoryMapMaxSizeMB("5000")
if memoryMapSize != 5000 {
t.Errorf("empty memoryMapSize:%v", memoryMapSize)
}
}

4
weed/storage/store.go
View File

@@ -59,7 +59,7 @@ func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, di
return
}
func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64, memoryMapped bool) error {
func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64, memoryMapped uint32) error {
rt, e := NewReplicaPlacementFromString(replicaPlacement)
if e != nil {
return e
@@ -101,7 +101,7 @@ func (s *Store) FindFreeLocation() (ret *DiskLocation) {
}
return ret
}
func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapped bool) error {
func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapped uint32) error {
if s.findVolume(vid) != nil {
return fmt.Errorf("Volume Id %d already exists!", vid)
}

4
weed/storage/volume.go
View File

@@ -25,7 +25,7 @@ type Volume struct {
nm NeedleMapper
needleMapKind NeedleMapType
readOnly bool
MemoryMapped bool
MemoryMapped uint32
SuperBlock
@@ -39,7 +39,7 @@ type Volume struct {
isCompacting bool
}
func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapped bool) (v *Volume, e error) {
func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapped uint32) (v *Volume, e error) {
// if replicaPlacement is nil, the superblock will be loaded from disk
v = &Volume{dir: dirname, Collection: collection, Id: id, MemoryMapped: memoryMapped}
v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}

2
weed/storage/volume_create.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/joeslay/seaweedfs/weed/glog"
)
func createVolumeFile(fileName string, preallocate int64, useMemoryMap bool) (*os.File, error) {
func createVolumeFile(fileName string, preallocate int64, useMemoryMap uint32) (*os.File, error) {
file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if preallocate > 0 {
glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName)

2
weed/storage/volume_create_linux.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/joeslay/seaweedfs/weed/glog"
)
func createVolumeFile(fileName string, preallocate int64, useMemoryMap bool) (file *os.File, e error) {
func createVolumeFile(fileName string, preallocate int64, useMemoryMap uint32) (file *os.File, e error) {
file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if preallocate != 0 {
syscall.Fallocate(int(file.Fd()), 1, 0, preallocate)

6
weed/storage/volume_create_windows.go
View File

@@ -12,12 +12,12 @@ import (
"github.com/joeslay/seaweedfs/weed/os_overloads"
)
func createVolumeFile(fileName string, preallocate int64, useMemoryMap bool) (*os.File, error) {
func createVolumeFile(fileName string, preallocate int64, useMemoryMap uint32) (*os.File, error) {
mMap, exists := memory_map.FileMemoryMap[fileName]
if !exists {
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, useMemoryMap)
if useMemoryMap {
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, useMemoryMap > 0)
if useMemoryMap > 0 {
memory_map.FileMemoryMap[fileName] = new(memory_map.MemoryMap)
new_mMap := memory_map.FileMemoryMap[fileName]

6
weed/storage/volume_vacuum.go
View File

@@ -23,7 +23,7 @@ func (v *Volume) garbageLevel() float64 {
func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error {
if !v.MemoryMapped { //it makes no sense to compact in memory
if v.MemoryMapped > 0 { //it makes no sense to compact in memory
glog.V(3).Infof("Compacting volume %d ...", v.Id)
//no need to lock for copy on write
//v.accessLock.Lock()
@@ -46,7 +46,7 @@ func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error
func (v *Volume) Compact2() error {
if !v.MemoryMapped { //it makes no sense to compact in memory
if v.MemoryMapped > 0 { //it makes no sense to compact in memory
glog.V(3).Infof("Compact2 volume %d ...", v.Id)
v.isCompacting = true
@@ -63,7 +63,7 @@ func (v *Volume) Compact2() error {
}
func (v *Volume) CommitCompact() error {
if !v.MemoryMapped { //it makes no sense to compact in memory
if v.MemoryMapped>0 { //it makes no sense to compact in memory
glog.V(0).Infof("Committing volume %d vacuuming...", v.Id)
v.isCompacting = true