lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-21 16:37:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Changes to try and pass the URL parameters through - in memory flag not working still

Browse Source
This commit is contained in:
Tom Maxwell
2019-09-03 15:41:28 +01:00
parent 9a459d984b
commit d637d86d22
15 changed files with 74 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/storage/disk_location.go
View File

@@ -60,7 +60,7 @@ func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind Ne
_, found := l.volumes[vid]
l.RUnlock()
if !found {
if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0); e == nil {
if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0, false); e == nil {
l.Lock()
l.volumes[vid] = v
l.Unlock()

8
weed/storage/store.go
View File

@@ -59,7 +59,7 @@ func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, di
return
}
func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64) error {
func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64, in_memory bool) error {
rt, e := NewReplicaPlacementFromString(replicaPlacement)
if e != nil {
return e
@@ -68,7 +68,7 @@ func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMap
if e != nil {
return e
}
e = s.addVolume(volumeId, collection, needleMapKind, rt, ttl, preallocate)
e = s.addVolume(volumeId, collection, needleMapKind, rt, ttl, preallocate, in_memory)
return e
}
func (s *Store) DeleteCollection(collection string) (e error) {
@@ -101,14 +101,14 @@ func (s *Store) FindFreeLocation() (ret *DiskLocation) {
}
return ret
}
func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64) error {
func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, in_memory bool) error {
if s.findVolume(vid) != nil {
return fmt.Errorf("Volume Id %d already exists!", vid)
}
if location := s.FindFreeLocation(); location != nil {
glog.V(0).Infof("In dir %s adds volume:%v collection:%s replicaPlacement:%v ttl:%v",
location.Directory, vid, collection, replicaPlacement, ttl)
if volume, err := NewVolume(location.Directory, collection, vid, needleMapKind, replicaPlacement, ttl, preallocate); err == nil {
if volume, err := NewVolume(location.Directory, collection, vid, needleMapKind, replicaPlacement, ttl, preallocate, in_memory); err == nil {
location.SetVolume(vid, volume)
glog.V(0).Infof("add volume %d", vid)
s.NewVolumesChan <- master_pb.VolumeShortInformationMessage{

4
weed/storage/store_vacuum.go
View File

@@ -14,9 +14,9 @@ func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (float64, error) {
}
return 0, fmt.Errorf("volume id %d is not found during check compact", volumeId)
}
func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64) error {
func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, in_memory bool) error {
if v := s.findVolume(vid); v != nil {
return v.Compact(preallocate, compactionBytePerSecond)
return v.Compact(preallocate, compactionBytePerSecond, in_memory)
}
return fmt.Errorf("volume id %d is not found during compact", vid)
}

4
weed/storage/volume.go
View File

@@ -38,12 +38,12 @@ type Volume struct {
isCompacting bool
}
func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64) (v *Volume, e error) {
func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, in_memory bool) (v *Volume, e error) {
// if replicaPlacement is nil, the superblock will be loaded from disk
v = &Volume{dir: dirname, Collection: collection, Id: id}
v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}
v.needleMapKind = needleMapKind
e = v.load(true, true, needleMapKind, preallocate)
e = v.load(true, true, needleMapKind, preallocate, in_memory)
return
}
func (v *Volume) String() string {

12
weed/storage/volume_create_windows.go
View File

@@ -12,15 +12,17 @@ import (
"github.com/joeslay/seaweedfs/weed/os_overloads"
)
func createVolumeFile(fileName string, preallocate int64) (*os.File, error) {
func createVolumeFile(fileName string, preallocate int64, in_memory bool) (*os.File, error) {
mem_map, exists := memory_map.FileMemoryMap[fileName]
if !exists {
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, true)
memory_map.FileMemoryMap[fileName] = new(memory_map.MemoryMap)
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, in_memory)
if in_memory {
memory_map.FileMemoryMap[fileName] = new(memory_map.MemoryMap)
new_mem_map := memory_map.FileMemoryMap[fileName]
new_mem_map.CreateMemoryMap(file, 1024*1024*1024*2)
new_mem_map := memory_map.FileMemoryMap[fileName]
new_mem_map.CreateMemoryMap(file, 1024*1024*1024*2)
}
if preallocate > 0 {
glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName)

6
weed/storage/volume_loading.go
View File

@@ -16,11 +16,11 @@ func loadVolumeWithoutIndex(dirname string, collection string, id needle.VolumeI
v = &Volume{dir: dirname, Collection: collection, Id: id}
v.SuperBlock = SuperBlock{}
v.needleMapKind = needleMapKind
e = v.load(false, false, needleMapKind, 0)
e = v.load(false, false, needleMapKind, 0, false)
return
}
func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType, preallocate int64) error {
func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType, preallocate int64, in_memory bool) error {
var e error
fileName := v.FileName()
alreadyHasSuperBlock := false
@@ -42,7 +42,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
}
} else {
if createDatIfMissing {
v.dataFile, e = createVolumeFile(fileName+".dat", preallocate)
v.dataFile, e = createVolumeFile(fileName+".dat", preallocate, in_memory)
} else {
return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName)
}

10
weed/storage/volume_vacuum.go
View File

@@ -22,7 +22,7 @@ func (v *Volume) garbageLevel() float64 {
return float64(v.DeletedSize()) / float64(v.ContentSize())
}
func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error {
func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64, in_memory bool) error {
_, exists := memory_map.FileMemoryMap[v.dataFile.Name()]
if !exists { //it makes no sense to compact in memory
@@ -40,7 +40,7 @@ func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error
v.lastCompactIndexOffset = v.IndexFileSize()
v.lastCompactRevision = v.SuperBlock.CompactionRevision
glog.V(3).Infof("creating copies for volume %d ,last offset %d...", v.Id, v.lastCompactIndexOffset)
return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx", preallocate, compactionBytePerSecond)
return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx", preallocate, compactionBytePerSecond, in_memory)
} else {
return nil
}
@@ -114,7 +114,7 @@ func (v *Volume) CommitCompact() error {
os.RemoveAll(v.FileName() + ".bdb")
glog.V(3).Infof("Loading volume %d commit file...", v.Id)
if e = v.load(true, false, v.needleMapKind, 0); e != nil {
if e = v.load(true, false, v.needleMapKind, 0, false); e != nil {
return e
}
}
@@ -311,11 +311,11 @@ func (scanner *VolumeFileScanner4Vacuum) VisitNeedle(n *needle.Needle, offset in
return nil
}
func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, preallocate int64, compactionBytePerSecond int64) (err error) {
func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, preallocate int64, compactionBytePerSecond int64, in_memory bool) (err error) {
var (
dst, idx *os.File
)
if dst, err = createVolumeFile(dstName, preallocate); err != nil {
if dst, err = createVolumeFile(dstName, preallocate, in_memory); err != nil {
return
}
defer dst.Close()