mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-08-20 09:17:35 +08:00
fix disk space calculation
This commit is contained in:
parent
3023a6f3a4
commit
2d0d429d2f
@ -18,10 +18,27 @@ func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (float64, error) {
|
||||
}
|
||||
func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
|
||||
if v := s.findVolume(vid); v != nil {
|
||||
s := stats.NewDiskStatus(v.dir)
|
||||
if int64(s.Free) < preallocate {
|
||||
return fmt.Errorf("free space: %d bytes, not enough for %d bytes", s.Free, preallocate)
|
||||
// Get current volume size for space calculation
|
||||
volumeSize, indexSize, _ := v.FileStat()
|
||||
|
||||
// Calculate space needed for compaction:
|
||||
// 1. Space for the new compacted volume (approximately same as current volume size)
|
||||
// 2. Use the larger of preallocate or estimated volume size
|
||||
estimatedCompactSize := int64(volumeSize + indexSize)
|
||||
spaceNeeded := preallocate
|
||||
if estimatedCompactSize > preallocate {
|
||||
spaceNeeded = estimatedCompactSize
|
||||
}
|
||||
|
||||
diskStatus := stats.NewDiskStatus(v.dir)
|
||||
if int64(diskStatus.Free) < spaceNeeded {
|
||||
return fmt.Errorf("insufficient free space for compaction: need %d bytes (volume: %d, index: %d, buffer: 10%%), but only %d bytes available",
|
||||
spaceNeeded, volumeSize, indexSize, diskStatus.Free)
|
||||
}
|
||||
|
||||
glog.V(1).Infof("volume %d compaction space check: volume=%d, index=%d, space_needed=%d, free_space=%d",
|
||||
vid, volumeSize, indexSize, spaceNeeded, diskStatus.Free)
|
||||
|
||||
return v.Compact2(preallocate, compactionBytePerSecond, progressFn)
|
||||
}
|
||||
return fmt.Errorf("volume id %d is not found during compact", vid)
|
||||
|
||||
99
weed/storage/store_vacuum_test.go
Normal file
99
weed/storage/store_vacuum_test.go
Normal file
@ -0,0 +1,99 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
)
|
||||
|
||||
func TestCompactVolumeSpaceCheck(t *testing.T) {
|
||||
// Create a temporary directory for testing
|
||||
dir, err := os.MkdirTemp("", "seaweedfs_test")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
// Create a disk location
|
||||
location := &DiskLocation{
|
||||
Directory: dir,
|
||||
IdxDirectory: dir,
|
||||
volumes: make(map[needle.VolumeId]*Volume),
|
||||
}
|
||||
|
||||
// Create a store
|
||||
store := &Store{
|
||||
Locations: []*DiskLocation{location},
|
||||
}
|
||||
|
||||
// Create a volume with some data
|
||||
vid := needle.VolumeId(1)
|
||||
replication, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
volume, err := NewVolume(dir, dir, "", vid, NeedleMapInMemory, replication, nil, 0, needle.GetCurrentVersion(), 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create volume: %v", err)
|
||||
}
|
||||
|
||||
location.SetVolume(vid, volume)
|
||||
|
||||
// Test space checking logic
|
||||
t.Run("InsufficientSpace", func(t *testing.T) {
|
||||
// This should fail because we're testing the improved space checking
|
||||
err := store.CompactVolume(vid, 0, 0, nil)
|
||||
if err == nil {
|
||||
t.Error("Expected compaction to fail due to insufficient space")
|
||||
}
|
||||
if err != nil {
|
||||
t.Logf("Expected error: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Clean up
|
||||
volume.Close()
|
||||
}
|
||||
|
||||
func TestSpaceCalculation(t *testing.T) {
|
||||
// Test the space calculation logic
|
||||
testCases := []struct {
|
||||
name string
|
||||
volumeSize uint64
|
||||
indexSize uint64
|
||||
preallocate int64
|
||||
expectedMinimum int64
|
||||
}{
|
||||
{
|
||||
name: "SmallVolume",
|
||||
volumeSize: 1024 * 1024, // 1MB
|
||||
indexSize: 1024, // 1KB
|
||||
preallocate: 0,
|
||||
expectedMinimum: int64((1024*1024 + 1024) * 110 / 100), // 110% of volume+index size
|
||||
},
|
||||
{
|
||||
name: "LargePreallocate",
|
||||
volumeSize: 1024 * 1024, // 1MB
|
||||
indexSize: 1024, // 1KB
|
||||
preallocate: 10 * 1024 * 1024, // 10MB
|
||||
expectedMinimum: int64(10 * 1024 * 1024 * 110 / 100), // 110% of preallocate
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
estimatedCompactSize := int64(tc.volumeSize + tc.indexSize)
|
||||
spaceNeeded := tc.preallocate
|
||||
if estimatedCompactSize > tc.preallocate {
|
||||
spaceNeeded = estimatedCompactSize
|
||||
}
|
||||
// Add 10% safety buffer
|
||||
spaceNeeded = spaceNeeded + (spaceNeeded / 10)
|
||||
|
||||
if spaceNeeded < tc.expectedMinimum {
|
||||
t.Errorf("Space calculation incorrect: got %d, expected at least %d", spaceNeeded, tc.expectedMinimum)
|
||||
}
|
||||
t.Logf("Volume: %d, Index: %d, Preallocate: %d -> SpaceNeeded: %d",
|
||||
tc.volumeSize, tc.indexSize, tc.preallocate, spaceNeeded)
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user