mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-10-08 00:24:23 +08:00
garbage percentage threshold
This commit is contained in:
@@ -245,7 +245,7 @@ func printTestingInstructions() {
|
|||||||
fmt.Println("1. Configure Vacuum for Testing:")
|
fmt.Println("1. Configure Vacuum for Testing:")
|
||||||
fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum")
|
fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum")
|
||||||
fmt.Println(" Set:")
|
fmt.Println(" Set:")
|
||||||
fmt.Printf(" - Garbage Threshold: 0.20 (20%% - lower than default)\n")
|
fmt.Printf(" - Garbage Percentage Threshold: 20 (20%% - lower than default 30)\n")
|
||||||
fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n")
|
fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n")
|
||||||
fmt.Printf(" - Min Volume Age: [0] [Minutes] (no age requirement)\n")
|
fmt.Printf(" - Min Volume Age: [0] [Minutes] (no age requirement)\n")
|
||||||
fmt.Printf(" - Max Concurrent: 2\n")
|
fmt.Printf(" - Max Concurrent: 2\n")
|
||||||
@@ -258,7 +258,8 @@ func printTestingInstructions() {
|
|||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
|
||||||
fmt.Println("3. Manual Vacuum (Optional):")
|
fmt.Println("3. Manual Vacuum (Optional):")
|
||||||
fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.01'")
|
fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.20'")
|
||||||
|
fmt.Println(" (Note: Master API still uses 0.0-1.0 decimal format)")
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
|
||||||
fmt.Println("4. Check Logs:")
|
fmt.Println("4. Check Logs:")
|
||||||
|
@@ -3,11 +3,13 @@ package task
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -311,11 +313,17 @@ func (ms *MasterSynchronizer) checkVacuumCandidate(volumeID uint32, state *Volum
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Get the current garbage threshold from the vacuum detector
|
||||||
|
vacuumDetector, _ := vacuum.GetSharedInstances()
|
||||||
|
var vacuumThresholdPercent float64 = 0.3 // Default fallback
|
||||||
|
if vacuumDetector != nil {
|
||||||
|
vacuumThresholdPercent = vacuumDetector.GetGarbageThreshold()
|
||||||
|
}
|
||||||
|
|
||||||
// Vacuum criteria:
|
// Vacuum criteria:
|
||||||
// 1. Significant deleted bytes (> 30% of volume size or > 1GB)
|
// 1. Significant deleted bytes (> configured threshold or > 1GB)
|
||||||
// 2. Not currently being written to heavily
|
// 2. Not currently being written to heavily
|
||||||
|
|
||||||
const vacuumThresholdPercent = 0.3
|
|
||||||
const vacuumMinBytes = 1024 * 1024 * 1024 // 1GB
|
const vacuumMinBytes = 1024 * 1024 * 1024 // 1GB
|
||||||
|
|
||||||
deletedRatio := float64(volume.DeletedByteCount) / float64(volume.Size)
|
deletedRatio := float64(volume.DeletedByteCount) / float64(volume.Size)
|
||||||
@@ -331,8 +339,8 @@ func (ms *MasterSynchronizer) checkVacuumCandidate(volumeID uint32, state *Volum
|
|||||||
Server: volume.Server,
|
Server: volume.Server,
|
||||||
TaskType: "vacuum",
|
TaskType: "vacuum",
|
||||||
Priority: types.TaskPriorityNormal,
|
Priority: types.TaskPriorityNormal,
|
||||||
Reason: fmt.Sprintf("Deleted bytes %d (%.1f%%) exceed vacuum threshold",
|
Reason: fmt.Sprintf("Deleted bytes %d (%.1f%%) exceed vacuum threshold (%.1f%%)",
|
||||||
volume.DeletedByteCount, deletedRatio*100),
|
volume.DeletedByteCount, deletedRatio*100, vacuumThresholdPercent*100),
|
||||||
VolumeInfo: volume,
|
VolumeInfo: volume,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -408,7 +416,13 @@ func (ms *MasterSynchronizer) createTaskFromCandidate(candidate *VolumeMaintenan
|
|||||||
task.Parameters["replication"] = "001" // Default replication for EC
|
task.Parameters["replication"] = "001" // Default replication for EC
|
||||||
task.Parameters["collection"] = candidate.VolumeInfo.Collection
|
task.Parameters["collection"] = candidate.VolumeInfo.Collection
|
||||||
case "vacuum":
|
case "vacuum":
|
||||||
task.Parameters["garbage_threshold"] = "0.3" // 30% threshold
|
// Get the current garbage threshold from the vacuum detector
|
||||||
|
vacuumDetector, _ := vacuum.GetSharedInstances()
|
||||||
|
var garbageThreshold float64 = 0.3 // Default fallback
|
||||||
|
if vacuumDetector != nil {
|
||||||
|
garbageThreshold = vacuumDetector.GetGarbageThreshold()
|
||||||
|
}
|
||||||
|
task.Parameters["garbage_threshold"] = strconv.FormatFloat(garbageThreshold, 'f', -1, 64)
|
||||||
case "ec_rebuild":
|
case "ec_rebuild":
|
||||||
// Add info about which shards need rebuilding
|
// Add info about which shards need rebuilding
|
||||||
}
|
}
|
||||||
|
@@ -95,25 +95,25 @@ func (ui *UIProvider) RenderConfigForm(currentConfig interface{}) (template.HTML
|
|||||||
|
|
||||||
form.AddNumberField(
|
form.AddNumberField(
|
||||||
"garbage_threshold",
|
"garbage_threshold",
|
||||||
"Garbage Threshold (%)",
|
"Garbage Percentage Threshold",
|
||||||
"Trigger vacuum when garbage ratio exceeds this percentage (0.0-1.0)",
|
"Trigger vacuum when garbage ratio exceeds this percentage (0-100)",
|
||||||
config.GarbageThreshold,
|
config.GarbageThreshold*100, // Convert 0.0-1.0 to 0-100 for display
|
||||||
true,
|
true,
|
||||||
)
|
)
|
||||||
|
|
||||||
form.AddIntervalField(
|
form.AddDurationField(
|
||||||
"scan_interval",
|
"scan_interval",
|
||||||
"Scan Interval",
|
"Scan Interval",
|
||||||
"How often to scan for volumes needing vacuum",
|
"How often to scan for volumes needing vacuum",
|
||||||
config.ScanIntervalSeconds,
|
secondsToDuration(config.ScanIntervalSeconds),
|
||||||
true,
|
true,
|
||||||
)
|
)
|
||||||
|
|
||||||
form.AddIntervalField(
|
form.AddDurationField(
|
||||||
"min_volume_age",
|
"min_volume_age",
|
||||||
"Minimum Volume Age",
|
"Minimum Volume Age",
|
||||||
"Only vacuum volumes older than this duration",
|
"Only vacuum volumes older than this duration",
|
||||||
config.MinVolumeAgeSeconds,
|
secondsToDuration(config.MinVolumeAgeSeconds),
|
||||||
true,
|
true,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -157,7 +157,7 @@ function resetForm() {
|
|||||||
if (confirm('Reset all vacuum settings to defaults?')) {
|
if (confirm('Reset all vacuum settings to defaults?')) {
|
||||||
// Reset to default values
|
// Reset to default values
|
||||||
document.querySelector('input[name="enabled"]').checked = true;
|
document.querySelector('input[name="enabled"]').checked = true;
|
||||||
document.querySelector('input[name="garbage_threshold"]').value = '0.3';
|
document.querySelector('input[name="garbage_threshold"]').value = '30';
|
||||||
document.querySelector('input[name="scan_interval"]').value = '30m';
|
document.querySelector('input[name="scan_interval"]').value = '30m';
|
||||||
document.querySelector('input[name="min_volume_age"]').value = '1h';
|
document.querySelector('input[name="min_volume_age"]').value = '1h';
|
||||||
document.querySelector('input[name="max_concurrent"]').value = '2';
|
document.querySelector('input[name="max_concurrent"]').value = '2';
|
||||||
@@ -177,47 +177,33 @@ func (ui *UIProvider) ParseConfigForm(formData map[string][]string) (interface{}
|
|||||||
// Parse enabled checkbox
|
// Parse enabled checkbox
|
||||||
config.Enabled = len(formData["enabled"]) > 0 && formData["enabled"][0] == "on"
|
config.Enabled = len(formData["enabled"]) > 0 && formData["enabled"][0] == "on"
|
||||||
|
|
||||||
// Parse garbage threshold
|
// Parse garbage threshold (convert from 0-100 to 0.0-1.0)
|
||||||
if thresholdStr := formData["garbage_threshold"]; len(thresholdStr) > 0 {
|
if thresholdStr := formData["garbage_threshold"]; len(thresholdStr) > 0 {
|
||||||
if threshold, err := strconv.ParseFloat(thresholdStr[0], 64); err != nil {
|
if threshold, err := strconv.ParseFloat(thresholdStr[0], 64); err != nil {
|
||||||
return nil, fmt.Errorf("invalid garbage threshold: %w", err)
|
return nil, fmt.Errorf("invalid garbage percentage threshold: %w", err)
|
||||||
} else if threshold < 0 || threshold > 1 {
|
} else if threshold < 0 || threshold > 100 {
|
||||||
return nil, fmt.Errorf("garbage threshold must be between 0.0 and 1.0")
|
return nil, fmt.Errorf("garbage percentage threshold must be between 0 and 100")
|
||||||
} else {
|
} else {
|
||||||
config.GarbageThreshold = threshold
|
config.GarbageThreshold = threshold / 100.0 // Convert percentage to decimal
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse scan interval
|
// Parse scan interval
|
||||||
if values, ok := formData["scan_interval_value"]; ok && len(values) > 0 {
|
if intervalStr := formData["scan_interval"]; len(intervalStr) > 0 {
|
||||||
value, err := strconv.Atoi(values[0])
|
if interval, err := time.ParseDuration(intervalStr[0]); err != nil {
|
||||||
if err != nil {
|
return nil, fmt.Errorf("invalid scan interval: %w", err)
|
||||||
return nil, fmt.Errorf("invalid scan interval value: %w", err)
|
} else {
|
||||||
|
config.ScanIntervalSeconds = durationToSeconds(interval)
|
||||||
}
|
}
|
||||||
|
|
||||||
unit := "minute" // default
|
|
||||||
if units, ok := formData["scan_interval_unit"]; ok && len(units) > 0 {
|
|
||||||
unit = units[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert to seconds
|
|
||||||
config.ScanIntervalSeconds = types.IntervalValueUnitToSeconds(value, unit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse min volume age
|
// Parse min volume age
|
||||||
if values, ok := formData["min_volume_age_value"]; ok && len(values) > 0 {
|
if ageStr := formData["min_volume_age"]; len(ageStr) > 0 {
|
||||||
value, err := strconv.Atoi(values[0])
|
if age, err := time.ParseDuration(ageStr[0]); err != nil {
|
||||||
if err != nil {
|
return nil, fmt.Errorf("invalid min volume age: %w", err)
|
||||||
return nil, fmt.Errorf("invalid min volume age value: %w", err)
|
} else {
|
||||||
|
config.MinVolumeAgeSeconds = durationToSeconds(age)
|
||||||
}
|
}
|
||||||
|
|
||||||
unit := "minute" // default
|
|
||||||
if units, ok := formData["min_volume_age_unit"]; ok && len(units) > 0 {
|
|
||||||
unit = units[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert to seconds
|
|
||||||
config.MinVolumeAgeSeconds = types.IntervalValueUnitToSeconds(value, unit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse max concurrent
|
// Parse max concurrent
|
||||||
|
Reference in New Issue
Block a user