garbage percentage threshold

This commit is contained in:
chrislu
2025-07-26 13:28:52 -07:00
parent a1e508cd8d
commit 965ec4f51b
3 changed files with 45 additions and 44 deletions

View File

@@ -245,7 +245,7 @@ func printTestingInstructions() {
fmt.Println("1. Configure Vacuum for Testing:") fmt.Println("1. Configure Vacuum for Testing:")
fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum") fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum")
fmt.Println(" Set:") fmt.Println(" Set:")
fmt.Printf(" - Garbage Threshold: 0.20 (20%% - lower than default)\n") fmt.Printf(" - Garbage Percentage Threshold: 20 (20%% - lower than default 30)\n")
fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n") fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n")
fmt.Printf(" - Min Volume Age: [0] [Minutes] (no age requirement)\n") fmt.Printf(" - Min Volume Age: [0] [Minutes] (no age requirement)\n")
fmt.Printf(" - Max Concurrent: 2\n") fmt.Printf(" - Max Concurrent: 2\n")
@@ -258,7 +258,8 @@ func printTestingInstructions() {
fmt.Println() fmt.Println()
fmt.Println("3. Manual Vacuum (Optional):") fmt.Println("3. Manual Vacuum (Optional):")
fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.01'") fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.20'")
fmt.Println(" (Note: Master API still uses 0.0-1.0 decimal format)")
fmt.Println() fmt.Println()
fmt.Println("4. Check Logs:") fmt.Println("4. Check Logs:")

View File

@@ -3,11 +3,13 @@ package task
import ( import (
"context" "context"
"fmt" "fmt"
"strconv"
"time" "time"
"github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/wdclient" "github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
"github.com/seaweedfs/seaweedfs/weed/worker/types" "github.com/seaweedfs/seaweedfs/weed/worker/types"
) )
@@ -311,11 +313,17 @@ func (ms *MasterSynchronizer) checkVacuumCandidate(volumeID uint32, state *Volum
return nil return nil
} }
// Get the current garbage threshold from the vacuum detector
vacuumDetector, _ := vacuum.GetSharedInstances()
var vacuumThresholdPercent float64 = 0.3 // Default fallback
if vacuumDetector != nil {
vacuumThresholdPercent = vacuumDetector.GetGarbageThreshold()
}
// Vacuum criteria: // Vacuum criteria:
// 1. Significant deleted bytes (> 30% of volume size or > 1GB) // 1. Significant deleted bytes (> configured threshold or > 1GB)
// 2. Not currently being written to heavily // 2. Not currently being written to heavily
const vacuumThresholdPercent = 0.3
const vacuumMinBytes = 1024 * 1024 * 1024 // 1GB const vacuumMinBytes = 1024 * 1024 * 1024 // 1GB
deletedRatio := float64(volume.DeletedByteCount) / float64(volume.Size) deletedRatio := float64(volume.DeletedByteCount) / float64(volume.Size)
@@ -331,8 +339,8 @@ func (ms *MasterSynchronizer) checkVacuumCandidate(volumeID uint32, state *Volum
Server: volume.Server, Server: volume.Server,
TaskType: "vacuum", TaskType: "vacuum",
Priority: types.TaskPriorityNormal, Priority: types.TaskPriorityNormal,
Reason: fmt.Sprintf("Deleted bytes %d (%.1f%%) exceed vacuum threshold", Reason: fmt.Sprintf("Deleted bytes %d (%.1f%%) exceed vacuum threshold (%.1f%%)",
volume.DeletedByteCount, deletedRatio*100), volume.DeletedByteCount, deletedRatio*100, vacuumThresholdPercent*100),
VolumeInfo: volume, VolumeInfo: volume,
} }
} }
@@ -408,7 +416,13 @@ func (ms *MasterSynchronizer) createTaskFromCandidate(candidate *VolumeMaintenan
task.Parameters["replication"] = "001" // Default replication for EC task.Parameters["replication"] = "001" // Default replication for EC
task.Parameters["collection"] = candidate.VolumeInfo.Collection task.Parameters["collection"] = candidate.VolumeInfo.Collection
case "vacuum": case "vacuum":
task.Parameters["garbage_threshold"] = "0.3" // 30% threshold // Get the current garbage threshold from the vacuum detector
vacuumDetector, _ := vacuum.GetSharedInstances()
var garbageThreshold float64 = 0.3 // Default fallback
if vacuumDetector != nil {
garbageThreshold = vacuumDetector.GetGarbageThreshold()
}
task.Parameters["garbage_threshold"] = strconv.FormatFloat(garbageThreshold, 'f', -1, 64)
case "ec_rebuild": case "ec_rebuild":
// Add info about which shards need rebuilding // Add info about which shards need rebuilding
} }

View File

@@ -95,25 +95,25 @@ func (ui *UIProvider) RenderConfigForm(currentConfig interface{}) (template.HTML
form.AddNumberField( form.AddNumberField(
"garbage_threshold", "garbage_threshold",
"Garbage Threshold (%)", "Garbage Percentage Threshold",
"Trigger vacuum when garbage ratio exceeds this percentage (0.0-1.0)", "Trigger vacuum when garbage ratio exceeds this percentage (0-100)",
config.GarbageThreshold, config.GarbageThreshold*100, // Convert 0.0-1.0 to 0-100 for display
true, true,
) )
form.AddIntervalField( form.AddDurationField(
"scan_interval", "scan_interval",
"Scan Interval", "Scan Interval",
"How often to scan for volumes needing vacuum", "How often to scan for volumes needing vacuum",
config.ScanIntervalSeconds, secondsToDuration(config.ScanIntervalSeconds),
true, true,
) )
form.AddIntervalField( form.AddDurationField(
"min_volume_age", "min_volume_age",
"Minimum Volume Age", "Minimum Volume Age",
"Only vacuum volumes older than this duration", "Only vacuum volumes older than this duration",
config.MinVolumeAgeSeconds, secondsToDuration(config.MinVolumeAgeSeconds),
true, true,
) )
@@ -157,7 +157,7 @@ function resetForm() {
if (confirm('Reset all vacuum settings to defaults?')) { if (confirm('Reset all vacuum settings to defaults?')) {
// Reset to default values // Reset to default values
document.querySelector('input[name="enabled"]').checked = true; document.querySelector('input[name="enabled"]').checked = true;
document.querySelector('input[name="garbage_threshold"]').value = '0.3'; document.querySelector('input[name="garbage_threshold"]').value = '30';
document.querySelector('input[name="scan_interval"]').value = '30m'; document.querySelector('input[name="scan_interval"]').value = '30m';
document.querySelector('input[name="min_volume_age"]').value = '1h'; document.querySelector('input[name="min_volume_age"]').value = '1h';
document.querySelector('input[name="max_concurrent"]').value = '2'; document.querySelector('input[name="max_concurrent"]').value = '2';
@@ -177,47 +177,33 @@ func (ui *UIProvider) ParseConfigForm(formData map[string][]string) (interface{}
// Parse enabled checkbox // Parse enabled checkbox
config.Enabled = len(formData["enabled"]) > 0 && formData["enabled"][0] == "on" config.Enabled = len(formData["enabled"]) > 0 && formData["enabled"][0] == "on"
// Parse garbage threshold // Parse garbage threshold (convert from 0-100 to 0.0-1.0)
if thresholdStr := formData["garbage_threshold"]; len(thresholdStr) > 0 { if thresholdStr := formData["garbage_threshold"]; len(thresholdStr) > 0 {
if threshold, err := strconv.ParseFloat(thresholdStr[0], 64); err != nil { if threshold, err := strconv.ParseFloat(thresholdStr[0], 64); err != nil {
return nil, fmt.Errorf("invalid garbage threshold: %w", err) return nil, fmt.Errorf("invalid garbage percentage threshold: %w", err)
} else if threshold < 0 || threshold > 1 { } else if threshold < 0 || threshold > 100 {
return nil, fmt.Errorf("garbage threshold must be between 0.0 and 1.0") return nil, fmt.Errorf("garbage percentage threshold must be between 0 and 100")
} else { } else {
config.GarbageThreshold = threshold config.GarbageThreshold = threshold / 100.0 // Convert percentage to decimal
} }
} }
// Parse scan interval // Parse scan interval
if values, ok := formData["scan_interval_value"]; ok && len(values) > 0 { if intervalStr := formData["scan_interval"]; len(intervalStr) > 0 {
value, err := strconv.Atoi(values[0]) if interval, err := time.ParseDuration(intervalStr[0]); err != nil {
if err != nil { return nil, fmt.Errorf("invalid scan interval: %w", err)
return nil, fmt.Errorf("invalid scan interval value: %w", err) } else {
config.ScanIntervalSeconds = durationToSeconds(interval)
} }
unit := "minute" // default
if units, ok := formData["scan_interval_unit"]; ok && len(units) > 0 {
unit = units[0]
}
// Convert to seconds
config.ScanIntervalSeconds = types.IntervalValueUnitToSeconds(value, unit)
} }
// Parse min volume age // Parse min volume age
if values, ok := formData["min_volume_age_value"]; ok && len(values) > 0 { if ageStr := formData["min_volume_age"]; len(ageStr) > 0 {
value, err := strconv.Atoi(values[0]) if age, err := time.ParseDuration(ageStr[0]); err != nil {
if err != nil { return nil, fmt.Errorf("invalid min volume age: %w", err)
return nil, fmt.Errorf("invalid min volume age value: %w", err) } else {
config.MinVolumeAgeSeconds = durationToSeconds(age)
} }
unit := "minute" // default
if units, ok := formData["min_volume_age_unit"]; ok && len(units) > 0 {
unit = units[0]
}
// Convert to seconds
config.MinVolumeAgeSeconds = types.IntervalValueUnitToSeconds(value, unit)
} }
// Parse max concurrent // Parse max concurrent