mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-10-07 20:24:23 +08:00
garbage percentage threshold
This commit is contained in:
@@ -245,7 +245,7 @@ func printTestingInstructions() {
|
||||
fmt.Println("1. Configure Vacuum for Testing:")
|
||||
fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum")
|
||||
fmt.Println(" Set:")
|
||||
fmt.Printf(" - Garbage Threshold: 0.20 (20%% - lower than default)\n")
|
||||
fmt.Printf(" - Garbage Percentage Threshold: 20 (20%% - lower than default 30)\n")
|
||||
fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n")
|
||||
fmt.Printf(" - Min Volume Age: [0] [Minutes] (no age requirement)\n")
|
||||
fmt.Printf(" - Max Concurrent: 2\n")
|
||||
@@ -258,7 +258,8 @@ func printTestingInstructions() {
|
||||
fmt.Println()
|
||||
|
||||
fmt.Println("3. Manual Vacuum (Optional):")
|
||||
fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.01'")
|
||||
fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.20'")
|
||||
fmt.Println(" (Note: Master API still uses 0.0-1.0 decimal format)")
|
||||
fmt.Println()
|
||||
|
||||
fmt.Println("4. Check Logs:")
|
||||
|
@@ -3,11 +3,13 @@ package task
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
@@ -311,11 +313,17 @@ func (ms *MasterSynchronizer) checkVacuumCandidate(volumeID uint32, state *Volum
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get the current garbage threshold from the vacuum detector
|
||||
vacuumDetector, _ := vacuum.GetSharedInstances()
|
||||
var vacuumThresholdPercent float64 = 0.3 // Default fallback
|
||||
if vacuumDetector != nil {
|
||||
vacuumThresholdPercent = vacuumDetector.GetGarbageThreshold()
|
||||
}
|
||||
|
||||
// Vacuum criteria:
|
||||
// 1. Significant deleted bytes (> 30% of volume size or > 1GB)
|
||||
// 1. Significant deleted bytes (> configured threshold or > 1GB)
|
||||
// 2. Not currently being written to heavily
|
||||
|
||||
const vacuumThresholdPercent = 0.3
|
||||
const vacuumMinBytes = 1024 * 1024 * 1024 // 1GB
|
||||
|
||||
deletedRatio := float64(volume.DeletedByteCount) / float64(volume.Size)
|
||||
@@ -331,8 +339,8 @@ func (ms *MasterSynchronizer) checkVacuumCandidate(volumeID uint32, state *Volum
|
||||
Server: volume.Server,
|
||||
TaskType: "vacuum",
|
||||
Priority: types.TaskPriorityNormal,
|
||||
Reason: fmt.Sprintf("Deleted bytes %d (%.1f%%) exceed vacuum threshold",
|
||||
volume.DeletedByteCount, deletedRatio*100),
|
||||
Reason: fmt.Sprintf("Deleted bytes %d (%.1f%%) exceed vacuum threshold (%.1f%%)",
|
||||
volume.DeletedByteCount, deletedRatio*100, vacuumThresholdPercent*100),
|
||||
VolumeInfo: volume,
|
||||
}
|
||||
}
|
||||
@@ -408,7 +416,13 @@ func (ms *MasterSynchronizer) createTaskFromCandidate(candidate *VolumeMaintenan
|
||||
task.Parameters["replication"] = "001" // Default replication for EC
|
||||
task.Parameters["collection"] = candidate.VolumeInfo.Collection
|
||||
case "vacuum":
|
||||
task.Parameters["garbage_threshold"] = "0.3" // 30% threshold
|
||||
// Get the current garbage threshold from the vacuum detector
|
||||
vacuumDetector, _ := vacuum.GetSharedInstances()
|
||||
var garbageThreshold float64 = 0.3 // Default fallback
|
||||
if vacuumDetector != nil {
|
||||
garbageThreshold = vacuumDetector.GetGarbageThreshold()
|
||||
}
|
||||
task.Parameters["garbage_threshold"] = strconv.FormatFloat(garbageThreshold, 'f', -1, 64)
|
||||
case "ec_rebuild":
|
||||
// Add info about which shards need rebuilding
|
||||
}
|
||||
|
@@ -95,25 +95,25 @@ func (ui *UIProvider) RenderConfigForm(currentConfig interface{}) (template.HTML
|
||||
|
||||
form.AddNumberField(
|
||||
"garbage_threshold",
|
||||
"Garbage Threshold (%)",
|
||||
"Trigger vacuum when garbage ratio exceeds this percentage (0.0-1.0)",
|
||||
config.GarbageThreshold,
|
||||
"Garbage Percentage Threshold",
|
||||
"Trigger vacuum when garbage ratio exceeds this percentage (0-100)",
|
||||
config.GarbageThreshold*100, // Convert 0.0-1.0 to 0-100 for display
|
||||
true,
|
||||
)
|
||||
|
||||
form.AddIntervalField(
|
||||
form.AddDurationField(
|
||||
"scan_interval",
|
||||
"Scan Interval",
|
||||
"How often to scan for volumes needing vacuum",
|
||||
config.ScanIntervalSeconds,
|
||||
secondsToDuration(config.ScanIntervalSeconds),
|
||||
true,
|
||||
)
|
||||
|
||||
form.AddIntervalField(
|
||||
form.AddDurationField(
|
||||
"min_volume_age",
|
||||
"Minimum Volume Age",
|
||||
"Only vacuum volumes older than this duration",
|
||||
config.MinVolumeAgeSeconds,
|
||||
secondsToDuration(config.MinVolumeAgeSeconds),
|
||||
true,
|
||||
)
|
||||
|
||||
@@ -157,7 +157,7 @@ function resetForm() {
|
||||
if (confirm('Reset all vacuum settings to defaults?')) {
|
||||
// Reset to default values
|
||||
document.querySelector('input[name="enabled"]').checked = true;
|
||||
document.querySelector('input[name="garbage_threshold"]').value = '0.3';
|
||||
document.querySelector('input[name="garbage_threshold"]').value = '30';
|
||||
document.querySelector('input[name="scan_interval"]').value = '30m';
|
||||
document.querySelector('input[name="min_volume_age"]').value = '1h';
|
||||
document.querySelector('input[name="max_concurrent"]').value = '2';
|
||||
@@ -177,47 +177,33 @@ func (ui *UIProvider) ParseConfigForm(formData map[string][]string) (interface{}
|
||||
// Parse enabled checkbox
|
||||
config.Enabled = len(formData["enabled"]) > 0 && formData["enabled"][0] == "on"
|
||||
|
||||
// Parse garbage threshold
|
||||
// Parse garbage threshold (convert from 0-100 to 0.0-1.0)
|
||||
if thresholdStr := formData["garbage_threshold"]; len(thresholdStr) > 0 {
|
||||
if threshold, err := strconv.ParseFloat(thresholdStr[0], 64); err != nil {
|
||||
return nil, fmt.Errorf("invalid garbage threshold: %w", err)
|
||||
} else if threshold < 0 || threshold > 1 {
|
||||
return nil, fmt.Errorf("garbage threshold must be between 0.0 and 1.0")
|
||||
return nil, fmt.Errorf("invalid garbage percentage threshold: %w", err)
|
||||
} else if threshold < 0 || threshold > 100 {
|
||||
return nil, fmt.Errorf("garbage percentage threshold must be between 0 and 100")
|
||||
} else {
|
||||
config.GarbageThreshold = threshold
|
||||
config.GarbageThreshold = threshold / 100.0 // Convert percentage to decimal
|
||||
}
|
||||
}
|
||||
|
||||
// Parse scan interval
|
||||
if values, ok := formData["scan_interval_value"]; ok && len(values) > 0 {
|
||||
value, err := strconv.Atoi(values[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid scan interval value: %w", err)
|
||||
if intervalStr := formData["scan_interval"]; len(intervalStr) > 0 {
|
||||
if interval, err := time.ParseDuration(intervalStr[0]); err != nil {
|
||||
return nil, fmt.Errorf("invalid scan interval: %w", err)
|
||||
} else {
|
||||
config.ScanIntervalSeconds = durationToSeconds(interval)
|
||||
}
|
||||
|
||||
unit := "minute" // default
|
||||
if units, ok := formData["scan_interval_unit"]; ok && len(units) > 0 {
|
||||
unit = units[0]
|
||||
}
|
||||
|
||||
// Convert to seconds
|
||||
config.ScanIntervalSeconds = types.IntervalValueUnitToSeconds(value, unit)
|
||||
}
|
||||
|
||||
// Parse min volume age
|
||||
if values, ok := formData["min_volume_age_value"]; ok && len(values) > 0 {
|
||||
value, err := strconv.Atoi(values[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid min volume age value: %w", err)
|
||||
if ageStr := formData["min_volume_age"]; len(ageStr) > 0 {
|
||||
if age, err := time.ParseDuration(ageStr[0]); err != nil {
|
||||
return nil, fmt.Errorf("invalid min volume age: %w", err)
|
||||
} else {
|
||||
config.MinVolumeAgeSeconds = durationToSeconds(age)
|
||||
}
|
||||
|
||||
unit := "minute" // default
|
||||
if units, ok := formData["min_volume_age_unit"]; ok && len(units) > 0 {
|
||||
unit = units[0]
|
||||
}
|
||||
|
||||
// Convert to seconds
|
||||
config.MinVolumeAgeSeconds = types.IntervalValueUnitToSeconds(value, unit)
|
||||
}
|
||||
|
||||
// Parse max concurrent
|
||||
|
Reference in New Issue
Block a user