lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-11-24 16:53:14 +08:00
Code Issues Actions 17 Packages Projects Releases Wiki Activity

fix min volume age

Browse Source
This commit is contained in:
chrislu
2025-08-11 01:54:10 -07:00
parent 424bb7fe11
commit 9df006b49d
5 changed files with 102 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/admin/maintenance/maintenance_queue.go
View File

@@ -272,7 +272,7 @@ func (mq *MaintenanceQueue) GetNextTask(workerID string, capabilities []Maintena
// If no task found, return nil
if selectedTask == nil {
glog.V(2).Infof("No suitable tasks available for worker %s (checked %d pending tasks)", workerID, len(mq.pendingTasks))
glog.V(3).Infof("No suitable tasks available for worker %s (checked %d pending tasks)", workerID, len(mq.pendingTasks))
return nil
}

74
weed/admin/view/app/task_config_schema.templ
View File

@@ -6,9 +6,9 @@ import (
"fmt"
"reflect"
"strings"
"github.com/seaweedfs/seaweedfs/weed/admin/config"
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/admin/config"
"github.com/seaweedfs/seaweedfs/weed/admin/view/components"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
)
@@ -207,7 +207,7 @@ templ TaskConfigField(field *config.Field, config interface{}) {
class="form-control"
id={ field.JSONName + "_value" }
name={ field.JSONName + "_value" }
value={ fmt.Sprintf("%.0f", components.ConvertInt32SecondsToDisplayValue(getTaskConfigInt32Field(config, field.JSONName))) }
value={ fmt.Sprintf("%.0f", components.ConvertInt32SecondsToDisplayValue(getTaskConfigInt32FieldWithDefault(config, field))) }
step="1"
min="1"
if field.Required {
@@ -223,30 +223,30 @@ templ TaskConfigField(field *config.Field, config interface{}) {
required
}
>
<option
value="minutes"
if components.GetInt32DisplayUnit(getTaskConfigInt32Field(config, field.JSONName)) == "minutes" {
selected
}
>
Minutes
</option>
<option
value="hours"
if components.GetInt32DisplayUnit(getTaskConfigInt32Field(config, field.JSONName)) == "hours" {
selected
}
>
Hours
</option>
<option
value="days"
if components.GetInt32DisplayUnit(getTaskConfigInt32Field(config, field.JSONName)) == "days" {
selected
}
>
Days
</option>
<option
value="minutes"
if components.GetInt32DisplayUnit(getTaskConfigInt32FieldWithDefault(config, field)) == "minutes" {
selected
}
>
Minutes
</option>
<option
value="hours"
if components.GetInt32DisplayUnit(getTaskConfigInt32FieldWithDefault(config, field)) == "hours" {
selected
}
>
Hours
</option>
<option
value="days"
if components.GetInt32DisplayUnit(getTaskConfigInt32FieldWithDefault(config, field)) == "days" {
selected
}
>
Days
</option>
</select>
</div>
if field.Description != "" {
@@ -388,6 +388,26 @@ func getTaskConfigInt32Field(config interface{}, fieldName string) int32 {
}
}
func getTaskConfigInt32FieldWithDefault(config interface{}, field *config.Field) int32 {
value := getTaskConfigInt32Field(config, field.JSONName)
// If no value is stored (value is 0), use the schema default
if value == 0 && field.DefaultValue != nil {
switch defaultVal := field.DefaultValue.(type) {
case int:
return int32(defaultVal)
case int32:
return defaultVal
case int64:
return int32(defaultVal)
case float64:
return int32(defaultVal)
}
}
return value
}
func getTaskConfigFloatField(config interface{}, fieldName string) float64 {
if value := getTaskFieldValue(config, fieldName); value != nil {
switch v := value.(type) {
@@ -429,7 +449,7 @@ func getTaskConfigStringField(config interface{}, fieldName string) string {
}
func getTaskNumberStep(field *config.Field) string {
if field.Type == config.FieldTypeFloat {
if field.Type == "float" {
return "0.01"
}
return "1"

32
weed/admin/view/app/task_config_schema_templ.go
View File

@@ -281,9 +281,9 @@ func TaskConfigField(field *config.Field, config interface{}) templ.Component {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var14 string
templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.0f", components.ConvertInt32SecondsToDisplayValue(getTaskConfigInt32Field(config, field.JSONName))))
templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.0f", components.ConvertInt32SecondsToDisplayValue(getTaskConfigInt32FieldWithDefault(config, field))))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/task_config_schema.templ`, Line: 210, Col: 142}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/task_config_schema.templ`, Line: 210, Col: 144}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
if templ_7745c5c3_Err != nil {
@@ -339,7 +339,7 @@ func TaskConfigField(field *config.Field, config interface{}) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if components.GetInt32DisplayUnit(getTaskConfigInt32Field(config, field.JSONName)) == "minutes" {
if components.GetInt32DisplayUnit(getTaskConfigInt32FieldWithDefault(config, field)) == "minutes" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, " selected")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
@@ -349,7 +349,7 @@ func TaskConfigField(field *config.Field, config interface{}) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if components.GetInt32DisplayUnit(getTaskConfigInt32Field(config, field.JSONName)) == "hours" {
if components.GetInt32DisplayUnit(getTaskConfigInt32FieldWithDefault(config, field)) == "hours" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, " selected")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
@@ -359,7 +359,7 @@ func TaskConfigField(field *config.Field, config interface{}) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if components.GetInt32DisplayUnit(getTaskConfigInt32Field(config, field.JSONName)) == "days" {
if components.GetInt32DisplayUnit(getTaskConfigInt32FieldWithDefault(config, field)) == "days" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, " selected")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
@@ -849,6 +849,26 @@ func getTaskConfigInt32Field(config interface{}, fieldName string) int32 {
}
}
func getTaskConfigInt32FieldWithDefault(config interface{}, field *config.Field) int32 {
value := getTaskConfigInt32Field(config, field.JSONName)
// If no value is stored (value is 0), use the schema default
if value == 0 && field.DefaultValue != nil {
switch defaultVal := field.DefaultValue.(type) {
case int:
return int32(defaultVal)
case int32:
return defaultVal
case int64:
return int32(defaultVal)
case float64:
return int32(defaultVal)
}
}
return value
}
func getTaskConfigFloatField(config interface{}, fieldName string) float64 {
if value := getTaskFieldValue(config, fieldName); value != nil {
switch v := value.(type) {
@@ -890,7 +910,7 @@ func getTaskConfigStringField(config interface{}, fieldName string) string {
}
func getTaskNumberStep(field *config.Field) string {
if field.Type == config.FieldTypeFloat {
if field.Type == "float" {
return "0.01"
}
return "1"

26
weed/worker/tasks/ec_vacuum/config.go
View File

@@ -12,10 +12,10 @@ import (
// Config extends BaseConfig with EC vacuum specific settings
type Config struct {
base.BaseConfig
DeletionThreshold float64 `json:"deletion_threshold"` // Minimum deletion ratio to trigger vacuum
MinVolumeAgeHours int `json:"min_volume_age_hours"` // Minimum age before considering vacuum
CollectionFilter string `json:"collection_filter"` // Filter by collection
MinSizeMB int `json:"min_size_mb"` // Minimum original volume size
DeletionThreshold float64 `json:"deletion_threshold"` // Minimum deletion ratio to trigger vacuum
MinVolumeAgeSeconds int `json:"min_volume_age_seconds"` // Minimum age before considering vacuum (in seconds)
CollectionFilter string `json:"collection_filter"` // Filter by collection
MinSizeMB int `json:"min_size_mb"` // Minimum original volume size
}
// NewDefaultConfig creates a new default EC vacuum configuration
@@ -26,10 +26,10 @@ func NewDefaultConfig() *Config {
ScanIntervalSeconds: 24 * 60 * 60, // 24 hours
MaxConcurrent: 1,
},
DeletionThreshold: 0.3, // 30% deletions trigger vacuum
MinVolumeAgeHours: 72, // 3 days minimum age
CollectionFilter: "", // No filter by default
MinSizeMB: 100, // 100MB minimum size
DeletionThreshold: 0.3, // 30% deletions trigger vacuum
MinVolumeAgeSeconds: 72 * 60 * 60, // 3 days minimum age (72 hours in seconds)
CollectionFilter: "", // No filter by default
MinSizeMB: 100, // 100MB minimum size
}
}
@@ -98,12 +98,12 @@ func GetConfigSpec() base.ConfigSpec {
CSSClasses: "form-control",
},
{
Name: "min_volume_age_hours",
JSONName: "min_volume_age_hours",
Name: "min_volume_age_seconds",
JSONName: "min_volume_age_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 72,
MinValue: 24,
MaxValue: 30 * 24, // 30 days
DefaultValue: 72 * 60 * 60, // 72 hours in seconds
MinValue: 24 * 60 * 60, // 24 hours in seconds
MaxValue: 30 * 24 * 60 * 60, // 30 days in seconds
Required: true,
DisplayName: "Minimum Volume Age",
Description: "Minimum age before considering EC volume for vacuum",

23
weed/worker/tasks/ec_vacuum/detection.go
View File

@@ -63,9 +63,9 @@ func Detection(metrics []*wtypes.VolumeHealthMetrics, info *wtypes.ClusterInfo,
Server: ecInfo.PrimaryNode,
Collection: ecInfo.Collection,
Priority: wtypes.TaskPriorityLow, // EC vacuum is not urgent
Reason: fmt.Sprintf("EC volume needs vacuum: deletion_ratio=%.1f%% (>%.1f%%), age=%.1fh (>%dh), size=%.1fMB (>%dMB)",
Reason: fmt.Sprintf("EC volume needs vacuum: deletion_ratio=%.1f%% (>%.1f%%), age=%.1fh (>%.1fh), size=%.1fMB (>%dMB)",
deletionRatio*100, ecVacuumConfig.DeletionThreshold*100,
ecInfo.Age.Hours(), ecVacuumConfig.MinVolumeAgeHours,
ecInfo.Age.Hours(), (time.Duration(ecVacuumConfig.MinVolumeAgeSeconds) * time.Second).Hours(),
float64(ecInfo.Size)/(1024*1024), ecVacuumConfig.MinSizeMB),
ScheduleAt: now,
}
@@ -98,12 +98,18 @@ func Detection(metrics []*wtypes.VolumeHealthMetrics, info *wtypes.ClusterInfo,
deletionRatio := calculateDeletionRatio(ecInfo)
sizeMB := float64(ecInfo.Size) / (1024 * 1024)
deletedMB := deletionRatio * sizeMB
ageRequired := time.Duration(ecVacuumConfig.MinVolumeAgeHours) * time.Hour
ageRequired := time.Duration(ecVacuumConfig.MinVolumeAgeSeconds) * time.Second
glog.Infof("EC VACUUM: Volume %d: deleted=%.1fMB, ratio=%.1f%% (need ≥%.1f%%), age=%s (need ≥%s), size=%.1fMB (need ≥%dMB)",
// Check shard availability
totalShards := 0
for _, shardBits := range ecInfo.ShardNodes {
totalShards += shardBits.ShardIdCount()
}
glog.Infof("EC VACUUM: Volume %d: deleted=%.1fMB, ratio=%.1f%% (need ≥%.1f%%), age=%s (need ≥%s), size=%.1fMB (need ≥%dMB), shards=%d (need ≥%d)",
volumeID, deletedMB, deletionRatio*100, ecVacuumConfig.DeletionThreshold*100,
ecInfo.Age.Truncate(time.Minute), ageRequired.Truncate(time.Minute),
sizeMB, ecVacuumConfig.MinSizeMB)
sizeMB, ecVacuumConfig.MinSizeMB, totalShards, erasure_coding.DataShardsCount)
count++
}
}
@@ -173,9 +179,10 @@ func collectEcVolumeInfo(metrics []*wtypes.VolumeHealthMetrics) map[uint32]*EcVo
// shouldVacuumEcVolume determines if an EC volume should be considered for vacuum
func shouldVacuumEcVolume(ecInfo *EcVolumeInfo, config *Config, now time.Time) bool {
// Check minimum age
if ecInfo.Age < time.Duration(config.MinVolumeAgeHours)*time.Hour {
glog.V(3).Infof("EC volume %d too young: age=%.1fh < %dh",
ecInfo.VolumeID, ecInfo.Age.Hours(), config.MinVolumeAgeHours)
minAge := time.Duration(config.MinVolumeAgeSeconds) * time.Second
if ecInfo.Age < minAge {
glog.V(3).Infof("EC volume %d too young: age=%.1fh < %.1fh",
ecInfo.VolumeID, ecInfo.Age.Hours(), minAge.Hours())
return false
}