mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-11-24 08:46:54 +08:00
Self-Contained Design
To prove the system is truly self-contained: To add a new task: Create a task package (e.g., worker/tasks/compression/) Import it: _ "github.com/.../worker/tasks/compression" That's it! No other changes needed. To remove a task: Delete the task package directory Remove the import line That's it! No other changes needed.
This commit is contained in:
@@ -455,6 +455,44 @@ func (cp *ConfigPersistence) IsConfigured() bool {
|
||||
return cp.dataDir != ""
|
||||
}
|
||||
|
||||
// SaveTaskPolicyGeneric saves a task policy for any task type dynamically
|
||||
func (cp *ConfigPersistence) SaveTaskPolicyGeneric(taskType string, policy *worker_pb.TaskPolicy) error {
|
||||
filename := fmt.Sprintf("task_%s.pb", taskType)
|
||||
return cp.saveTaskConfig(filename, policy)
|
||||
}
|
||||
|
||||
// LoadTaskPolicyGeneric loads a task policy for any task type dynamically
|
||||
func (cp *ConfigPersistence) LoadTaskPolicyGeneric(taskType string) (*worker_pb.TaskPolicy, error) {
|
||||
filename := fmt.Sprintf("task_%s.pb", taskType)
|
||||
|
||||
if cp.dataDir == "" {
|
||||
return nil, fmt.Errorf("no data directory configured")
|
||||
}
|
||||
|
||||
confDir := filepath.Join(cp.dataDir, ConfigSubdir)
|
||||
configPath := filepath.Join(confDir, filename)
|
||||
|
||||
// Check if file exists
|
||||
if _, err := os.Stat(configPath); os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("no configuration found for task type: %s", taskType)
|
||||
}
|
||||
|
||||
// Read file
|
||||
configData, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read task config file: %w", err)
|
||||
}
|
||||
|
||||
// Unmarshal as TaskPolicy
|
||||
var policy worker_pb.TaskPolicy
|
||||
if err := proto.Unmarshal(configData, &policy); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal task configuration: %w", err)
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Loaded task policy for %s from %s", taskType, configPath)
|
||||
return &policy, nil
|
||||
}
|
||||
|
||||
// GetConfigInfo returns information about the configuration storage
|
||||
func (cp *ConfigPersistence) GetConfigInfo() map[string]interface{} {
|
||||
info := map[string]interface{}{
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
@@ -230,27 +230,15 @@ func (h *MaintenanceHandlers) UpdateTaskConfig(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// Create a new config instance based on task type and apply schema defaults
|
||||
var config TaskConfig
|
||||
switch taskType {
|
||||
case types.TaskTypeErasureCoding:
|
||||
config = &erasure_coding.Config{}
|
||||
default:
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Unsupported task type: " + taskTypeName})
|
||||
return
|
||||
}
|
||||
|
||||
// Apply schema defaults first using type-safe method
|
||||
if err := schema.ApplyDefaultsToConfig(config); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to apply defaults: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// First, get the current configuration to preserve existing values
|
||||
// Get the config instance from the UI provider - this is a dynamic approach
|
||||
// that doesn't require hardcoding task types
|
||||
currentUIRegistry := tasks.GetGlobalUIRegistry()
|
||||
currentTypesRegistry := tasks.GetGlobalTypesRegistry()
|
||||
|
||||
var config types.TaskConfig
|
||||
var currentProvider types.TaskUIProvider
|
||||
|
||||
// Find the UI provider for this task type
|
||||
for workerTaskType := range currentTypesRegistry.GetAllDetectors() {
|
||||
if string(workerTaskType) == string(taskType) {
|
||||
currentProvider = currentUIRegistry.GetProvider(workerTaskType)
|
||||
@@ -258,16 +246,26 @@ func (h *MaintenanceHandlers) UpdateTaskConfig(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
if currentProvider != nil {
|
||||
// Copy current config values to the new config
|
||||
currentConfig := currentProvider.GetCurrentConfig()
|
||||
if currentConfigProtobuf, ok := currentConfig.(TaskConfig); ok {
|
||||
// Apply current values using protobuf directly - no map conversion needed!
|
||||
currentPolicy := currentConfigProtobuf.ToTaskPolicy()
|
||||
if err := config.FromTaskPolicy(currentPolicy); err != nil {
|
||||
glog.Warningf("Failed to load current config for %s: %v", taskTypeName, err)
|
||||
}
|
||||
}
|
||||
if currentProvider == nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Unsupported task type: " + taskTypeName})
|
||||
return
|
||||
}
|
||||
|
||||
// Get a config instance from the UI provider
|
||||
config = currentProvider.GetCurrentConfig()
|
||||
if config == nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get config for task type: " + taskTypeName})
|
||||
return
|
||||
}
|
||||
|
||||
// Apply schema defaults - config instances should already have defaults applied during creation
|
||||
glog.V(2).Infof("Using config defaults for task type: %s", taskTypeName)
|
||||
|
||||
// Copy current config values (currentProvider is already set above)
|
||||
// Apply current values using protobuf directly - no map conversion needed!
|
||||
currentPolicy := config.ToTaskPolicy()
|
||||
if err := config.FromTaskPolicy(currentPolicy); err != nil {
|
||||
glog.Warningf("Failed to load current config for %s: %v", taskTypeName, err)
|
||||
}
|
||||
|
||||
// Parse form data using schema-based approach (this will override with new values)
|
||||
@@ -277,14 +275,8 @@ func (h *MaintenanceHandlers) UpdateTaskConfig(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// Debug logging - show parsed config values
|
||||
switch taskType {
|
||||
case types.TaskTypeErasureCoding:
|
||||
if ecConfig, ok := config.(*erasure_coding.Config); ok {
|
||||
glog.V(1).Infof("Parsed EC config - FullnessRatio: %f, QuietForSeconds: %d, MinSizeMB: %d, CollectionFilter: '%s'",
|
||||
ecConfig.FullnessRatio, ecConfig.QuietForSeconds, ecConfig.MinSizeMB, ecConfig.CollectionFilter)
|
||||
}
|
||||
}
|
||||
// Debug logging - config parsed for task type
|
||||
glog.V(1).Infof("Parsed configuration for task type: %s", taskTypeName)
|
||||
|
||||
// Validate the configuration
|
||||
if validationErrors := schema.ValidateConfig(config); len(validationErrors) > 0 {
|
||||
@@ -553,7 +545,7 @@ func (h *MaintenanceHandlers) updateMaintenanceConfig(config *maintenance.Mainte
|
||||
}
|
||||
|
||||
// saveTaskConfigToProtobuf saves task configuration to protobuf file
|
||||
func (h *MaintenanceHandlers) saveTaskConfigToProtobuf(taskType types.TaskType, config TaskConfig) error {
|
||||
func (h *MaintenanceHandlers) saveTaskConfigToProtobuf(taskType types.TaskType, config types.TaskConfig) error {
|
||||
configPersistence := h.adminServer.GetConfigPersistence()
|
||||
if configPersistence == nil {
|
||||
return fmt.Errorf("config persistence not available")
|
||||
@@ -562,11 +554,6 @@ func (h *MaintenanceHandlers) saveTaskConfigToProtobuf(taskType types.TaskType,
|
||||
// Use the new ToTaskPolicy method - much simpler and more maintainable!
|
||||
taskPolicy := config.ToTaskPolicy()
|
||||
|
||||
// Save using task-specific methods
|
||||
switch taskType {
|
||||
case types.TaskTypeErasureCoding:
|
||||
return configPersistence.SaveErasureCodingTaskPolicy(taskPolicy)
|
||||
default:
|
||||
return fmt.Errorf("unsupported task type for protobuf persistence: %s", taskType)
|
||||
}
|
||||
// Save using generic method - no more hardcoded task types!
|
||||
return configPersistence.SaveTaskPolicyGeneric(string(taskType), taskPolicy)
|
||||
}
|
||||
|
||||
@@ -84,11 +84,10 @@ func (at *ActiveTopology) isDiskAvailable(disk *activeDisk, taskType TaskType) b
|
||||
|
||||
// areTaskTypesConflicting checks if two task types conflict
|
||||
func (at *ActiveTopology) areTaskTypesConflicting(existing, new TaskType) bool {
|
||||
// Examples of conflicting task types
|
||||
// Define conflicting task types dynamically
|
||||
// For now, assume no task types conflict (can be made configurable later)
|
||||
conflictMap := map[TaskType][]TaskType{
|
||||
TaskTypeVacuum: {TaskTypeBalance, TaskTypeErasureCoding},
|
||||
TaskTypeBalance: {TaskTypeVacuum, TaskTypeErasureCoding},
|
||||
TaskTypeErasureCoding: {TaskTypeVacuum, TaskTypeBalance},
|
||||
// No conflicts defined currently - this can be made configurable per task
|
||||
}
|
||||
|
||||
if conflicts, exists := conflictMap[existing]; exists {
|
||||
|
||||
@@ -7,30 +7,21 @@ import (
|
||||
|
||||
// CalculateTaskStorageImpact calculates storage impact for different task types
|
||||
func CalculateTaskStorageImpact(taskType TaskType, volumeSize int64) (sourceChange, targetChange StorageSlotChange) {
|
||||
switch taskType {
|
||||
case TaskTypeErasureCoding:
|
||||
switch string(taskType) {
|
||||
case "erasure_coding":
|
||||
// EC task: distributes shards to MULTIPLE targets, source reserves with zero impact
|
||||
// Source reserves capacity but with zero StorageSlotChange (no actual capacity consumption during planning)
|
||||
// WARNING: EC has multiple targets! Use AddPendingTask with multiple destinations for proper multi-target handling
|
||||
// WARNING: EC has multiple targets! Use AddPendingTask with multiple destinations for proper multi-destination calculation
|
||||
// This simplified function returns zero impact; real EC requires specialized multi-destination calculation
|
||||
return StorageSlotChange{VolumeSlots: 0, ShardSlots: 0}, StorageSlotChange{VolumeSlots: 0, ShardSlots: 0}
|
||||
|
||||
case TaskTypeBalance:
|
||||
// Balance task: moves volume from source to target
|
||||
// Source loses 1 volume, target gains 1 volume
|
||||
return StorageSlotChange{VolumeSlots: -1, ShardSlots: 0}, StorageSlotChange{VolumeSlots: 1, ShardSlots: 0}
|
||||
|
||||
case TaskTypeVacuum:
|
||||
// Vacuum task: frees space by removing deleted entries, no slot change
|
||||
return StorageSlotChange{VolumeSlots: 0, ShardSlots: 0}, StorageSlotChange{VolumeSlots: 0, ShardSlots: 0}
|
||||
|
||||
case TaskTypeReplication:
|
||||
case "replication":
|
||||
// Replication task: creates new replica on target
|
||||
return StorageSlotChange{VolumeSlots: 0, ShardSlots: 0}, StorageSlotChange{VolumeSlots: 1, ShardSlots: 0}
|
||||
|
||||
default:
|
||||
// Unknown task type, assume minimal impact
|
||||
glog.Warningf("unhandled task type %s in CalculateTaskStorageImpact, assuming default impact", taskType)
|
||||
glog.V(2).Infof("Task type %s not specifically handled in CalculateTaskStorageImpact, using default impact", taskType)
|
||||
return StorageSlotChange{VolumeSlots: 0, ShardSlots: 0}, StorageSlotChange{VolumeSlots: 1, ShardSlots: 0}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,16 +203,16 @@ func (at *ActiveTopology) AddPendingTask(spec TaskSpec) error {
|
||||
|
||||
// calculateSourceStorageImpact calculates storage impact for sources based on task type and cleanup type
|
||||
func (at *ActiveTopology) calculateSourceStorageImpact(taskType TaskType, cleanupType SourceCleanupType, volumeSize int64) StorageSlotChange {
|
||||
switch taskType {
|
||||
case TaskTypeErasureCoding:
|
||||
switch string(taskType) {
|
||||
case "erasure_coding":
|
||||
switch cleanupType {
|
||||
case CleanupVolumeReplica:
|
||||
impact, _ := CalculateTaskStorageImpact(TaskTypeErasureCoding, volumeSize)
|
||||
impact, _ := CalculateTaskStorageImpact(taskType, volumeSize)
|
||||
return impact
|
||||
case CleanupECShards:
|
||||
return CalculateECShardCleanupImpact(volumeSize)
|
||||
default:
|
||||
impact, _ := CalculateTaskStorageImpact(TaskTypeErasureCoding, volumeSize)
|
||||
impact, _ := CalculateTaskStorageImpact(taskType, volumeSize)
|
||||
return impact
|
||||
}
|
||||
default:
|
||||
|
||||
@@ -3,19 +3,12 @@ package topology
|
||||
import "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
|
||||
// TaskType represents different types of maintenance operations
|
||||
// Task types are now dynamically registered - use the worker/types package for task type operations
|
||||
type TaskType string
|
||||
|
||||
// TaskStatus represents the current status of a task
|
||||
type TaskStatus string
|
||||
|
||||
// Common task type constants
|
||||
const (
|
||||
TaskTypeVacuum TaskType = "vacuum"
|
||||
TaskTypeBalance TaskType = "balance"
|
||||
TaskTypeErasureCoding TaskType = "erasure_coding"
|
||||
TaskTypeReplication TaskType = "replication"
|
||||
)
|
||||
|
||||
// Common task status constants
|
||||
const (
|
||||
TaskStatusPending TaskStatus = "pending"
|
||||
|
||||
Reference in New Issue
Block a user