integration test with vauuming

This commit is contained in:
chrislu
2025-07-26 13:23:43 -07:00
parent 600bfbf671
commit a1e508cd8d
5 changed files with 570 additions and 5 deletions

View File

@@ -1,7 +1,7 @@
# SeaweedFS Admin Integration Test Makefile
# Tests the admin server and worker functionality using official weed commands
.PHONY: help build build-and-restart restart-workers start stop restart logs clean status test admin-ui worker-logs master-logs admin-logs
.PHONY: help build build-and-restart restart-workers start stop restart logs clean status test admin-ui worker-logs master-logs admin-logs vacuum-test vacuum-demo vacuum-status vacuum-data vacuum-data-high vacuum-data-low vacuum-continuous vacuum-clean vacuum-help
.DEFAULT_GOAL := help
COMPOSE_FILE := docker-compose-ec-test.yml
@@ -39,8 +39,24 @@ help: ## Show this help message
@echo "================================"
@echo "Tests admin server task distribution to workers using official weed commands"
@echo ""
@echo "Available targets:"
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-15s %s\n", $$1, $$2}'
@echo "🏗️ Cluster Management:"
@grep -E '^(start|stop|restart|clean|status|build):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
@echo ""
@echo "🧪 Testing:"
@grep -E '^(test|demo|validate|quick-test):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
@echo ""
@echo "🗑️ Vacuum Testing:"
@grep -E '^vacuum-.*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
@echo ""
@echo "📜 Monitoring:"
@grep -E '^(logs|admin-logs|worker-logs|master-logs|admin-ui):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
@echo ""
@echo "🚀 Quick Start:"
@echo " make start # Start cluster"
@echo " make vacuum-test # Test vacuum tasks"
@echo " make vacuum-help # Vacuum testing guide"
@echo ""
@echo "💡 For detailed vacuum testing: make vacuum-help"
start: ## Start the complete SeaweedFS cluster with admin and workers
@echo "🚀 Starting SeaweedFS cluster with admin and workers..."
@@ -219,4 +235,112 @@ demo: start ## Start cluster and run demonstration
@echo " • Tasks complete with shard distribution"
@echo ""
@echo "💡 Use 'make worker-logs' to see worker activity"
@echo "💡 Use 'make admin-logs' to see admin task management"
@echo "💡 Use 'make admin-logs' to see admin task management"
# Vacuum Testing Targets
vacuum-test: ## Create test data with garbage and verify vacuum detection
@echo "🧪 SeaweedFS Vacuum Task Testing"
@echo "================================"
@echo ""
@echo "1⃣ Checking cluster health..."
@curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "✅ Master ready" || (echo "❌ Master not ready. Run 'make start' first." && exit 1)
@curl -s http://localhost:23646/ | grep -q "Admin" && echo "✅ Admin ready" || (echo "❌ Admin not ready. Run 'make start' first." && exit 1)
@echo ""
@echo "2⃣ Creating test data with garbage..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=25 -delete=0.5 -size=200
@echo ""
@echo "3⃣ Configuration Instructions:"
@echo " Visit: http://localhost:23646/maintenance/config/vacuum"
@echo " Set for testing:"
@echo " • Enable Vacuum Tasks: ✅ Checked"
@echo " • Garbage Threshold: 0.20 (20%)"
@echo " • Scan Interval: [30] [Seconds]"
@echo " • Min Volume Age: [0] [Minutes]"
@echo " • Max Concurrent: 2"
@echo ""
@echo "4⃣ Monitor vacuum tasks at: http://localhost:23646/maintenance"
@echo ""
@echo "💡 Use 'make vacuum-status' to check volume garbage ratios"
vacuum-demo: ## Run automated vacuum testing demonstration
@echo "🎭 Vacuum Task Demo"
@echo "=================="
@echo ""
@echo "⚠️ This demo requires user interaction for configuration"
@echo "💡 Make sure cluster is running with 'make start'"
@echo ""
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester sh -c "chmod +x demo_vacuum_testing.sh && ./demo_vacuum_testing.sh"
vacuum-status: ## Check current volume status and garbage ratios
@echo "📊 Current Volume Status"
@echo "======================="
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester sh -c "chmod +x check_volumes.sh && ./check_volumes.sh"
vacuum-data: ## Create test data with configurable parameters
@echo "📁 Creating vacuum test data..."
@echo "Usage: make vacuum-data [FILES=20] [DELETE=0.4] [SIZE=100]"
@echo ""
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go \
-files=$${FILES:-20} \
-delete=$${DELETE:-0.4} \
-size=$${SIZE:-100}
vacuum-data-high: ## Create high garbage ratio test data (should trigger vacuum)
@echo "📁 Creating high garbage test data (70% garbage)..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=30 -delete=0.7 -size=150
vacuum-data-low: ## Create low garbage ratio test data (should NOT trigger vacuum)
@echo "📁 Creating low garbage test data (15% garbage)..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=30 -delete=0.15 -size=150
vacuum-continuous: ## Generate garbage continuously for testing
@echo "🔄 Generating continuous garbage for vacuum testing..."
@echo "Creating 5 rounds of test data with 30-second intervals..."
@for i in {1..5}; do \
echo "Round $$i: Creating garbage..."; \
docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=10 -delete=0.6 -size=100; \
echo "Waiting 30 seconds..."; \
sleep 30; \
done
@echo "✅ Continuous test complete. Check vacuum task activity!"
vacuum-clean: ## Clean up vacuum test data (removes all volumes!)
@echo "🧹 Cleaning up vacuum test data..."
@echo "⚠️ WARNING: This will delete ALL volumes!"
@read -p "Are you sure? (y/N): " confirm && [ "$$confirm" = "y" ] || exit 1
@echo "Stopping cluster..."
@docker-compose -f $(COMPOSE_FILE) down
@echo "Removing volume data..."
@rm -rf data/volume*/
@echo "Restarting cluster..."
@docker-compose -f $(COMPOSE_FILE) up -d
@echo "✅ Clean up complete. Fresh volumes ready for testing."
vacuum-help: ## Show vacuum testing help and examples
@echo "🧪 Vacuum Testing Commands (Docker-based)"
@echo "=========================================="
@echo ""
@echo "Quick Start:"
@echo " make start # Start SeaweedFS cluster with vacuum-tester"
@echo " make vacuum-test # Create test data and instructions"
@echo " make vacuum-status # Check volume status"
@echo ""
@echo "Data Generation:"
@echo " make vacuum-data-high # High garbage (should trigger)"
@echo " make vacuum-data-low # Low garbage (should NOT trigger)"
@echo " make vacuum-continuous # Continuous garbage generation"
@echo ""
@echo "Monitoring:"
@echo " make vacuum-status # Quick volume status check"
@echo " make vacuum-demo # Full guided demonstration"
@echo ""
@echo "Configuration:"
@echo " Visit: http://localhost:23646/maintenance/config/vacuum"
@echo " Monitor: http://localhost:23646/maintenance"
@echo ""
@echo "Custom Parameters:"
@echo " make vacuum-data FILES=50 DELETE=0.8 SIZE=200"
@echo ""
@echo "💡 All commands now run inside Docker containers"
@echo "Documentation:"
@echo " See: VACUUM_TEST_README.md for complete guide"

View File

@@ -0,0 +1,32 @@
#!/bin/sh
echo "📊 Quick Volume Status Check"
echo "============================"
echo ""
# Check if master is running
MASTER_URL="${MASTER_HOST:-master:9333}"
if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then
echo "❌ Master server not available at $MASTER_URL"
exit 1
fi
echo "🔍 Fetching volume status from master..."
curl -s "http://$MASTER_URL/vol/status" | jq -r '
if .Volumes and .Volumes.DataCenters then
.Volumes.DataCenters | to_entries[] | .value | to_entries[] | .value | to_entries[] | .value | if . then .[] else empty end |
"Volume \(.Id):
Size: \(.Size | if . < 1024 then "\(.) B" elif . < 1048576 then "\(. / 1024 | floor) KB" elif . < 1073741824 then "\(. / 1048576 * 100 | floor / 100) MB" else "\(. / 1073741824 * 100 | floor / 100) GB" end)
Files: \(.FileCount) active, \(.DeleteCount) deleted
Garbage: \(.DeletedByteCount | if . < 1024 then "\(.) B" elif . < 1048576 then "\(. / 1024 | floor) KB" elif . < 1073741824 then "\(. / 1048576 * 100 | floor / 100) MB" else "\(. / 1073741824 * 100 | floor / 100) GB" end) (\(if .Size > 0 then (.DeletedByteCount / .Size * 100 | floor) else 0 end)%)
Status: \(if (.DeletedByteCount / .Size * 100) > 30 then "🎯 NEEDS VACUUM" else "✅ OK" end)
"
else
"No volumes found"
end'
echo ""
echo "💡 Legend:"
echo " 🎯 NEEDS VACUUM: >30% garbage ratio"
echo " ✅ OK: <30% garbage ratio"
echo ""

View File

@@ -0,0 +1,279 @@
package main
import (
"bytes"
"crypto/rand"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net/http"
"time"
)
var (
master = flag.String("master", "master:9333", "SeaweedFS master server address")
fileCount = flag.Int("files", 20, "Number of files to create")
deleteRatio = flag.Float64("delete", 0.4, "Ratio of files to delete (0.0-1.0)")
fileSizeKB = flag.Int("size", 100, "Size of each file in KB")
)
type AssignResult struct {
Fid string `json:"fid"`
Url string `json:"url"`
PublicUrl string `json:"publicUrl"`
Count int `json:"count"`
Error string `json:"error"`
}
func main() {
flag.Parse()
fmt.Println("🧪 Creating fake data for vacuum task testing...")
fmt.Printf("Master: %s\n", *master)
fmt.Printf("Files to create: %d\n", *fileCount)
fmt.Printf("Delete ratio: %.1f%%\n", *deleteRatio*100)
fmt.Printf("File size: %d KB\n", *fileSizeKB)
fmt.Println()
if *fileCount == 0 {
// Just check volume status
fmt.Println("📊 Checking volume status...")
checkVolumeStatus()
return
}
// Step 1: Create test files
fmt.Println("📁 Step 1: Creating test files...")
fids := createTestFiles()
// Step 2: Delete some files to create garbage
fmt.Println("🗑️ Step 2: Deleting files to create garbage...")
deleteFiles(fids)
// Step 3: Check volume status
fmt.Println("📊 Step 3: Checking volume status...")
checkVolumeStatus()
// Step 4: Configure vacuum for testing
fmt.Println("⚙️ Step 4: Instructions for testing...")
printTestingInstructions()
}
func createTestFiles() []string {
var fids []string
for i := 0; i < *fileCount; i++ {
// Generate random file content
fileData := make([]byte, *fileSizeKB*1024)
rand.Read(fileData)
// Get file ID assignment
assign, err := assignFileId()
if err != nil {
log.Printf("Failed to assign file ID for file %d: %v", i, err)
continue
}
// Upload file
err = uploadFile(assign, fileData, fmt.Sprintf("test_file_%d.dat", i))
if err != nil {
log.Printf("Failed to upload file %d: %v", i, err)
continue
}
fids = append(fids, assign.Fid)
if (i+1)%5 == 0 {
fmt.Printf(" Created %d/%d files...\n", i+1, *fileCount)
}
}
fmt.Printf("✅ Created %d files successfully\n\n", len(fids))
return fids
}
func deleteFiles(fids []string) {
deleteCount := int(float64(len(fids)) * *deleteRatio)
for i := 0; i < deleteCount; i++ {
err := deleteFile(fids[i])
if err != nil {
log.Printf("Failed to delete file %s: %v", fids[i], err)
continue
}
if (i+1)%5 == 0 {
fmt.Printf(" Deleted %d/%d files...\n", i+1, deleteCount)
}
}
fmt.Printf("✅ Deleted %d files (%.1f%% of total)\n\n", deleteCount, *deleteRatio*100)
}
func assignFileId() (*AssignResult, error) {
resp, err := http.Get(fmt.Sprintf("http://%s/dir/assign", *master))
if err != nil {
return nil, err
}
defer resp.Body.Close()
var result AssignResult
err = json.NewDecoder(resp.Body).Decode(&result)
if err != nil {
return nil, err
}
if result.Error != "" {
return nil, fmt.Errorf("assignment error: %s", result.Error)
}
return &result, nil
}
func uploadFile(assign *AssignResult, data []byte, filename string) error {
url := fmt.Sprintf("http://%s/%s", assign.Url, assign.Fid)
body := &bytes.Buffer{}
body.Write(data)
req, err := http.NewRequest("POST", url, body)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/octet-stream")
if filename != "" {
req.Header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(body))
}
return nil
}
func deleteFile(fid string) error {
url := fmt.Sprintf("http://%s/%s", *master, fid)
req, err := http.NewRequest("DELETE", url, nil)
if err != nil {
return err
}
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func checkVolumeStatus() {
// Get volume list from master
resp, err := http.Get(fmt.Sprintf("http://%s/vol/status", *master))
if err != nil {
log.Printf("Failed to get volume status: %v", err)
return
}
defer resp.Body.Close()
var volumes map[string]interface{}
err = json.NewDecoder(resp.Body).Decode(&volumes)
if err != nil {
log.Printf("Failed to decode volume status: %v", err)
return
}
fmt.Println("📊 Volume Status Summary:")
if vols, ok := volumes["Volumes"].([]interface{}); ok {
for _, vol := range vols {
if v, ok := vol.(map[string]interface{}); ok {
id := int(v["Id"].(float64))
size := uint64(v["Size"].(float64))
fileCount := int(v["FileCount"].(float64))
deleteCount := int(v["DeleteCount"].(float64))
deletedBytes := uint64(v["DeletedByteCount"].(float64))
garbageRatio := 0.0
if size > 0 {
garbageRatio = float64(deletedBytes) / float64(size) * 100
}
fmt.Printf(" Volume %d:\n", id)
fmt.Printf(" Size: %s\n", formatBytes(size))
fmt.Printf(" Files: %d (active), %d (deleted)\n", fileCount, deleteCount)
fmt.Printf(" Garbage: %s (%.1f%%)\n", formatBytes(deletedBytes), garbageRatio)
if garbageRatio > 30 {
fmt.Printf(" 🎯 This volume should trigger vacuum (>30%% garbage)\n")
}
fmt.Println()
}
}
}
}
func formatBytes(bytes uint64) string {
if bytes < 1024 {
return fmt.Sprintf("%d B", bytes)
} else if bytes < 1024*1024 {
return fmt.Sprintf("%.1f KB", float64(bytes)/1024)
} else if bytes < 1024*1024*1024 {
return fmt.Sprintf("%.1f MB", float64(bytes)/(1024*1024))
} else {
return fmt.Sprintf("%.1f GB", float64(bytes)/(1024*1024*1024))
}
}
func printTestingInstructions() {
fmt.Println("🧪 Testing Instructions:")
fmt.Println()
fmt.Println("1. Configure Vacuum for Testing:")
fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum")
fmt.Println(" Set:")
fmt.Printf(" - Garbage Threshold: 0.20 (20%% - lower than default)\n")
fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n")
fmt.Printf(" - Min Volume Age: [0] [Minutes] (no age requirement)\n")
fmt.Printf(" - Max Concurrent: 2\n")
fmt.Printf(" - Min Interval: 1m (faster repeat)\n")
fmt.Println()
fmt.Println("2. Monitor Vacuum Tasks:")
fmt.Println(" Visit: http://localhost:23646/maintenance")
fmt.Println(" Watch for vacuum tasks to appear in the queue")
fmt.Println()
fmt.Println("3. Manual Vacuum (Optional):")
fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.01'")
fmt.Println()
fmt.Println("4. Check Logs:")
fmt.Println(" Look for messages like:")
fmt.Println(" - 'Vacuum detector found X volumes needing vacuum'")
fmt.Println(" - 'Applied vacuum configuration'")
fmt.Println(" - 'Worker executing task: vacuum'")
fmt.Println()
fmt.Println("5. Verify Results:")
fmt.Println(" Re-run this script with -files=0 to check volume status")
fmt.Println(" Garbage ratios should decrease after vacuum operations")
fmt.Println()
fmt.Printf("🚀 Quick test command:\n")
fmt.Printf(" go run create_vacuum_test_data.go -files=0\n")
fmt.Println()
}

View File

@@ -0,0 +1,105 @@
#!/bin/sh
echo "🧪 SeaweedFS Vacuum Task Testing Demo"
echo "======================================"
echo ""
# Check if SeaweedFS is running
echo "📋 Checking SeaweedFS status..."
MASTER_URL="${MASTER_HOST:-master:9333}"
ADMIN_URL="${ADMIN_HOST:-admin:23646}"
if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then
echo "❌ SeaweedFS master not running at $MASTER_URL"
echo " Please ensure Docker cluster is running: make start"
exit 1
fi
if ! curl -s http://volume1:8080/status > /dev/null; then
echo "❌ SeaweedFS volume servers not running"
echo " Please ensure Docker cluster is running: make start"
exit 1
fi
if ! curl -s http://$ADMIN_URL/ > /dev/null; then
echo "❌ SeaweedFS admin server not running at $ADMIN_URL"
echo " Please ensure Docker cluster is running: make start"
exit 1
fi
echo "✅ All SeaweedFS components are running"
echo ""
# Phase 1: Create test data
echo "📁 Phase 1: Creating test data with garbage..."
go run create_vacuum_test_data.go -master=$MASTER_URL -files=15 -delete=0.5 -size=150
echo ""
# Phase 2: Check initial status
echo "📊 Phase 2: Checking initial volume status..."
go run create_vacuum_test_data.go -master=$MASTER_URL -files=0
echo ""
# Phase 3: Configure vacuum
echo "⚙️ Phase 3: Vacuum configuration instructions..."
echo " 1. Visit: http://localhost:23646/maintenance/config/vacuum"
echo " 2. Set these values for testing:"
echo " - Enable Vacuum Tasks: ✅ Checked"
echo " - Garbage Threshold: 0.30"
echo " - Scan Interval: [30] [Seconds]"
echo " - Min Volume Age: [0] [Minutes]"
echo " - Max Concurrent: 2"
echo " 3. Click 'Save Configuration'"
echo ""
read -p " Press ENTER after configuring vacuum settings..."
echo ""
# Phase 4: Monitor tasks
echo "🎯 Phase 4: Monitoring vacuum tasks..."
echo " Visit: http://localhost:23646/maintenance"
echo " You should see vacuum tasks appear within 30 seconds"
echo ""
echo " Waiting 60 seconds for vacuum detection and execution..."
for i in {60..1}; do
printf "\r Countdown: %02d seconds" $i
sleep 1
done
echo ""
echo ""
# Phase 5: Check results
echo "📈 Phase 5: Checking results after vacuum..."
go run create_vacuum_test_data.go -master=$MASTER_URL -files=0
echo ""
# Phase 6: Create more garbage for continuous testing
echo "🔄 Phase 6: Creating additional garbage for continuous testing..."
echo " Running 3 rounds of garbage creation..."
for round in {1..3}; do
echo " Round $round: Creating garbage..."
go run create_vacuum_test_data.go -master=$MASTER_URL -files=8 -delete=0.6 -size=100
echo " Waiting 30 seconds before next round..."
sleep 30
done
echo ""
echo "📊 Final volume status:"
go run create_vacuum_test_data.go -master=$MASTER_URL -files=0
echo ""
echo "🎉 Demo Complete!"
echo ""
echo "🔍 Things to check:"
echo " 1. Maintenance Queue: http://localhost:23646/maintenance"
echo " 2. Volume Status: http://localhost:9333/vol/status"
echo " 3. Admin Dashboard: http://localhost:23646"
echo ""
echo "💡 Next Steps:"
echo " - Try different garbage thresholds (0.10, 0.50, 0.80)"
echo " - Adjust scan intervals (10s, 1m, 5m)"
echo " - Monitor logs for vacuum operations"
echo " - Test with multiple volumes"
echo ""

View File

@@ -213,4 +213,29 @@ services:
- admin
- filer
networks:
- seaweed_net
- seaweed_net
vacuum-tester:
image: chrislusf/seaweedfs:local
entrypoint: ["/bin/sh"]
command: >
-c "
echo 'Installing dependencies for vacuum testing...';
apk add --no-cache jq curl go bash;
echo 'Vacuum tester ready...';
echo 'Use: docker-compose exec vacuum-tester sh';
echo 'Available commands: go, weed, curl, jq, bash, sh';
sleep infinity
"
depends_on:
- master
- admin
- filer
volumes:
- .:/testing
working_dir: /testing
networks:
- seaweed_net
environment:
- MASTER_HOST=master:9333
- ADMIN_HOST=admin:23646