chrislu
4 months ago
5 changed files with 570 additions and 5 deletions
-
130docker/admin_integration/Makefile
-
32docker/admin_integration/check_volumes.sh
-
279docker/admin_integration/create_vacuum_test_data.go
-
105docker/admin_integration/demo_vacuum_testing.sh
-
25docker/admin_integration/docker-compose-ec-test.yml
@ -0,0 +1,32 @@ |
|||||
|
#!/bin/sh |
||||
|
|
||||
|
echo "📊 Quick Volume Status Check" |
||||
|
echo "============================" |
||||
|
echo "" |
||||
|
|
||||
|
# Check if master is running |
||||
|
MASTER_URL="${MASTER_HOST:-master:9333}" |
||||
|
if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then |
||||
|
echo "❌ Master server not available at $MASTER_URL" |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
echo "🔍 Fetching volume status from master..." |
||||
|
curl -s "http://$MASTER_URL/vol/status" | jq -r ' |
||||
|
if .Volumes and .Volumes.DataCenters then |
||||
|
.Volumes.DataCenters | to_entries[] | .value | to_entries[] | .value | to_entries[] | .value | if . then .[] else empty end | |
||||
|
"Volume \(.Id): |
||||
|
Size: \(.Size | if . < 1024 then "\(.) B" elif . < 1048576 then "\(. / 1024 | floor) KB" elif . < 1073741824 then "\(. / 1048576 * 100 | floor / 100) MB" else "\(. / 1073741824 * 100 | floor / 100) GB" end) |
||||
|
Files: \(.FileCount) active, \(.DeleteCount) deleted |
||||
|
Garbage: \(.DeletedByteCount | if . < 1024 then "\(.) B" elif . < 1048576 then "\(. / 1024 | floor) KB" elif . < 1073741824 then "\(. / 1048576 * 100 | floor / 100) MB" else "\(. / 1073741824 * 100 | floor / 100) GB" end) (\(if .Size > 0 then (.DeletedByteCount / .Size * 100 | floor) else 0 end)%) |
||||
|
Status: \(if (.DeletedByteCount / .Size * 100) > 30 then "🎯 NEEDS VACUUM" else "✅ OK" end) |
||||
|
" |
||||
|
else |
||||
|
"No volumes found" |
||||
|
end' |
||||
|
|
||||
|
echo "" |
||||
|
echo "💡 Legend:" |
||||
|
echo " 🎯 NEEDS VACUUM: >30% garbage ratio" |
||||
|
echo " ✅ OK: <30% garbage ratio" |
||||
|
echo "" |
||||
@ -0,0 +1,279 @@ |
|||||
|
package main |
||||
|
|
||||
|
import ( |
||||
|
"bytes" |
||||
|
"crypto/rand" |
||||
|
"encoding/json" |
||||
|
"flag" |
||||
|
"fmt" |
||||
|
"io" |
||||
|
"log" |
||||
|
"net/http" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
var ( |
||||
|
master = flag.String("master", "master:9333", "SeaweedFS master server address") |
||||
|
fileCount = flag.Int("files", 20, "Number of files to create") |
||||
|
deleteRatio = flag.Float64("delete", 0.4, "Ratio of files to delete (0.0-1.0)") |
||||
|
fileSizeKB = flag.Int("size", 100, "Size of each file in KB") |
||||
|
) |
||||
|
|
||||
|
type AssignResult struct { |
||||
|
Fid string `json:"fid"` |
||||
|
Url string `json:"url"` |
||||
|
PublicUrl string `json:"publicUrl"` |
||||
|
Count int `json:"count"` |
||||
|
Error string `json:"error"` |
||||
|
} |
||||
|
|
||||
|
func main() { |
||||
|
flag.Parse() |
||||
|
|
||||
|
fmt.Println("🧪 Creating fake data for vacuum task testing...") |
||||
|
fmt.Printf("Master: %s\n", *master) |
||||
|
fmt.Printf("Files to create: %d\n", *fileCount) |
||||
|
fmt.Printf("Delete ratio: %.1f%%\n", *deleteRatio*100) |
||||
|
fmt.Printf("File size: %d KB\n", *fileSizeKB) |
||||
|
fmt.Println() |
||||
|
|
||||
|
if *fileCount == 0 { |
||||
|
// Just check volume status
|
||||
|
fmt.Println("📊 Checking volume status...") |
||||
|
checkVolumeStatus() |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// Step 1: Create test files
|
||||
|
fmt.Println("📁 Step 1: Creating test files...") |
||||
|
fids := createTestFiles() |
||||
|
|
||||
|
// Step 2: Delete some files to create garbage
|
||||
|
fmt.Println("🗑️ Step 2: Deleting files to create garbage...") |
||||
|
deleteFiles(fids) |
||||
|
|
||||
|
// Step 3: Check volume status
|
||||
|
fmt.Println("📊 Step 3: Checking volume status...") |
||||
|
checkVolumeStatus() |
||||
|
|
||||
|
// Step 4: Configure vacuum for testing
|
||||
|
fmt.Println("⚙️ Step 4: Instructions for testing...") |
||||
|
printTestingInstructions() |
||||
|
} |
||||
|
|
||||
|
func createTestFiles() []string { |
||||
|
var fids []string |
||||
|
|
||||
|
for i := 0; i < *fileCount; i++ { |
||||
|
// Generate random file content
|
||||
|
fileData := make([]byte, *fileSizeKB*1024) |
||||
|
rand.Read(fileData) |
||||
|
|
||||
|
// Get file ID assignment
|
||||
|
assign, err := assignFileId() |
||||
|
if err != nil { |
||||
|
log.Printf("Failed to assign file ID for file %d: %v", i, err) |
||||
|
continue |
||||
|
} |
||||
|
|
||||
|
// Upload file
|
||||
|
err = uploadFile(assign, fileData, fmt.Sprintf("test_file_%d.dat", i)) |
||||
|
if err != nil { |
||||
|
log.Printf("Failed to upload file %d: %v", i, err) |
||||
|
continue |
||||
|
} |
||||
|
|
||||
|
fids = append(fids, assign.Fid) |
||||
|
|
||||
|
if (i+1)%5 == 0 { |
||||
|
fmt.Printf(" Created %d/%d files...\n", i+1, *fileCount) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
fmt.Printf("✅ Created %d files successfully\n\n", len(fids)) |
||||
|
return fids |
||||
|
} |
||||
|
|
||||
|
func deleteFiles(fids []string) { |
||||
|
deleteCount := int(float64(len(fids)) * *deleteRatio) |
||||
|
|
||||
|
for i := 0; i < deleteCount; i++ { |
||||
|
err := deleteFile(fids[i]) |
||||
|
if err != nil { |
||||
|
log.Printf("Failed to delete file %s: %v", fids[i], err) |
||||
|
continue |
||||
|
} |
||||
|
|
||||
|
if (i+1)%5 == 0 { |
||||
|
fmt.Printf(" Deleted %d/%d files...\n", i+1, deleteCount) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
fmt.Printf("✅ Deleted %d files (%.1f%% of total)\n\n", deleteCount, *deleteRatio*100) |
||||
|
} |
||||
|
|
||||
|
func assignFileId() (*AssignResult, error) { |
||||
|
resp, err := http.Get(fmt.Sprintf("http://%s/dir/assign", *master)) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
defer resp.Body.Close() |
||||
|
|
||||
|
var result AssignResult |
||||
|
err = json.NewDecoder(resp.Body).Decode(&result) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
|
||||
|
if result.Error != "" { |
||||
|
return nil, fmt.Errorf("assignment error: %s", result.Error) |
||||
|
} |
||||
|
|
||||
|
return &result, nil |
||||
|
} |
||||
|
|
||||
|
func uploadFile(assign *AssignResult, data []byte, filename string) error { |
||||
|
url := fmt.Sprintf("http://%s/%s", assign.Url, assign.Fid) |
||||
|
|
||||
|
body := &bytes.Buffer{} |
||||
|
body.Write(data) |
||||
|
|
||||
|
req, err := http.NewRequest("POST", url, body) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
req.Header.Set("Content-Type", "application/octet-stream") |
||||
|
if filename != "" { |
||||
|
req.Header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) |
||||
|
} |
||||
|
|
||||
|
client := &http.Client{Timeout: 30 * time.Second} |
||||
|
resp, err := client.Do(req) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
defer resp.Body.Close() |
||||
|
|
||||
|
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { |
||||
|
body, _ := io.ReadAll(resp.Body) |
||||
|
return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(body)) |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func deleteFile(fid string) error { |
||||
|
url := fmt.Sprintf("http://%s/%s", *master, fid) |
||||
|
|
||||
|
req, err := http.NewRequest("DELETE", url, nil) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
client := &http.Client{Timeout: 10 * time.Second} |
||||
|
resp, err := client.Do(req) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
defer resp.Body.Close() |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func checkVolumeStatus() { |
||||
|
// Get volume list from master
|
||||
|
resp, err := http.Get(fmt.Sprintf("http://%s/vol/status", *master)) |
||||
|
if err != nil { |
||||
|
log.Printf("Failed to get volume status: %v", err) |
||||
|
return |
||||
|
} |
||||
|
defer resp.Body.Close() |
||||
|
|
||||
|
var volumes map[string]interface{} |
||||
|
err = json.NewDecoder(resp.Body).Decode(&volumes) |
||||
|
if err != nil { |
||||
|
log.Printf("Failed to decode volume status: %v", err) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
fmt.Println("📊 Volume Status Summary:") |
||||
|
|
||||
|
if vols, ok := volumes["Volumes"].([]interface{}); ok { |
||||
|
for _, vol := range vols { |
||||
|
if v, ok := vol.(map[string]interface{}); ok { |
||||
|
id := int(v["Id"].(float64)) |
||||
|
size := uint64(v["Size"].(float64)) |
||||
|
fileCount := int(v["FileCount"].(float64)) |
||||
|
deleteCount := int(v["DeleteCount"].(float64)) |
||||
|
deletedBytes := uint64(v["DeletedByteCount"].(float64)) |
||||
|
|
||||
|
garbageRatio := 0.0 |
||||
|
if size > 0 { |
||||
|
garbageRatio = float64(deletedBytes) / float64(size) * 100 |
||||
|
} |
||||
|
|
||||
|
fmt.Printf(" Volume %d:\n", id) |
||||
|
fmt.Printf(" Size: %s\n", formatBytes(size)) |
||||
|
fmt.Printf(" Files: %d (active), %d (deleted)\n", fileCount, deleteCount) |
||||
|
fmt.Printf(" Garbage: %s (%.1f%%)\n", formatBytes(deletedBytes), garbageRatio) |
||||
|
|
||||
|
if garbageRatio > 30 { |
||||
|
fmt.Printf(" 🎯 This volume should trigger vacuum (>30%% garbage)\n") |
||||
|
} |
||||
|
fmt.Println() |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func formatBytes(bytes uint64) string { |
||||
|
if bytes < 1024 { |
||||
|
return fmt.Sprintf("%d B", bytes) |
||||
|
} else if bytes < 1024*1024 { |
||||
|
return fmt.Sprintf("%.1f KB", float64(bytes)/1024) |
||||
|
} else if bytes < 1024*1024*1024 { |
||||
|
return fmt.Sprintf("%.1f MB", float64(bytes)/(1024*1024)) |
||||
|
} else { |
||||
|
return fmt.Sprintf("%.1f GB", float64(bytes)/(1024*1024*1024)) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func printTestingInstructions() { |
||||
|
fmt.Println("🧪 Testing Instructions:") |
||||
|
fmt.Println() |
||||
|
fmt.Println("1. Configure Vacuum for Testing:") |
||||
|
fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum") |
||||
|
fmt.Println(" Set:") |
||||
|
fmt.Printf(" - Garbage Threshold: 0.20 (20%% - lower than default)\n") |
||||
|
fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n") |
||||
|
fmt.Printf(" - Min Volume Age: [0] [Minutes] (no age requirement)\n") |
||||
|
fmt.Printf(" - Max Concurrent: 2\n") |
||||
|
fmt.Printf(" - Min Interval: 1m (faster repeat)\n") |
||||
|
fmt.Println() |
||||
|
|
||||
|
fmt.Println("2. Monitor Vacuum Tasks:") |
||||
|
fmt.Println(" Visit: http://localhost:23646/maintenance") |
||||
|
fmt.Println(" Watch for vacuum tasks to appear in the queue") |
||||
|
fmt.Println() |
||||
|
|
||||
|
fmt.Println("3. Manual Vacuum (Optional):") |
||||
|
fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.01'") |
||||
|
fmt.Println() |
||||
|
|
||||
|
fmt.Println("4. Check Logs:") |
||||
|
fmt.Println(" Look for messages like:") |
||||
|
fmt.Println(" - 'Vacuum detector found X volumes needing vacuum'") |
||||
|
fmt.Println(" - 'Applied vacuum configuration'") |
||||
|
fmt.Println(" - 'Worker executing task: vacuum'") |
||||
|
fmt.Println() |
||||
|
|
||||
|
fmt.Println("5. Verify Results:") |
||||
|
fmt.Println(" Re-run this script with -files=0 to check volume status") |
||||
|
fmt.Println(" Garbage ratios should decrease after vacuum operations") |
||||
|
fmt.Println() |
||||
|
|
||||
|
fmt.Printf("🚀 Quick test command:\n") |
||||
|
fmt.Printf(" go run create_vacuum_test_data.go -files=0\n") |
||||
|
fmt.Println() |
||||
|
} |
||||
@ -0,0 +1,105 @@ |
|||||
|
#!/bin/sh |
||||
|
|
||||
|
echo "🧪 SeaweedFS Vacuum Task Testing Demo" |
||||
|
echo "======================================" |
||||
|
echo "" |
||||
|
|
||||
|
# Check if SeaweedFS is running |
||||
|
echo "📋 Checking SeaweedFS status..." |
||||
|
MASTER_URL="${MASTER_HOST:-master:9333}" |
||||
|
ADMIN_URL="${ADMIN_HOST:-admin:23646}" |
||||
|
|
||||
|
if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then |
||||
|
echo "❌ SeaweedFS master not running at $MASTER_URL" |
||||
|
echo " Please ensure Docker cluster is running: make start" |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
if ! curl -s http://volume1:8080/status > /dev/null; then |
||||
|
echo "❌ SeaweedFS volume servers not running" |
||||
|
echo " Please ensure Docker cluster is running: make start" |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
if ! curl -s http://$ADMIN_URL/ > /dev/null; then |
||||
|
echo "❌ SeaweedFS admin server not running at $ADMIN_URL" |
||||
|
echo " Please ensure Docker cluster is running: make start" |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
echo "✅ All SeaweedFS components are running" |
||||
|
echo "" |
||||
|
|
||||
|
# Phase 1: Create test data |
||||
|
echo "📁 Phase 1: Creating test data with garbage..." |
||||
|
go run create_vacuum_test_data.go -master=$MASTER_URL -files=15 -delete=0.5 -size=150 |
||||
|
echo "" |
||||
|
|
||||
|
# Phase 2: Check initial status |
||||
|
echo "📊 Phase 2: Checking initial volume status..." |
||||
|
go run create_vacuum_test_data.go -master=$MASTER_URL -files=0 |
||||
|
echo "" |
||||
|
|
||||
|
# Phase 3: Configure vacuum |
||||
|
echo "⚙️ Phase 3: Vacuum configuration instructions..." |
||||
|
echo " 1. Visit: http://localhost:23646/maintenance/config/vacuum" |
||||
|
echo " 2. Set these values for testing:" |
||||
|
echo " - Enable Vacuum Tasks: ✅ Checked" |
||||
|
echo " - Garbage Threshold: 0.30" |
||||
|
echo " - Scan Interval: [30] [Seconds]" |
||||
|
echo " - Min Volume Age: [0] [Minutes]" |
||||
|
echo " - Max Concurrent: 2" |
||||
|
echo " 3. Click 'Save Configuration'" |
||||
|
echo "" |
||||
|
|
||||
|
read -p " Press ENTER after configuring vacuum settings..." |
||||
|
echo "" |
||||
|
|
||||
|
# Phase 4: Monitor tasks |
||||
|
echo "🎯 Phase 4: Monitoring vacuum tasks..." |
||||
|
echo " Visit: http://localhost:23646/maintenance" |
||||
|
echo " You should see vacuum tasks appear within 30 seconds" |
||||
|
echo "" |
||||
|
|
||||
|
echo " Waiting 60 seconds for vacuum detection and execution..." |
||||
|
for i in {60..1}; do |
||||
|
printf "\r Countdown: %02d seconds" $i |
||||
|
sleep 1 |
||||
|
done |
||||
|
echo "" |
||||
|
echo "" |
||||
|
|
||||
|
# Phase 5: Check results |
||||
|
echo "📈 Phase 5: Checking results after vacuum..." |
||||
|
go run create_vacuum_test_data.go -master=$MASTER_URL -files=0 |
||||
|
echo "" |
||||
|
|
||||
|
# Phase 6: Create more garbage for continuous testing |
||||
|
echo "🔄 Phase 6: Creating additional garbage for continuous testing..." |
||||
|
echo " Running 3 rounds of garbage creation..." |
||||
|
|
||||
|
for round in {1..3}; do |
||||
|
echo " Round $round: Creating garbage..." |
||||
|
go run create_vacuum_test_data.go -master=$MASTER_URL -files=8 -delete=0.6 -size=100 |
||||
|
echo " Waiting 30 seconds before next round..." |
||||
|
sleep 30 |
||||
|
done |
||||
|
|
||||
|
echo "" |
||||
|
echo "📊 Final volume status:" |
||||
|
go run create_vacuum_test_data.go -master=$MASTER_URL -files=0 |
||||
|
echo "" |
||||
|
|
||||
|
echo "🎉 Demo Complete!" |
||||
|
echo "" |
||||
|
echo "🔍 Things to check:" |
||||
|
echo " 1. Maintenance Queue: http://localhost:23646/maintenance" |
||||
|
echo " 2. Volume Status: http://localhost:9333/vol/status" |
||||
|
echo " 3. Admin Dashboard: http://localhost:23646" |
||||
|
echo "" |
||||
|
echo "💡 Next Steps:" |
||||
|
echo " - Try different garbage thresholds (0.10, 0.50, 0.80)" |
||||
|
echo " - Adjust scan intervals (10s, 1m, 5m)" |
||||
|
echo " - Monitor logs for vacuum operations" |
||||
|
echo " - Test with multiple volumes" |
||||
|
echo "" |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue