chrislu
4 months ago
5 changed files with 570 additions and 5 deletions
-
132docker/admin_integration/Makefile
-
32docker/admin_integration/check_volumes.sh
-
279docker/admin_integration/create_vacuum_test_data.go
-
105docker/admin_integration/demo_vacuum_testing.sh
-
27docker/admin_integration/docker-compose-ec-test.yml
@ -0,0 +1,32 @@ |
|||
#!/bin/sh |
|||
|
|||
echo "📊 Quick Volume Status Check" |
|||
echo "============================" |
|||
echo "" |
|||
|
|||
# Check if master is running |
|||
MASTER_URL="${MASTER_HOST:-master:9333}" |
|||
if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then |
|||
echo "❌ Master server not available at $MASTER_URL" |
|||
exit 1 |
|||
fi |
|||
|
|||
echo "🔍 Fetching volume status from master..." |
|||
curl -s "http://$MASTER_URL/vol/status" | jq -r ' |
|||
if .Volumes and .Volumes.DataCenters then |
|||
.Volumes.DataCenters | to_entries[] | .value | to_entries[] | .value | to_entries[] | .value | if . then .[] else empty end | |
|||
"Volume \(.Id): |
|||
Size: \(.Size | if . < 1024 then "\(.) B" elif . < 1048576 then "\(. / 1024 | floor) KB" elif . < 1073741824 then "\(. / 1048576 * 100 | floor / 100) MB" else "\(. / 1073741824 * 100 | floor / 100) GB" end) |
|||
Files: \(.FileCount) active, \(.DeleteCount) deleted |
|||
Garbage: \(.DeletedByteCount | if . < 1024 then "\(.) B" elif . < 1048576 then "\(. / 1024 | floor) KB" elif . < 1073741824 then "\(. / 1048576 * 100 | floor / 100) MB" else "\(. / 1073741824 * 100 | floor / 100) GB" end) (\(if .Size > 0 then (.DeletedByteCount / .Size * 100 | floor) else 0 end)%) |
|||
Status: \(if (.DeletedByteCount / .Size * 100) > 30 then "🎯 NEEDS VACUUM" else "✅ OK" end) |
|||
" |
|||
else |
|||
"No volumes found" |
|||
end' |
|||
|
|||
echo "" |
|||
echo "💡 Legend:" |
|||
echo " 🎯 NEEDS VACUUM: >30% garbage ratio" |
|||
echo " ✅ OK: <30% garbage ratio" |
|||
echo "" |
|||
@ -0,0 +1,279 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"bytes" |
|||
"crypto/rand" |
|||
"encoding/json" |
|||
"flag" |
|||
"fmt" |
|||
"io" |
|||
"log" |
|||
"net/http" |
|||
"time" |
|||
) |
|||
|
|||
var ( |
|||
master = flag.String("master", "master:9333", "SeaweedFS master server address") |
|||
fileCount = flag.Int("files", 20, "Number of files to create") |
|||
deleteRatio = flag.Float64("delete", 0.4, "Ratio of files to delete (0.0-1.0)") |
|||
fileSizeKB = flag.Int("size", 100, "Size of each file in KB") |
|||
) |
|||
|
|||
type AssignResult struct { |
|||
Fid string `json:"fid"` |
|||
Url string `json:"url"` |
|||
PublicUrl string `json:"publicUrl"` |
|||
Count int `json:"count"` |
|||
Error string `json:"error"` |
|||
} |
|||
|
|||
func main() { |
|||
flag.Parse() |
|||
|
|||
fmt.Println("🧪 Creating fake data for vacuum task testing...") |
|||
fmt.Printf("Master: %s\n", *master) |
|||
fmt.Printf("Files to create: %d\n", *fileCount) |
|||
fmt.Printf("Delete ratio: %.1f%%\n", *deleteRatio*100) |
|||
fmt.Printf("File size: %d KB\n", *fileSizeKB) |
|||
fmt.Println() |
|||
|
|||
if *fileCount == 0 { |
|||
// Just check volume status
|
|||
fmt.Println("📊 Checking volume status...") |
|||
checkVolumeStatus() |
|||
return |
|||
} |
|||
|
|||
// Step 1: Create test files
|
|||
fmt.Println("📁 Step 1: Creating test files...") |
|||
fids := createTestFiles() |
|||
|
|||
// Step 2: Delete some files to create garbage
|
|||
fmt.Println("🗑️ Step 2: Deleting files to create garbage...") |
|||
deleteFiles(fids) |
|||
|
|||
// Step 3: Check volume status
|
|||
fmt.Println("📊 Step 3: Checking volume status...") |
|||
checkVolumeStatus() |
|||
|
|||
// Step 4: Configure vacuum for testing
|
|||
fmt.Println("⚙️ Step 4: Instructions for testing...") |
|||
printTestingInstructions() |
|||
} |
|||
|
|||
func createTestFiles() []string { |
|||
var fids []string |
|||
|
|||
for i := 0; i < *fileCount; i++ { |
|||
// Generate random file content
|
|||
fileData := make([]byte, *fileSizeKB*1024) |
|||
rand.Read(fileData) |
|||
|
|||
// Get file ID assignment
|
|||
assign, err := assignFileId() |
|||
if err != nil { |
|||
log.Printf("Failed to assign file ID for file %d: %v", i, err) |
|||
continue |
|||
} |
|||
|
|||
// Upload file
|
|||
err = uploadFile(assign, fileData, fmt.Sprintf("test_file_%d.dat", i)) |
|||
if err != nil { |
|||
log.Printf("Failed to upload file %d: %v", i, err) |
|||
continue |
|||
} |
|||
|
|||
fids = append(fids, assign.Fid) |
|||
|
|||
if (i+1)%5 == 0 { |
|||
fmt.Printf(" Created %d/%d files...\n", i+1, *fileCount) |
|||
} |
|||
} |
|||
|
|||
fmt.Printf("✅ Created %d files successfully\n\n", len(fids)) |
|||
return fids |
|||
} |
|||
|
|||
func deleteFiles(fids []string) { |
|||
deleteCount := int(float64(len(fids)) * *deleteRatio) |
|||
|
|||
for i := 0; i < deleteCount; i++ { |
|||
err := deleteFile(fids[i]) |
|||
if err != nil { |
|||
log.Printf("Failed to delete file %s: %v", fids[i], err) |
|||
continue |
|||
} |
|||
|
|||
if (i+1)%5 == 0 { |
|||
fmt.Printf(" Deleted %d/%d files...\n", i+1, deleteCount) |
|||
} |
|||
} |
|||
|
|||
fmt.Printf("✅ Deleted %d files (%.1f%% of total)\n\n", deleteCount, *deleteRatio*100) |
|||
} |
|||
|
|||
func assignFileId() (*AssignResult, error) { |
|||
resp, err := http.Get(fmt.Sprintf("http://%s/dir/assign", *master)) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
defer resp.Body.Close() |
|||
|
|||
var result AssignResult |
|||
err = json.NewDecoder(resp.Body).Decode(&result) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
if result.Error != "" { |
|||
return nil, fmt.Errorf("assignment error: %s", result.Error) |
|||
} |
|||
|
|||
return &result, nil |
|||
} |
|||
|
|||
func uploadFile(assign *AssignResult, data []byte, filename string) error { |
|||
url := fmt.Sprintf("http://%s/%s", assign.Url, assign.Fid) |
|||
|
|||
body := &bytes.Buffer{} |
|||
body.Write(data) |
|||
|
|||
req, err := http.NewRequest("POST", url, body) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
req.Header.Set("Content-Type", "application/octet-stream") |
|||
if filename != "" { |
|||
req.Header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) |
|||
} |
|||
|
|||
client := &http.Client{Timeout: 30 * time.Second} |
|||
resp, err := client.Do(req) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
defer resp.Body.Close() |
|||
|
|||
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { |
|||
body, _ := io.ReadAll(resp.Body) |
|||
return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(body)) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func deleteFile(fid string) error { |
|||
url := fmt.Sprintf("http://%s/%s", *master, fid) |
|||
|
|||
req, err := http.NewRequest("DELETE", url, nil) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
client := &http.Client{Timeout: 10 * time.Second} |
|||
resp, err := client.Do(req) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
defer resp.Body.Close() |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func checkVolumeStatus() { |
|||
// Get volume list from master
|
|||
resp, err := http.Get(fmt.Sprintf("http://%s/vol/status", *master)) |
|||
if err != nil { |
|||
log.Printf("Failed to get volume status: %v", err) |
|||
return |
|||
} |
|||
defer resp.Body.Close() |
|||
|
|||
var volumes map[string]interface{} |
|||
err = json.NewDecoder(resp.Body).Decode(&volumes) |
|||
if err != nil { |
|||
log.Printf("Failed to decode volume status: %v", err) |
|||
return |
|||
} |
|||
|
|||
fmt.Println("📊 Volume Status Summary:") |
|||
|
|||
if vols, ok := volumes["Volumes"].([]interface{}); ok { |
|||
for _, vol := range vols { |
|||
if v, ok := vol.(map[string]interface{}); ok { |
|||
id := int(v["Id"].(float64)) |
|||
size := uint64(v["Size"].(float64)) |
|||
fileCount := int(v["FileCount"].(float64)) |
|||
deleteCount := int(v["DeleteCount"].(float64)) |
|||
deletedBytes := uint64(v["DeletedByteCount"].(float64)) |
|||
|
|||
garbageRatio := 0.0 |
|||
if size > 0 { |
|||
garbageRatio = float64(deletedBytes) / float64(size) * 100 |
|||
} |
|||
|
|||
fmt.Printf(" Volume %d:\n", id) |
|||
fmt.Printf(" Size: %s\n", formatBytes(size)) |
|||
fmt.Printf(" Files: %d (active), %d (deleted)\n", fileCount, deleteCount) |
|||
fmt.Printf(" Garbage: %s (%.1f%%)\n", formatBytes(deletedBytes), garbageRatio) |
|||
|
|||
if garbageRatio > 30 { |
|||
fmt.Printf(" 🎯 This volume should trigger vacuum (>30%% garbage)\n") |
|||
} |
|||
fmt.Println() |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
func formatBytes(bytes uint64) string { |
|||
if bytes < 1024 { |
|||
return fmt.Sprintf("%d B", bytes) |
|||
} else if bytes < 1024*1024 { |
|||
return fmt.Sprintf("%.1f KB", float64(bytes)/1024) |
|||
} else if bytes < 1024*1024*1024 { |
|||
return fmt.Sprintf("%.1f MB", float64(bytes)/(1024*1024)) |
|||
} else { |
|||
return fmt.Sprintf("%.1f GB", float64(bytes)/(1024*1024*1024)) |
|||
} |
|||
} |
|||
|
|||
func printTestingInstructions() { |
|||
fmt.Println("🧪 Testing Instructions:") |
|||
fmt.Println() |
|||
fmt.Println("1. Configure Vacuum for Testing:") |
|||
fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum") |
|||
fmt.Println(" Set:") |
|||
fmt.Printf(" - Garbage Threshold: 0.20 (20%% - lower than default)\n") |
|||
fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n") |
|||
fmt.Printf(" - Min Volume Age: [0] [Minutes] (no age requirement)\n") |
|||
fmt.Printf(" - Max Concurrent: 2\n") |
|||
fmt.Printf(" - Min Interval: 1m (faster repeat)\n") |
|||
fmt.Println() |
|||
|
|||
fmt.Println("2. Monitor Vacuum Tasks:") |
|||
fmt.Println(" Visit: http://localhost:23646/maintenance") |
|||
fmt.Println(" Watch for vacuum tasks to appear in the queue") |
|||
fmt.Println() |
|||
|
|||
fmt.Println("3. Manual Vacuum (Optional):") |
|||
fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.01'") |
|||
fmt.Println() |
|||
|
|||
fmt.Println("4. Check Logs:") |
|||
fmt.Println(" Look for messages like:") |
|||
fmt.Println(" - 'Vacuum detector found X volumes needing vacuum'") |
|||
fmt.Println(" - 'Applied vacuum configuration'") |
|||
fmt.Println(" - 'Worker executing task: vacuum'") |
|||
fmt.Println() |
|||
|
|||
fmt.Println("5. Verify Results:") |
|||
fmt.Println(" Re-run this script with -files=0 to check volume status") |
|||
fmt.Println(" Garbage ratios should decrease after vacuum operations") |
|||
fmt.Println() |
|||
|
|||
fmt.Printf("🚀 Quick test command:\n") |
|||
fmt.Printf(" go run create_vacuum_test_data.go -files=0\n") |
|||
fmt.Println() |
|||
} |
|||
@ -0,0 +1,105 @@ |
|||
#!/bin/sh |
|||
|
|||
echo "🧪 SeaweedFS Vacuum Task Testing Demo" |
|||
echo "======================================" |
|||
echo "" |
|||
|
|||
# Check if SeaweedFS is running |
|||
echo "📋 Checking SeaweedFS status..." |
|||
MASTER_URL="${MASTER_HOST:-master:9333}" |
|||
ADMIN_URL="${ADMIN_HOST:-admin:23646}" |
|||
|
|||
if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then |
|||
echo "❌ SeaweedFS master not running at $MASTER_URL" |
|||
echo " Please ensure Docker cluster is running: make start" |
|||
exit 1 |
|||
fi |
|||
|
|||
if ! curl -s http://volume1:8080/status > /dev/null; then |
|||
echo "❌ SeaweedFS volume servers not running" |
|||
echo " Please ensure Docker cluster is running: make start" |
|||
exit 1 |
|||
fi |
|||
|
|||
if ! curl -s http://$ADMIN_URL/ > /dev/null; then |
|||
echo "❌ SeaweedFS admin server not running at $ADMIN_URL" |
|||
echo " Please ensure Docker cluster is running: make start" |
|||
exit 1 |
|||
fi |
|||
|
|||
echo "✅ All SeaweedFS components are running" |
|||
echo "" |
|||
|
|||
# Phase 1: Create test data |
|||
echo "📁 Phase 1: Creating test data with garbage..." |
|||
go run create_vacuum_test_data.go -master=$MASTER_URL -files=15 -delete=0.5 -size=150 |
|||
echo "" |
|||
|
|||
# Phase 2: Check initial status |
|||
echo "📊 Phase 2: Checking initial volume status..." |
|||
go run create_vacuum_test_data.go -master=$MASTER_URL -files=0 |
|||
echo "" |
|||
|
|||
# Phase 3: Configure vacuum |
|||
echo "⚙️ Phase 3: Vacuum configuration instructions..." |
|||
echo " 1. Visit: http://localhost:23646/maintenance/config/vacuum" |
|||
echo " 2. Set these values for testing:" |
|||
echo " - Enable Vacuum Tasks: ✅ Checked" |
|||
echo " - Garbage Threshold: 0.30" |
|||
echo " - Scan Interval: [30] [Seconds]" |
|||
echo " - Min Volume Age: [0] [Minutes]" |
|||
echo " - Max Concurrent: 2" |
|||
echo " 3. Click 'Save Configuration'" |
|||
echo "" |
|||
|
|||
read -p " Press ENTER after configuring vacuum settings..." |
|||
echo "" |
|||
|
|||
# Phase 4: Monitor tasks |
|||
echo "🎯 Phase 4: Monitoring vacuum tasks..." |
|||
echo " Visit: http://localhost:23646/maintenance" |
|||
echo " You should see vacuum tasks appear within 30 seconds" |
|||
echo "" |
|||
|
|||
echo " Waiting 60 seconds for vacuum detection and execution..." |
|||
for i in {60..1}; do |
|||
printf "\r Countdown: %02d seconds" $i |
|||
sleep 1 |
|||
done |
|||
echo "" |
|||
echo "" |
|||
|
|||
# Phase 5: Check results |
|||
echo "📈 Phase 5: Checking results after vacuum..." |
|||
go run create_vacuum_test_data.go -master=$MASTER_URL -files=0 |
|||
echo "" |
|||
|
|||
# Phase 6: Create more garbage for continuous testing |
|||
echo "🔄 Phase 6: Creating additional garbage for continuous testing..." |
|||
echo " Running 3 rounds of garbage creation..." |
|||
|
|||
for round in {1..3}; do |
|||
echo " Round $round: Creating garbage..." |
|||
go run create_vacuum_test_data.go -master=$MASTER_URL -files=8 -delete=0.6 -size=100 |
|||
echo " Waiting 30 seconds before next round..." |
|||
sleep 30 |
|||
done |
|||
|
|||
echo "" |
|||
echo "📊 Final volume status:" |
|||
go run create_vacuum_test_data.go -master=$MASTER_URL -files=0 |
|||
echo "" |
|||
|
|||
echo "🎉 Demo Complete!" |
|||
echo "" |
|||
echo "🔍 Things to check:" |
|||
echo " 1. Maintenance Queue: http://localhost:23646/maintenance" |
|||
echo " 2. Volume Status: http://localhost:9333/vol/status" |
|||
echo " 3. Admin Dashboard: http://localhost:23646" |
|||
echo "" |
|||
echo "💡 Next Steps:" |
|||
echo " - Try different garbage thresholds (0.10, 0.50, 0.80)" |
|||
echo " - Adjust scan intervals (10s, 1m, 5m)" |
|||
echo " - Monitor logs for vacuum operations" |
|||
echo " - Test with multiple volumes" |
|||
echo "" |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue