diff --git a/docker/admin_integration/Makefile b/docker/admin_integration/Makefile index 68fb0cec6..62284af00 100644 --- a/docker/admin_integration/Makefile +++ b/docker/admin_integration/Makefile @@ -48,15 +48,22 @@ help: ## Show this help message @echo "๐Ÿ—‘๏ธ Vacuum Testing:" @grep -E '^vacuum-.*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}' @echo "" + @echo "๐Ÿ”ง EC Vacuum Testing:" + @grep -E '^ec-vacuum-.*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}' + @echo "" @echo "๐Ÿ“œ Monitoring:" @grep -E '^(logs|admin-logs|worker-logs|master-logs|admin-ui):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}' @echo "" @echo "๐Ÿš€ Quick Start:" @echo " make start # Start cluster" - @echo " make vacuum-test # Test vacuum tasks" - @echo " make vacuum-help # Vacuum testing guide" - @echo "" - @echo "๐Ÿ’ก For detailed vacuum testing: make vacuum-help" + @echo " make vacuum-test # Test regular vacuum tasks" + @echo " make ec-vacuum-test # Test EC vacuum tasks" + @echo " make vacuum-help # Regular vacuum guide" + @echo " make ec-vacuum-help # EC vacuum guide" + @echo "" + @echo "๐Ÿ’ก For detailed testing guides:" + @echo " make vacuum-help # Regular vacuum testing" + @echo " make ec-vacuum-help # EC vacuum testing" start: ## Start the complete SeaweedFS cluster with admin and workers @echo "๐Ÿš€ Starting SeaweedFS cluster with admin and workers..." @@ -316,6 +323,141 @@ vacuum-clean: ## Clean up vacuum test data (removes all volumes!) @docker-compose -f $(COMPOSE_FILE) up -d @echo "โœ… Clean up complete. Fresh volumes ready for testing." +# EC Vacuum Testing Targets +ec-vacuum-test: ## Generate EC volumes and test EC vacuum functionality + @echo "๐Ÿงช SeaweedFS EC Vacuum Task Testing" + @echo "====================================" + @echo "" + @echo "1๏ธโƒฃ Checking cluster health..." + @curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "โœ… Master ready" || (echo "โŒ Master not ready. Run 'make start' first." && exit 1) + @curl -s http://localhost:23646/ | grep -q "Admin" && echo "โœ… Admin ready" || (echo "โŒ Admin not ready. Run 'make start' first." && exit 1) + @echo "" + @echo "2๏ธโƒฃ Generating data to trigger EC encoding..." + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=30 -size=3000 + @echo "" + @echo "3๏ธโƒฃ Waiting for EC encoding to complete..." + @echo "โณ This may take 2-3 minutes..." + @sleep 120 + @echo "" + @echo "4๏ธโƒฃ Generating deletions on EC volumes..." + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=0.4 + @echo "" + @echo "5๏ธโƒฃ Configuration Instructions:" + @echo " Visit: http://localhost:23646/maintenance/config/ec_vacuum" + @echo " Set for testing:" + @echo " โ€ข Enable EC Vacuum Tasks: โœ… Checked" + @echo " โ€ข Garbage Threshold: 0.30 (30%)" + @echo " โ€ข Scan Interval: [60] [Seconds]" + @echo " โ€ข Min Volume Age: [2] [Minutes]" + @echo " โ€ข Max Concurrent: 2" + @echo "" + @echo "6๏ธโƒฃ Monitor EC vacuum tasks at: http://localhost:23646/maintenance" + @echo "" + @echo "๐Ÿ’ก Use 'make ec-vacuum-status' to check EC volume garbage ratios" + +ec-vacuum-generate: ## Generate large files to trigger EC encoding + @echo "๐Ÿ“ Generating data to trigger EC encoding..." + @echo "Creating large files targeting >50MB per volume..." + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=25 -size=3000 + @echo "" + @echo "โณ Wait 2-3 minutes for EC encoding, then run 'make ec-vacuum-delete'" + +ec-vacuum-delete: ## Create deletions on EC volumes to generate garbage + @echo "๐Ÿ—‘๏ธ Creating deletions on EC volumes..." + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=$${DELETE:-0.4} + @echo "" + @echo "๐Ÿ’ก Use 'make ec-vacuum-status' to check garbage ratios" + +ec-vacuum-status: ## Check EC volume status and garbage ratios + @echo "๐Ÿ“Š EC Volume Status and Garbage Ratios" + @echo "=====================================" + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=status + +ec-vacuum-continuous: ## Generate continuous EC garbage for testing + @echo "๐Ÿ”„ Generating continuous EC garbage for vacuum testing..." + @echo "Running 3 rounds with 60-second intervals..." + @for i in {1..3}; do \ + echo "Round $$i: Generating large files..."; \ + docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=15 -size=4000; \ + echo "Waiting 90 seconds for EC encoding..."; \ + sleep 90; \ + echo "Creating deletions..."; \ + docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=0.5; \ + echo "Waiting 60 seconds before next round..."; \ + sleep 60; \ + done + @echo "โœ… Continuous EC vacuum test complete. Monitor admin UI for ec_vacuum tasks!" + +ec-vacuum-high: ## Create high garbage on EC volumes (should trigger EC vacuum) + @echo "๐Ÿ“ Creating high garbage EC volumes (60% garbage)..." + @echo "1. Generating files for EC..." + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=20 -size=4000 + @echo "2. Waiting for EC encoding..." + @sleep 120 + @echo "3. Creating high garbage ratio..." + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=0.6 + +ec-vacuum-low: ## Create low garbage on EC volumes (should NOT trigger EC vacuum) + @echo "๐Ÿ“ Creating low garbage EC volumes (20% garbage)..." + @echo "1. Generating files for EC..." + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=20 -size=4000 + @echo "2. Waiting for EC encoding..." + @sleep 120 + @echo "3. Creating low garbage ratio..." + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=0.2 + +ec-vacuum-monitor: ## Monitor EC vacuum task activity in real-time + @echo "๐Ÿ“Š Monitoring EC Vacuum Task Activity" + @echo "====================================" + @echo "Press Ctrl+C to stop monitoring" + @echo "" + @while true; do \ + echo "=== $(date) ==="; \ + docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=status; \ + echo ""; \ + echo "๐Ÿ” Recent admin logs (EC vacuum activity):"; \ + docker-compose -f $(COMPOSE_FILE) logs --tail=5 admin | grep -i "ec_vacuum\|vacuum.*ec" || echo "No recent EC vacuum activity"; \ + echo ""; \ + sleep 30; \ + done + +ec-vacuum-help: ## Show EC vacuum testing help and examples + @echo "๐Ÿงช EC Vacuum Testing Commands" + @echo "=============================" + @echo "" + @echo "Quick Start:" + @echo " make start # Start SeaweedFS cluster" + @echo " make ec-vacuum-test # Full EC vacuum test cycle" + @echo " make ec-vacuum-status # Check EC volume status" + @echo "" + @echo "Manual Testing:" + @echo " make ec-vacuum-generate # 1. Generate data โ†’ trigger EC" + @echo " # Wait 2-3 minutes for EC encoding to complete" + @echo " make ec-vacuum-delete # 2. Create deletions โ†’ garbage" + @echo " make ec-vacuum-status # 3. Check garbage ratios" + @echo "" + @echo "Automated Testing:" + @echo " make ec-vacuum-high # High garbage (should trigger)" + @echo " make ec-vacuum-low # Low garbage (should NOT trigger)" + @echo " make ec-vacuum-continuous # Continuous testing cycle" + @echo "" + @echo "Monitoring:" + @echo " make ec-vacuum-status # Quick EC volume status" + @echo " make ec-vacuum-monitor # Real-time monitoring" + @echo "" + @echo "Configuration:" + @echo " Visit: http://localhost:23646/maintenance/config/ec_vacuum" + @echo " Monitor: http://localhost:23646/maintenance" + @echo "" + @echo "๐Ÿ’ก EC volumes need time to encode after data generation" + @echo "๐Ÿ’ก Wait 2-3 minutes between generate and delete phases" + @echo "" + @echo "Understanding EC Vacuum:" + @echo " โ€ข Regular volumes โ†’ EC volumes (when >50MB)" + @echo " โ€ข EC vacuum cleans garbage from EC volumes" + @echo " โ€ข Requires different thresholds than regular vacuum" + @echo " โ€ข More complex due to shard distribution" + vacuum-help: ## Show vacuum testing help and examples @echo "๐Ÿงช Vacuum Testing Commands (Docker-based)" @echo "==========================================" diff --git a/docker/admin_integration/create_vacuum_test_data.go b/docker/admin_integration/create_vacuum_test_data.go index 46acdd4cd..5f43a9744 100644 --- a/docker/admin_integration/create_vacuum_test_data.go +++ b/docker/admin_integration/create_vacuum_test_data.go @@ -7,31 +7,36 @@ import ( "flag" "fmt" "io" + "io/ioutil" "log" "net/http" + "os" "time" ) var ( master = flag.String("master", "master:9333", "SeaweedFS master server address") + filer = flag.String("filer", "filer:8888", "SeaweedFS filer server address") + phase = flag.String("phase", "", "Phase to execute: generate, delete, status (for EC vacuum testing)") fileCount = flag.Int("files", 20, "Number of files to create") deleteRatio = flag.Float64("delete", 0.4, "Ratio of files to delete (0.0-1.0)") fileSizeKB = flag.Int("size", 100, "Size of each file in KB") ) -type AssignResult struct { - Fid string `json:"fid"` - Url string `json:"url"` - PublicUrl string `json:"publicUrl"` - Count int `json:"count"` - Error string `json:"error"` -} +// No longer needed - using filer-based operations func main() { flag.Parse() + // Handle EC vacuum testing phases + if *phase != "" { + handleECVacuumPhase() + return + } + fmt.Println("๐Ÿงช Creating fake data for vacuum task testing...") fmt.Printf("Master: %s\n", *master) + fmt.Printf("Filer: %s\n", *filer) fmt.Printf("Files to create: %d\n", *fileCount) fmt.Printf("Delete ratio: %.1f%%\n", *deleteRatio*100) fmt.Printf("File size: %d KB\n", *fileSizeKB) @@ -46,11 +51,11 @@ func main() { // Step 1: Create test files fmt.Println("๐Ÿ“ Step 1: Creating test files...") - fids := createTestFiles() + filePaths := createTestFiles() // Step 2: Delete some files to create garbage fmt.Println("๐Ÿ—‘๏ธ Step 2: Deleting files to create garbage...") - deleteFiles(fids) + deleteFiles(filePaths) // Step 3: Check volume status fmt.Println("๐Ÿ“Š Step 3: Checking volume status...") @@ -62,45 +67,41 @@ func main() { } func createTestFiles() []string { - var fids []string + var filePaths []string for i := 0; i < *fileCount; i++ { // Generate random file content fileData := make([]byte, *fileSizeKB*1024) rand.Read(fileData) - // Get file ID assignment - assign, err := assignFileId() - if err != nil { - log.Printf("Failed to assign file ID for file %d: %v", i, err) - continue - } + // Create file path + filePath := fmt.Sprintf("/vacuum_test/test_file_%d_%d.dat", time.Now().Unix(), i) - // Upload file - err = uploadFile(assign, fileData, fmt.Sprintf("test_file_%d.dat", i)) + // Upload file to filer + err := uploadFileToFiler(filePath, fileData) if err != nil { - log.Printf("Failed to upload file %d: %v", i, err) + log.Printf("Failed to upload file %d to filer: %v", i, err) continue } - fids = append(fids, assign.Fid) + filePaths = append(filePaths, filePath) if (i+1)%5 == 0 { fmt.Printf(" Created %d/%d files...\n", i+1, *fileCount) } } - fmt.Printf("โœ… Created %d files successfully\n\n", len(fids)) - return fids + fmt.Printf("โœ… Created %d files successfully\n\n", len(filePaths)) + return filePaths } -func deleteFiles(fids []string) { - deleteCount := int(float64(len(fids)) * *deleteRatio) +func deleteFiles(filePaths []string) { + deleteCount := int(float64(len(filePaths)) * *deleteRatio) for i := 0; i < deleteCount; i++ { - err := deleteFile(fids[i]) + err := deleteFileFromFiler(filePaths[i]) if err != nil { - log.Printf("Failed to delete file %s: %v", fids[i], err) + log.Printf("Failed to delete file %s: %v", filePaths[i], err) continue } @@ -112,46 +113,23 @@ func deleteFiles(fids []string) { fmt.Printf("โœ… Deleted %d files (%.1f%% of total)\n\n", deleteCount, *deleteRatio*100) } -func assignFileId() (*AssignResult, error) { - resp, err := http.Get(fmt.Sprintf("http://%s/dir/assign", *master)) - if err != nil { - return nil, err - } - defer resp.Body.Close() +// Filer-based functions for file operations - var result AssignResult - err = json.NewDecoder(resp.Body).Decode(&result) - if err != nil { - return nil, err - } +func uploadFileToFiler(filePath string, data []byte) error { + url := fmt.Sprintf("http://%s%s", *filer, filePath) - if result.Error != "" { - return nil, fmt.Errorf("assignment error: %s", result.Error) - } - - return &result, nil -} - -func uploadFile(assign *AssignResult, data []byte, filename string) error { - url := fmt.Sprintf("http://%s/%s", assign.Url, assign.Fid) - - body := &bytes.Buffer{} - body.Write(data) - - req, err := http.NewRequest("POST", url, body) + req, err := http.NewRequest("PUT", url, bytes.NewReader(data)) if err != nil { - return err + return fmt.Errorf("failed to create request: %v", err) } req.Header.Set("Content-Type", "application/octet-stream") - if filename != "" { - req.Header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) - } + req.ContentLength = int64(len(data)) - client := &http.Client{Timeout: 30 * time.Second} + client := &http.Client{Timeout: 60 * time.Second} resp, err := client.Do(req) if err != nil { - return err + return fmt.Errorf("failed to upload to filer: %v", err) } defer resp.Body.Close() @@ -163,21 +141,28 @@ func uploadFile(assign *AssignResult, data []byte, filename string) error { return nil } -func deleteFile(fid string) error { - url := fmt.Sprintf("http://%s/%s", *master, fid) +func deleteFileFromFiler(filePath string) error { + url := fmt.Sprintf("http://%s%s", *filer, filePath) req, err := http.NewRequest("DELETE", url, nil) if err != nil { - return err + return fmt.Errorf("failed to create delete request: %v", err) } - client := &http.Client{Timeout: 10 * time.Second} + client := &http.Client{Timeout: 30 * time.Second} resp, err := client.Do(req) if err != nil { - return err + return fmt.Errorf("failed to delete from filer: %v", err) } defer resp.Body.Close() + // Accept both 204 (No Content) and 404 (Not Found) as success + // 404 means file was already deleted + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusNotFound { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("delete failed with status %d: %s", resp.StatusCode, string(body)) + } + return nil } @@ -274,7 +259,302 @@ func printTestingInstructions() { fmt.Println(" Garbage ratios should decrease after vacuum operations") fmt.Println() - fmt.Printf("๐Ÿš€ Quick test command:\n") - fmt.Printf(" go run create_vacuum_test_data.go -files=0\n") + fmt.Printf("๐Ÿš€ Quick test commands:\n") + fmt.Printf(" go run create_vacuum_test_data.go -files=0 # Check volume status\n") + fmt.Printf(" go run create_vacuum_test_data.go -phase=status # Check EC volumes\n") + fmt.Println() + fmt.Println("๐Ÿ’ก All operations now use the filer for realistic file management") +} + +// EC Vacuum Testing Functions + +func handleECVacuumPhase() { + fmt.Printf("๐Ÿงช EC Vacuum Test Data Script - Phase: %s\n", *phase) + fmt.Printf("Master: %s\n", *master) + fmt.Printf("Filer: %s\n", *filer) + fmt.Println() + + switch *phase { + case "generate": + generateECTestData() + case "delete": + deleteFromECVolumes() + case "status": + checkECVolumeStatus() + default: + fmt.Printf("โŒ Unknown phase: %s\n", *phase) + fmt.Println("Valid phases: generate, delete, status") + } +} + +func generateECTestData() { + fmt.Println("๐Ÿ“ Generating large files to trigger EC encoding...") + fmt.Printf("Files to create: %d\n", *fileCount) + fmt.Printf("File size: %d KB\n", *fileSizeKB) + fmt.Printf("Filer: %s\n", *filer) + fmt.Println() + + var filePaths []string + + for i := 0; i < *fileCount; i++ { + // Generate random file content + fileData := make([]byte, *fileSizeKB*1024) + rand.Read(fileData) + + // Create file path + filePath := fmt.Sprintf("/ec_test/large_file_%d_%d.dat", time.Now().Unix(), i) + + // Upload file to filer + err := uploadFileToFiler(filePath, fileData) + if err != nil { + log.Printf("Failed to upload file %d to filer: %v", i, err) + continue + } + + filePaths = append(filePaths, filePath) + + if (i+1)%5 == 0 { + fmt.Printf(" Created %d/%d files... (latest: %s)\n", i+1, *fileCount, filePath) + } + } + + fmt.Printf("โœ… Created %d files successfully\n", len(filePaths)) + + // Store file paths for later deletion (using mounted working directory) + err := storeFilePathsToFile(filePaths, "ec_test_files.json") + if err != nil { + fmt.Printf("โš ๏ธ Warning: Failed to store file paths for deletion: %v\n", err) + fmt.Println("๐Ÿ’ก You can still test EC vacuum manually through the admin UI") + } else { + fmt.Printf("๐Ÿ“ Stored %d file paths for deletion phase\n", len(filePaths)) + } + + fmt.Println() + fmt.Println("๐Ÿ“Š Current volume status:") + checkVolumeStatus() + + fmt.Println() + fmt.Println("โณ Wait 2-3 minutes for EC encoding to complete...") + fmt.Println("๐Ÿ’ก EC encoding happens when volumes exceed 50MB") + fmt.Println("๐Ÿ’ก Run 'make ec-vacuum-status' to check EC volume creation") + fmt.Println("๐Ÿ’ก Then run 'make ec-vacuum-delete' to create garbage") +} + +func deleteFromECVolumes() { + fmt.Printf("๐Ÿ—‘๏ธ Creating deletions on EC volumes (ratio: %.1f%%)\n", *deleteRatio*100) + fmt.Printf("Filer: %s\n", *filer) + fmt.Println() + + // Load stored file paths from previous generation (using mounted working directory) + filePaths, err := loadFilePathsFromFile("ec_test_files.json") + if err != nil { + fmt.Printf("โŒ Failed to load stored file paths: %v\n", err) + fmt.Println("๐Ÿ’ก Run 'make ec-vacuum-generate' first to create files") + return + } + + if len(filePaths) == 0 { + fmt.Println("โŒ No stored file paths found. Run generate phase first.") + return + } + + fmt.Printf("Found %d stored file paths from previous generation\n", len(filePaths)) + + deleteCount := int(float64(len(filePaths)) * *deleteRatio) + fmt.Printf("Will delete %d files to create garbage\n", deleteCount) + fmt.Println() + + deletedCount := 0 + for i := 0; i < deleteCount && i < len(filePaths); i++ { + err := deleteFileFromFiler(filePaths[i]) + if err != nil { + log.Printf("Failed to delete file %s: %v", filePaths[i], err) + } else { + deletedCount++ + } + + if (i+1)%5 == 0 { + fmt.Printf(" Deleted %d/%d files...\n", i+1, deleteCount) + } + } + + fmt.Printf("โœ… Successfully deleted %d files (%.1f%% of total)\n", deletedCount, *deleteRatio*100) + fmt.Println() + fmt.Println("๐Ÿ“Š Updated status:") + time.Sleep(5 * time.Second) // Wait for deletion to be processed + checkECVolumeStatus() +} + +func checkECVolumeStatus() { + fmt.Println("๐Ÿ“Š EC Volume Status and Garbage Analysis") + fmt.Println("========================================") + + volumes := getVolumeStatusForDeletion() + if len(volumes) == 0 { + fmt.Println("โŒ No volumes found") + return + } + + fmt.Println() + fmt.Println("๐Ÿ“ˆ Volume Analysis (potential EC candidates and EC volumes):") + + regularECCandidates := 0 + ecVolumes := 0 + highGarbageCount := 0 + + for _, vol := range volumes { + garbageRatio := 0.0 + if vol.Size > 0 { + garbageRatio = float64(vol.DeletedByteCount) / float64(vol.Size) * 100 + } + + status := "๐Ÿ“" + volumeType := "Regular" + + if vol.ReadOnly && vol.Size > 40*1024*1024 { + status = "๐Ÿ”ง" + volumeType = "EC Volume" + ecVolumes++ + if garbageRatio > 30 { + status = "๐Ÿงน" + highGarbageCount++ + } + } else if vol.Size > 40*1024*1024 { + status = "๐Ÿ“ˆ" + volumeType = "EC Candidate" + regularECCandidates++ + } + + fmt.Printf(" %s Volume %d (%s): %s, Files: %d/%d, Garbage: %.1f%%", + status, vol.Id, volumeType, formatBytes(vol.Size), vol.FileCount, vol.DeleteCount, garbageRatio) + + if volumeType == "EC Volume" && garbageRatio > 30 { + fmt.Printf(" (Should trigger EC vacuum!)") + } + fmt.Printf("\n") + } + fmt.Println() + fmt.Println("๐ŸŽฏ EC Vacuum Testing Summary:") + fmt.Printf(" โ€ข Total volumes: %d\n", len(volumes)) + fmt.Printf(" โ€ข EC volumes (read-only >40MB): %d\n", ecVolumes) + fmt.Printf(" โ€ข EC candidates (>40MB): %d\n", regularECCandidates) + fmt.Printf(" โ€ข EC volumes with >30%% garbage: %d\n", highGarbageCount) + + if highGarbageCount > 0 { + fmt.Println() + fmt.Println("โœ… EC volumes with high garbage found!") + fmt.Println("๐Ÿ’ก Configure EC vacuum at: http://localhost:23646/maintenance/config/ec_vacuum") + fmt.Println("๐Ÿ’ก Monitor tasks at: http://localhost:23646/maintenance") + } else if ecVolumes > 0 { + fmt.Println() + fmt.Println("โ„น๏ธ EC volumes exist but garbage ratio is low") + fmt.Println("๐Ÿ’ก Run 'make ec-vacuum-delete' to create more garbage") + } else if regularECCandidates > 0 { + fmt.Println() + fmt.Println("โ„น๏ธ Large volumes found, waiting for EC encoding...") + fmt.Println("๐Ÿ’ก Wait a few more minutes for EC encoding to complete") + } else { + fmt.Println() + fmt.Println("โ„น๏ธ No large volumes found") + fmt.Println("๐Ÿ’ก Run 'make ec-vacuum-generate' to create large files for EC encoding") + } +} + +type VolumeInfo struct { + Id int `json:"Id"` + Size uint64 `json:"Size"` + FileCount int `json:"FileCount"` + DeleteCount int `json:"DeleteCount"` + DeletedByteCount uint64 `json:"DeletedByteCount"` + ReadOnly bool `json:"ReadOnly"` + Collection string `json:"Collection"` +} + +type VolumeStatus struct { + IsLeader bool `json:"IsLeader"` + Leader string `json:"Leader"` + Volumes []VolumeInfo `json:"Volumes"` +} + +func getVolumeStatusForDeletion() []VolumeInfo { + resp, err := http.Get(fmt.Sprintf("http://%s/vol/status", *master)) + if err != nil { + log.Printf("Failed to get volume status: %v", err) + return nil + } + defer resp.Body.Close() + + var volumeStatus VolumeStatus + err = json.NewDecoder(resp.Body).Decode(&volumeStatus) + if err != nil { + log.Printf("Failed to decode volume status: %v", err) + return nil + } + + return volumeStatus.Volumes +} + +type StoredFilePaths struct { + FilePaths []string `json:"file_paths"` + Timestamp time.Time `json:"timestamp"` + FileCount int `json:"file_count"` + FileSize int `json:"file_size_kb"` +} + +func storeFilePathsToFile(filePaths []string, filename string) error { + data := StoredFilePaths{ + FilePaths: filePaths, + Timestamp: time.Now(), + FileCount: len(filePaths), + FileSize: *fileSizeKB, + } + + jsonData, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal file paths: %v", err) + } + + err = ioutil.WriteFile(filename, jsonData, 0644) + if err != nil { + return fmt.Errorf("failed to write file paths to file: %v", err) + } + + return nil +} + +func loadFilePathsFromFile(filename string) ([]string, error) { + // Check if file exists + if _, err := os.Stat(filename); os.IsNotExist(err) { + return nil, fmt.Errorf("file paths storage file does not exist: %s", filename) + } + + data, err := ioutil.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("failed to read file paths file: %v", err) + } + + var storedData StoredFilePaths + err = json.Unmarshal(data, &storedData) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal file paths: %v", err) + } + + // Check if data is recent (within last 24 hours) + if time.Since(storedData.Timestamp) > 24*time.Hour { + return nil, fmt.Errorf("stored file paths are too old (%v), please regenerate", + time.Since(storedData.Timestamp)) + } + + fmt.Printf("Loaded %d file paths from %v (File size: %dKB each)\n", + len(storedData.FilePaths), storedData.Timestamp.Format("15:04:05"), storedData.FileSize) + + return storedData.FilePaths, nil +} + +func min(a, b int) int { + if a < b { + return a + } + return b } diff --git a/docker/admin_integration/ec_test_files.json b/docker/admin_integration/ec_test_files.json new file mode 100644 index 000000000..c1cbc0513 --- /dev/null +++ b/docker/admin_integration/ec_test_files.json @@ -0,0 +1,37 @@ +{ + "file_paths": [ + "/ec_test/large_file_1754816105_0.dat", + "/ec_test/large_file_1754816105_1.dat", + "/ec_test/large_file_1754816105_2.dat", + "/ec_test/large_file_1754816105_3.dat", + "/ec_test/large_file_1754816105_4.dat", + "/ec_test/large_file_1754816105_5.dat", + "/ec_test/large_file_1754816105_6.dat", + "/ec_test/large_file_1754816105_7.dat", + "/ec_test/large_file_1754816105_8.dat", + "/ec_test/large_file_1754816105_9.dat", + "/ec_test/large_file_1754816105_10.dat", + "/ec_test/large_file_1754816105_11.dat", + "/ec_test/large_file_1754816105_12.dat", + "/ec_test/large_file_1754816106_13.dat", + "/ec_test/large_file_1754816106_14.dat", + "/ec_test/large_file_1754816106_15.dat", + "/ec_test/large_file_1754816106_16.dat", + "/ec_test/large_file_1754816106_17.dat", + "/ec_test/large_file_1754816106_18.dat", + "/ec_test/large_file_1754816106_19.dat", + "/ec_test/large_file_1754816106_20.dat", + "/ec_test/large_file_1754816106_21.dat", + "/ec_test/large_file_1754816106_22.dat", + "/ec_test/large_file_1754816106_23.dat", + "/ec_test/large_file_1754816106_24.dat", + "/ec_test/large_file_1754816106_25.dat", + "/ec_test/large_file_1754816106_26.dat", + "/ec_test/large_file_1754816106_27.dat", + "/ec_test/large_file_1754816106_28.dat", + "/ec_test/large_file_1754816106_29.dat" + ], + "timestamp": "2025-08-10T08:55:06.363144049Z", + "file_count": 30, + "file_size_kb": 3000 +} \ No newline at end of file