Browse Source

generate ec data and deleted data

add-ec-vacuum
chrislu 4 months ago
parent
commit
bea9ca3616
  1. 150
      docker/admin_integration/Makefile
  2. 408
      docker/admin_integration/create_vacuum_test_data.go
  3. 37
      docker/admin_integration/ec_test_files.json

150
docker/admin_integration/Makefile

@ -48,15 +48,22 @@ help: ## Show this help message
@echo "🗑️ Vacuum Testing:"
@grep -E '^vacuum-.*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
@echo ""
@echo "🔧 EC Vacuum Testing:"
@grep -E '^ec-vacuum-.*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
@echo ""
@echo "📜 Monitoring:"
@grep -E '^(logs|admin-logs|worker-logs|master-logs|admin-ui):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
@echo ""
@echo "🚀 Quick Start:"
@echo " make start # Start cluster"
@echo " make vacuum-test # Test vacuum tasks"
@echo " make vacuum-help # Vacuum testing guide"
@echo ""
@echo "💡 For detailed vacuum testing: make vacuum-help"
@echo " make vacuum-test # Test regular vacuum tasks"
@echo " make ec-vacuum-test # Test EC vacuum tasks"
@echo " make vacuum-help # Regular vacuum guide"
@echo " make ec-vacuum-help # EC vacuum guide"
@echo ""
@echo "💡 For detailed testing guides:"
@echo " make vacuum-help # Regular vacuum testing"
@echo " make ec-vacuum-help # EC vacuum testing"
start: ## Start the complete SeaweedFS cluster with admin and workers
@echo "🚀 Starting SeaweedFS cluster with admin and workers..."
@ -316,6 +323,141 @@ vacuum-clean: ## Clean up vacuum test data (removes all volumes!)
@docker-compose -f $(COMPOSE_FILE) up -d
@echo "✅ Clean up complete. Fresh volumes ready for testing."
# EC Vacuum Testing Targets
ec-vacuum-test: ## Generate EC volumes and test EC vacuum functionality
@echo "🧪 SeaweedFS EC Vacuum Task Testing"
@echo "===================================="
@echo ""
@echo "1️⃣ Checking cluster health..."
@curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "✅ Master ready" || (echo "❌ Master not ready. Run 'make start' first." && exit 1)
@curl -s http://localhost:23646/ | grep -q "Admin" && echo "✅ Admin ready" || (echo "❌ Admin not ready. Run 'make start' first." && exit 1)
@echo ""
@echo "2️⃣ Generating data to trigger EC encoding..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=30 -size=3000
@echo ""
@echo "3️⃣ Waiting for EC encoding to complete..."
@echo "⏳ This may take 2-3 minutes..."
@sleep 120
@echo ""
@echo "4️⃣ Generating deletions on EC volumes..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=0.4
@echo ""
@echo "5️⃣ Configuration Instructions:"
@echo " Visit: http://localhost:23646/maintenance/config/ec_vacuum"
@echo " Set for testing:"
@echo " • Enable EC Vacuum Tasks: ✅ Checked"
@echo " • Garbage Threshold: 0.30 (30%)"
@echo " • Scan Interval: [60] [Seconds]"
@echo " • Min Volume Age: [2] [Minutes]"
@echo " • Max Concurrent: 2"
@echo ""
@echo "6️⃣ Monitor EC vacuum tasks at: http://localhost:23646/maintenance"
@echo ""
@echo "💡 Use 'make ec-vacuum-status' to check EC volume garbage ratios"
ec-vacuum-generate: ## Generate large files to trigger EC encoding
@echo "📁 Generating data to trigger EC encoding..."
@echo "Creating large files targeting >50MB per volume..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=25 -size=3000
@echo ""
@echo "⏳ Wait 2-3 minutes for EC encoding, then run 'make ec-vacuum-delete'"
ec-vacuum-delete: ## Create deletions on EC volumes to generate garbage
@echo "🗑️ Creating deletions on EC volumes..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=$${DELETE:-0.4}
@echo ""
@echo "💡 Use 'make ec-vacuum-status' to check garbage ratios"
ec-vacuum-status: ## Check EC volume status and garbage ratios
@echo "📊 EC Volume Status and Garbage Ratios"
@echo "====================================="
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=status
ec-vacuum-continuous: ## Generate continuous EC garbage for testing
@echo "🔄 Generating continuous EC garbage for vacuum testing..."
@echo "Running 3 rounds with 60-second intervals..."
@for i in {1..3}; do \
echo "Round $$i: Generating large files..."; \
docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=15 -size=4000; \
echo "Waiting 90 seconds for EC encoding..."; \
sleep 90; \
echo "Creating deletions..."; \
docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=0.5; \
echo "Waiting 60 seconds before next round..."; \
sleep 60; \
done
@echo "✅ Continuous EC vacuum test complete. Monitor admin UI for ec_vacuum tasks!"
ec-vacuum-high: ## Create high garbage on EC volumes (should trigger EC vacuum)
@echo "📁 Creating high garbage EC volumes (60% garbage)..."
@echo "1. Generating files for EC..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=20 -size=4000
@echo "2. Waiting for EC encoding..."
@sleep 120
@echo "3. Creating high garbage ratio..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=0.6
ec-vacuum-low: ## Create low garbage on EC volumes (should NOT trigger EC vacuum)
@echo "📁 Creating low garbage EC volumes (20% garbage)..."
@echo "1. Generating files for EC..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=20 -size=4000
@echo "2. Waiting for EC encoding..."
@sleep 120
@echo "3. Creating low garbage ratio..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=0.2
ec-vacuum-monitor: ## Monitor EC vacuum task activity in real-time
@echo "📊 Monitoring EC Vacuum Task Activity"
@echo "===================================="
@echo "Press Ctrl+C to stop monitoring"
@echo ""
@while true; do \
echo "=== $(date) ==="; \
docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=status; \
echo ""; \
echo "🔍 Recent admin logs (EC vacuum activity):"; \
docker-compose -f $(COMPOSE_FILE) logs --tail=5 admin | grep -i "ec_vacuum\|vacuum.*ec" || echo "No recent EC vacuum activity"; \
echo ""; \
sleep 30; \
done
ec-vacuum-help: ## Show EC vacuum testing help and examples
@echo "🧪 EC Vacuum Testing Commands"
@echo "============================="
@echo ""
@echo "Quick Start:"
@echo " make start # Start SeaweedFS cluster"
@echo " make ec-vacuum-test # Full EC vacuum test cycle"
@echo " make ec-vacuum-status # Check EC volume status"
@echo ""
@echo "Manual Testing:"
@echo " make ec-vacuum-generate # 1. Generate data → trigger EC"
@echo " # Wait 2-3 minutes for EC encoding to complete"
@echo " make ec-vacuum-delete # 2. Create deletions → garbage"
@echo " make ec-vacuum-status # 3. Check garbage ratios"
@echo ""
@echo "Automated Testing:"
@echo " make ec-vacuum-high # High garbage (should trigger)"
@echo " make ec-vacuum-low # Low garbage (should NOT trigger)"
@echo " make ec-vacuum-continuous # Continuous testing cycle"
@echo ""
@echo "Monitoring:"
@echo " make ec-vacuum-status # Quick EC volume status"
@echo " make ec-vacuum-monitor # Real-time monitoring"
@echo ""
@echo "Configuration:"
@echo " Visit: http://localhost:23646/maintenance/config/ec_vacuum"
@echo " Monitor: http://localhost:23646/maintenance"
@echo ""
@echo "💡 EC volumes need time to encode after data generation"
@echo "💡 Wait 2-3 minutes between generate and delete phases"
@echo ""
@echo "Understanding EC Vacuum:"
@echo " • Regular volumes → EC volumes (when >50MB)"
@echo " • EC vacuum cleans garbage from EC volumes"
@echo " • Requires different thresholds than regular vacuum"
@echo " • More complex due to shard distribution"
vacuum-help: ## Show vacuum testing help and examples
@echo "🧪 Vacuum Testing Commands (Docker-based)"
@echo "=========================================="

408
docker/admin_integration/create_vacuum_test_data.go

@ -7,31 +7,36 @@ import (
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"time"
)
var (
master = flag.String("master", "master:9333", "SeaweedFS master server address")
filer = flag.String("filer", "filer:8888", "SeaweedFS filer server address")
phase = flag.String("phase", "", "Phase to execute: generate, delete, status (for EC vacuum testing)")
fileCount = flag.Int("files", 20, "Number of files to create")
deleteRatio = flag.Float64("delete", 0.4, "Ratio of files to delete (0.0-1.0)")
fileSizeKB = flag.Int("size", 100, "Size of each file in KB")
)
type AssignResult struct {
Fid string `json:"fid"`
Url string `json:"url"`
PublicUrl string `json:"publicUrl"`
Count int `json:"count"`
Error string `json:"error"`
}
// No longer needed - using filer-based operations
func main() {
flag.Parse()
// Handle EC vacuum testing phases
if *phase != "" {
handleECVacuumPhase()
return
}
fmt.Println("🧪 Creating fake data for vacuum task testing...")
fmt.Printf("Master: %s\n", *master)
fmt.Printf("Filer: %s\n", *filer)
fmt.Printf("Files to create: %d\n", *fileCount)
fmt.Printf("Delete ratio: %.1f%%\n", *deleteRatio*100)
fmt.Printf("File size: %d KB\n", *fileSizeKB)
@ -46,11 +51,11 @@ func main() {
// Step 1: Create test files
fmt.Println("📁 Step 1: Creating test files...")
fids := createTestFiles()
filePaths := createTestFiles()
// Step 2: Delete some files to create garbage
fmt.Println("🗑️ Step 2: Deleting files to create garbage...")
deleteFiles(fids)
deleteFiles(filePaths)
// Step 3: Check volume status
fmt.Println("📊 Step 3: Checking volume status...")
@ -62,45 +67,41 @@ func main() {
}
func createTestFiles() []string {
var fids []string
var filePaths []string
for i := 0; i < *fileCount; i++ {
// Generate random file content
fileData := make([]byte, *fileSizeKB*1024)
rand.Read(fileData)
// Get file ID assignment
assign, err := assignFileId()
if err != nil {
log.Printf("Failed to assign file ID for file %d: %v", i, err)
continue
}
// Create file path
filePath := fmt.Sprintf("/vacuum_test/test_file_%d_%d.dat", time.Now().Unix(), i)
// Upload file
err = uploadFile(assign, fileData, fmt.Sprintf("test_file_%d.dat", i))
// Upload file to filer
err := uploadFileToFiler(filePath, fileData)
if err != nil {
log.Printf("Failed to upload file %d: %v", i, err)
log.Printf("Failed to upload file %d to filer: %v", i, err)
continue
}
fids = append(fids, assign.Fid)
filePaths = append(filePaths, filePath)
if (i+1)%5 == 0 {
fmt.Printf(" Created %d/%d files...\n", i+1, *fileCount)
}
}
fmt.Printf("✅ Created %d files successfully\n\n", len(fids))
return fids
fmt.Printf("✅ Created %d files successfully\n\n", len(filePaths))
return filePaths
}
func deleteFiles(fids []string) {
deleteCount := int(float64(len(fids)) * *deleteRatio)
func deleteFiles(filePaths []string) {
deleteCount := int(float64(len(filePaths)) * *deleteRatio)
for i := 0; i < deleteCount; i++ {
err := deleteFile(fids[i])
err := deleteFileFromFiler(filePaths[i])
if err != nil {
log.Printf("Failed to delete file %s: %v", fids[i], err)
log.Printf("Failed to delete file %s: %v", filePaths[i], err)
continue
}
@ -112,46 +113,23 @@ func deleteFiles(fids []string) {
fmt.Printf("✅ Deleted %d files (%.1f%% of total)\n\n", deleteCount, *deleteRatio*100)
}
func assignFileId() (*AssignResult, error) {
resp, err := http.Get(fmt.Sprintf("http://%s/dir/assign", *master))
if err != nil {
return nil, err
}
defer resp.Body.Close()
// Filer-based functions for file operations
var result AssignResult
err = json.NewDecoder(resp.Body).Decode(&result)
if err != nil {
return nil, err
}
func uploadFileToFiler(filePath string, data []byte) error {
url := fmt.Sprintf("http://%s%s", *filer, filePath)
if result.Error != "" {
return nil, fmt.Errorf("assignment error: %s", result.Error)
}
return &result, nil
}
func uploadFile(assign *AssignResult, data []byte, filename string) error {
url := fmt.Sprintf("http://%s/%s", assign.Url, assign.Fid)
body := &bytes.Buffer{}
body.Write(data)
req, err := http.NewRequest("POST", url, body)
req, err := http.NewRequest("PUT", url, bytes.NewReader(data))
if err != nil {
return err
return fmt.Errorf("failed to create request: %v", err)
}
req.Header.Set("Content-Type", "application/octet-stream")
if filename != "" {
req.Header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
}
req.ContentLength = int64(len(data))
client := &http.Client{Timeout: 30 * time.Second}
client := &http.Client{Timeout: 60 * time.Second}
resp, err := client.Do(req)
if err != nil {
return err
return fmt.Errorf("failed to upload to filer: %v", err)
}
defer resp.Body.Close()
@ -163,21 +141,28 @@ func uploadFile(assign *AssignResult, data []byte, filename string) error {
return nil
}
func deleteFile(fid string) error {
url := fmt.Sprintf("http://%s/%s", *master, fid)
func deleteFileFromFiler(filePath string) error {
url := fmt.Sprintf("http://%s%s", *filer, filePath)
req, err := http.NewRequest("DELETE", url, nil)
if err != nil {
return err
return fmt.Errorf("failed to create delete request: %v", err)
}
client := &http.Client{Timeout: 10 * time.Second}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return err
return fmt.Errorf("failed to delete from filer: %v", err)
}
defer resp.Body.Close()
// Accept both 204 (No Content) and 404 (Not Found) as success
// 404 means file was already deleted
if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusNotFound {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("delete failed with status %d: %s", resp.StatusCode, string(body))
}
return nil
}
@ -274,7 +259,302 @@ func printTestingInstructions() {
fmt.Println(" Garbage ratios should decrease after vacuum operations")
fmt.Println()
fmt.Printf("🚀 Quick test command:\n")
fmt.Printf(" go run create_vacuum_test_data.go -files=0\n")
fmt.Printf("🚀 Quick test commands:\n")
fmt.Printf(" go run create_vacuum_test_data.go -files=0 # Check volume status\n")
fmt.Printf(" go run create_vacuum_test_data.go -phase=status # Check EC volumes\n")
fmt.Println()
fmt.Println("💡 All operations now use the filer for realistic file management")
}
// EC Vacuum Testing Functions
func handleECVacuumPhase() {
fmt.Printf("🧪 EC Vacuum Test Data Script - Phase: %s\n", *phase)
fmt.Printf("Master: %s\n", *master)
fmt.Printf("Filer: %s\n", *filer)
fmt.Println()
switch *phase {
case "generate":
generateECTestData()
case "delete":
deleteFromECVolumes()
case "status":
checkECVolumeStatus()
default:
fmt.Printf("❌ Unknown phase: %s\n", *phase)
fmt.Println("Valid phases: generate, delete, status")
}
}
func generateECTestData() {
fmt.Println("📁 Generating large files to trigger EC encoding...")
fmt.Printf("Files to create: %d\n", *fileCount)
fmt.Printf("File size: %d KB\n", *fileSizeKB)
fmt.Printf("Filer: %s\n", *filer)
fmt.Println()
var filePaths []string
for i := 0; i < *fileCount; i++ {
// Generate random file content
fileData := make([]byte, *fileSizeKB*1024)
rand.Read(fileData)
// Create file path
filePath := fmt.Sprintf("/ec_test/large_file_%d_%d.dat", time.Now().Unix(), i)
// Upload file to filer
err := uploadFileToFiler(filePath, fileData)
if err != nil {
log.Printf("Failed to upload file %d to filer: %v", i, err)
continue
}
filePaths = append(filePaths, filePath)
if (i+1)%5 == 0 {
fmt.Printf(" Created %d/%d files... (latest: %s)\n", i+1, *fileCount, filePath)
}
}
fmt.Printf("✅ Created %d files successfully\n", len(filePaths))
// Store file paths for later deletion (using mounted working directory)
err := storeFilePathsToFile(filePaths, "ec_test_files.json")
if err != nil {
fmt.Printf("⚠️ Warning: Failed to store file paths for deletion: %v\n", err)
fmt.Println("💡 You can still test EC vacuum manually through the admin UI")
} else {
fmt.Printf("📝 Stored %d file paths for deletion phase\n", len(filePaths))
}
fmt.Println()
fmt.Println("📊 Current volume status:")
checkVolumeStatus()
fmt.Println()
fmt.Println("⏳ Wait 2-3 minutes for EC encoding to complete...")
fmt.Println("💡 EC encoding happens when volumes exceed 50MB")
fmt.Println("💡 Run 'make ec-vacuum-status' to check EC volume creation")
fmt.Println("💡 Then run 'make ec-vacuum-delete' to create garbage")
}
func deleteFromECVolumes() {
fmt.Printf("🗑️ Creating deletions on EC volumes (ratio: %.1f%%)\n", *deleteRatio*100)
fmt.Printf("Filer: %s\n", *filer)
fmt.Println()
// Load stored file paths from previous generation (using mounted working directory)
filePaths, err := loadFilePathsFromFile("ec_test_files.json")
if err != nil {
fmt.Printf("❌ Failed to load stored file paths: %v\n", err)
fmt.Println("💡 Run 'make ec-vacuum-generate' first to create files")
return
}
if len(filePaths) == 0 {
fmt.Println("❌ No stored file paths found. Run generate phase first.")
return
}
fmt.Printf("Found %d stored file paths from previous generation\n", len(filePaths))
deleteCount := int(float64(len(filePaths)) * *deleteRatio)
fmt.Printf("Will delete %d files to create garbage\n", deleteCount)
fmt.Println()
deletedCount := 0
for i := 0; i < deleteCount && i < len(filePaths); i++ {
err := deleteFileFromFiler(filePaths[i])
if err != nil {
log.Printf("Failed to delete file %s: %v", filePaths[i], err)
} else {
deletedCount++
}
if (i+1)%5 == 0 {
fmt.Printf(" Deleted %d/%d files...\n", i+1, deleteCount)
}
}
fmt.Printf("✅ Successfully deleted %d files (%.1f%% of total)\n", deletedCount, *deleteRatio*100)
fmt.Println()
fmt.Println("📊 Updated status:")
time.Sleep(5 * time.Second) // Wait for deletion to be processed
checkECVolumeStatus()
}
func checkECVolumeStatus() {
fmt.Println("📊 EC Volume Status and Garbage Analysis")
fmt.Println("========================================")
volumes := getVolumeStatusForDeletion()
if len(volumes) == 0 {
fmt.Println("❌ No volumes found")
return
}
fmt.Println()
fmt.Println("📈 Volume Analysis (potential EC candidates and EC volumes):")
regularECCandidates := 0
ecVolumes := 0
highGarbageCount := 0
for _, vol := range volumes {
garbageRatio := 0.0
if vol.Size > 0 {
garbageRatio = float64(vol.DeletedByteCount) / float64(vol.Size) * 100
}
status := "📁"
volumeType := "Regular"
if vol.ReadOnly && vol.Size > 40*1024*1024 {
status = "🔧"
volumeType = "EC Volume"
ecVolumes++
if garbageRatio > 30 {
status = "🧹"
highGarbageCount++
}
} else if vol.Size > 40*1024*1024 {
status = "📈"
volumeType = "EC Candidate"
regularECCandidates++
}
fmt.Printf(" %s Volume %d (%s): %s, Files: %d/%d, Garbage: %.1f%%",
status, vol.Id, volumeType, formatBytes(vol.Size), vol.FileCount, vol.DeleteCount, garbageRatio)
if volumeType == "EC Volume" && garbageRatio > 30 {
fmt.Printf(" (Should trigger EC vacuum!)")
}
fmt.Printf("\n")
}
fmt.Println()
fmt.Println("🎯 EC Vacuum Testing Summary:")
fmt.Printf(" • Total volumes: %d\n", len(volumes))
fmt.Printf(" • EC volumes (read-only >40MB): %d\n", ecVolumes)
fmt.Printf(" • EC candidates (>40MB): %d\n", regularECCandidates)
fmt.Printf(" • EC volumes with >30%% garbage: %d\n", highGarbageCount)
if highGarbageCount > 0 {
fmt.Println()
fmt.Println("✅ EC volumes with high garbage found!")
fmt.Println("💡 Configure EC vacuum at: http://localhost:23646/maintenance/config/ec_vacuum")
fmt.Println("💡 Monitor tasks at: http://localhost:23646/maintenance")
} else if ecVolumes > 0 {
fmt.Println()
fmt.Println("ℹ️ EC volumes exist but garbage ratio is low")
fmt.Println("💡 Run 'make ec-vacuum-delete' to create more garbage")
} else if regularECCandidates > 0 {
fmt.Println()
fmt.Println("ℹ️ Large volumes found, waiting for EC encoding...")
fmt.Println("💡 Wait a few more minutes for EC encoding to complete")
} else {
fmt.Println()
fmt.Println("ℹ️ No large volumes found")
fmt.Println("💡 Run 'make ec-vacuum-generate' to create large files for EC encoding")
}
}
type VolumeInfo struct {
Id int `json:"Id"`
Size uint64 `json:"Size"`
FileCount int `json:"FileCount"`
DeleteCount int `json:"DeleteCount"`
DeletedByteCount uint64 `json:"DeletedByteCount"`
ReadOnly bool `json:"ReadOnly"`
Collection string `json:"Collection"`
}
type VolumeStatus struct {
IsLeader bool `json:"IsLeader"`
Leader string `json:"Leader"`
Volumes []VolumeInfo `json:"Volumes"`
}
func getVolumeStatusForDeletion() []VolumeInfo {
resp, err := http.Get(fmt.Sprintf("http://%s/vol/status", *master))
if err != nil {
log.Printf("Failed to get volume status: %v", err)
return nil
}
defer resp.Body.Close()
var volumeStatus VolumeStatus
err = json.NewDecoder(resp.Body).Decode(&volumeStatus)
if err != nil {
log.Printf("Failed to decode volume status: %v", err)
return nil
}
return volumeStatus.Volumes
}
type StoredFilePaths struct {
FilePaths []string `json:"file_paths"`
Timestamp time.Time `json:"timestamp"`
FileCount int `json:"file_count"`
FileSize int `json:"file_size_kb"`
}
func storeFilePathsToFile(filePaths []string, filename string) error {
data := StoredFilePaths{
FilePaths: filePaths,
Timestamp: time.Now(),
FileCount: len(filePaths),
FileSize: *fileSizeKB,
}
jsonData, err := json.MarshalIndent(data, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal file paths: %v", err)
}
err = ioutil.WriteFile(filename, jsonData, 0644)
if err != nil {
return fmt.Errorf("failed to write file paths to file: %v", err)
}
return nil
}
func loadFilePathsFromFile(filename string) ([]string, error) {
// Check if file exists
if _, err := os.Stat(filename); os.IsNotExist(err) {
return nil, fmt.Errorf("file paths storage file does not exist: %s", filename)
}
data, err := ioutil.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("failed to read file paths file: %v", err)
}
var storedData StoredFilePaths
err = json.Unmarshal(data, &storedData)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal file paths: %v", err)
}
// Check if data is recent (within last 24 hours)
if time.Since(storedData.Timestamp) > 24*time.Hour {
return nil, fmt.Errorf("stored file paths are too old (%v), please regenerate",
time.Since(storedData.Timestamp))
}
fmt.Printf("Loaded %d file paths from %v (File size: %dKB each)\n",
len(storedData.FilePaths), storedData.Timestamp.Format("15:04:05"), storedData.FileSize)
return storedData.FilePaths, nil
}
func min(a, b int) int {
if a < b {
return a
}
return b
}

37
docker/admin_integration/ec_test_files.json

@ -0,0 +1,37 @@
{
"file_paths": [
"/ec_test/large_file_1754816105_0.dat",
"/ec_test/large_file_1754816105_1.dat",
"/ec_test/large_file_1754816105_2.dat",
"/ec_test/large_file_1754816105_3.dat",
"/ec_test/large_file_1754816105_4.dat",
"/ec_test/large_file_1754816105_5.dat",
"/ec_test/large_file_1754816105_6.dat",
"/ec_test/large_file_1754816105_7.dat",
"/ec_test/large_file_1754816105_8.dat",
"/ec_test/large_file_1754816105_9.dat",
"/ec_test/large_file_1754816105_10.dat",
"/ec_test/large_file_1754816105_11.dat",
"/ec_test/large_file_1754816105_12.dat",
"/ec_test/large_file_1754816106_13.dat",
"/ec_test/large_file_1754816106_14.dat",
"/ec_test/large_file_1754816106_15.dat",
"/ec_test/large_file_1754816106_16.dat",
"/ec_test/large_file_1754816106_17.dat",
"/ec_test/large_file_1754816106_18.dat",
"/ec_test/large_file_1754816106_19.dat",
"/ec_test/large_file_1754816106_20.dat",
"/ec_test/large_file_1754816106_21.dat",
"/ec_test/large_file_1754816106_22.dat",
"/ec_test/large_file_1754816106_23.dat",
"/ec_test/large_file_1754816106_24.dat",
"/ec_test/large_file_1754816106_25.dat",
"/ec_test/large_file_1754816106_26.dat",
"/ec_test/large_file_1754816106_27.dat",
"/ec_test/large_file_1754816106_28.dat",
"/ec_test/large_file_1754816106_29.dat"
],
"timestamp": "2025-08-10T08:55:06.363144049Z",
"file_count": 30,
"file_size_kb": 3000
}
Loading…
Cancel
Save