From 213dd9b470492aeeb730732a8036c15c6dc8c94f Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 11 Aug 2025 00:20:37 -0700 Subject: [PATCH] testing --- docker/admin_integration/Makefile | 68 +++- docker/admin_integration/ec_vacuum_test.go | 341 +++++++++++++++++++++ 2 files changed, 404 insertions(+), 5 deletions(-) create mode 100644 docker/admin_integration/ec_vacuum_test.go diff --git a/docker/admin_integration/Makefile b/docker/admin_integration/Makefile index 270b793f2..0ac53ce72 100644 --- a/docker/admin_integration/Makefile +++ b/docker/admin_integration/Makefile @@ -1,7 +1,7 @@ # SeaweedFS Admin Integration Test Makefile # Tests the admin server and worker functionality using official weed commands -.PHONY: help build build-and-restart restart-workers start stop restart logs clean status test admin-ui worker-logs master-logs admin-logs vacuum-test vacuum-demo vacuum-status vacuum-data vacuum-data-high vacuum-data-low vacuum-continuous vacuum-clean vacuum-help +.PHONY: help build build-and-restart restart-workers restart-admin start stop restart logs clean status test admin-ui worker-logs master-logs admin-logs vacuum-test vacuum-demo vacuum-status vacuum-data vacuum-data-high vacuum-data-low vacuum-continuous vacuum-clean vacuum-help file-generation-test file-deletion-test all-file-tests .DEFAULT_GOAL := help COMPOSE_FILE := docker-compose-ec-test.yml @@ -34,16 +34,22 @@ restart-workers: ## Restart all workers to reconnect to admin server @docker-compose -f $(COMPOSE_FILE) restart worker1 worker2 worker3 @echo "โœ… Workers restarted and will reconnect to admin server" +restart-admin: ## Restart admin server (useful after deadlock fixes) + @echo "๐Ÿ”„ Restarting admin server..." + @docker-compose -f $(COMPOSE_FILE) restart admin + @echo "โœ… Admin server restarted" + @echo "๐ŸŒ Admin UI: http://localhost:23646/" + help: ## Show this help message @echo "SeaweedFS Admin Integration Test" @echo "================================" @echo "Tests admin server task distribution to workers using official weed commands" @echo "" @echo "๐Ÿ—๏ธ Cluster Management:" - @grep -E '^(start|stop|restart|clean|status|build):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}' + @grep -E '^(start|stop|restart|restart-admin|restart-workers|clean|status|build):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}' @echo "" @echo "๐Ÿงช Testing:" - @grep -E '^(test|demo|validate|quick-test):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}' + @grep -E '^(test|demo|validate|quick-test|file-.*-test|all-file-tests):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}' @echo "" @echo "๐Ÿ—‘๏ธ Vacuum Testing:" @grep -E '^vacuum-.*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}' @@ -56,8 +62,11 @@ help: ## Show this help message @echo "" @echo "๐Ÿš€ Quick Start:" @echo " make start # Start cluster" - @echo " make vacuum-test # Test regular vacuum tasks" - @echo " make ec-vacuum-test # Test EC vacuum tasks" + @echo " make vacuum-test # Test regular vacuum tasks" + @echo " make ec-vacuum-test # Test EC vacuum tasks" + @echo " make file-generation-test # Test file generation (600 files)" + @echo " make file-deletion-test # Test file deletion (300 files)" + @echo " make all-file-tests # Run both file tests" @echo " make vacuum-help # Regular vacuum guide" @echo " make ec-vacuum-help # EC vacuum guide" @echo "" @@ -324,6 +333,55 @@ vacuum-clean: ## Clean up vacuum test data (removes all volumes!) @echo "โœ… Clean up complete. Fresh volumes ready for testing." # EC Vacuum Testing Targets +file-generation-test: ## Run the file generation test (600 files of 100KB to volume 1) + @echo "๐Ÿงช Running File Generation Test" + @echo "===============================" + @echo "1๏ธโƒฃ Ensuring cluster is running..." + @docker-compose -f $(COMPOSE_FILE) up -d + @echo "2๏ธโƒฃ Waiting for cluster to be ready..." + @sleep 10 + @echo "3๏ธโƒฃ Running file generation test..." + @go test -v . -run TestFileGeneration + @echo "โœ… File generation test completed!" + @echo "๐Ÿ’ก This test generates 600 files of 100KB each to volume 1 with hardcoded cookie" + +file-deletion-test: ## Run the file deletion test (delete 300 files from volume 1) + @echo "๐Ÿงช Running File Deletion Test" + @echo "=============================" + @echo "1๏ธโƒฃ Ensuring cluster is running..." + @docker-compose -f $(COMPOSE_FILE) up -d + @echo "2๏ธโƒฃ Waiting for cluster to be ready..." + @sleep 10 + @echo "3๏ธโƒฃ Running file deletion test..." + @go test -v . -run TestFileDeletion + @echo "โœ… File deletion test completed!" + @echo "๐Ÿ’ก This test generates 600 files then deletes exactly 300 of them" + +all-file-tests: ## Run both file generation and deletion tests + @echo "๐Ÿงช Running All File Tests" + @echo "=========================" + @echo "1๏ธโƒฃ Ensuring cluster is running..." + @docker-compose -f $(COMPOSE_FILE) up -d + @echo "2๏ธโƒฃ Waiting for cluster to be ready..." + @sleep 10 + @echo "3๏ธโƒฃ Running file generation test..." + @go test -v . -run TestFileGeneration + @echo "4๏ธโƒฃ Running file deletion test..." + @go test -v . -run TestFileDeletion + @echo "โœ… All file tests completed!" + +ec-vacuum-go-test: ## Run the Go-based EC vacuum integration test with detailed file ID tracking (legacy) + @echo "๐Ÿงช Running EC Vacuum Go Integration Test" + @echo "========================================" + @echo "1๏ธโƒฃ Ensuring cluster is running..." + @docker-compose -f $(COMPOSE_FILE) up -d + @echo "2๏ธโƒฃ Waiting for cluster to be ready..." + @sleep 10 + @echo "3๏ธโƒฃ Running Go test with file ID tracking..." + @go test -v . -run TestECVolumeVacuum + @echo "โœ… EC Vacuum Go test completed!" + @echo "๐Ÿ’ก This test shows which file IDs are written and deleted" + ec-vacuum-test: ## Generate EC volumes and test EC vacuum functionality @echo "๐Ÿงช SeaweedFS EC Vacuum Task Testing" @echo "====================================" diff --git a/docker/admin_integration/ec_vacuum_test.go b/docker/admin_integration/ec_vacuum_test.go new file mode 100644 index 000000000..3badf674f --- /dev/null +++ b/docker/admin_integration/ec_vacuum_test.go @@ -0,0 +1,341 @@ +package main + +import ( + "bytes" + "crypto/rand" + "encoding/json" + "fmt" + "io" + "net/http" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/stretchr/testify/require" +) + +// TestFileGeneration tests generating 600 files of 100KB each targeting volume 1 with hardcoded cookie +func TestFileGeneration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping file generation test in short mode") + } + + // Set up test cluster + cluster, cleanup := setupTestCluster(t) + defer cleanup() + + // Wait for cluster to be ready + require.NoError(t, waitForClusterReady()) + t.Logf("Test cluster ready with master at %s", cluster.masterAddress) + + // Generate 600 files of 100KB each targeting volume 1 + const targetVolumeId = needle.VolumeId(1) + const fileCount = 600 + const fileSize = 100 * 1024 // 100KB files + + fileIds := generateFilesToVolume1(t, fileCount, fileSize) + t.Logf("Generated %d files of %dKB each targeting volume %d", len(fileIds), fileSize/1024, targetVolumeId) + t.Logf("๐Ÿ“ Sample file IDs created: %v", fileIds[:5]) // Show first 5 file IDs + + // Summary + t.Logf("๐Ÿ“Š File Generation Summary:") + t.Logf(" โ€ข Volume ID: %d", targetVolumeId) + t.Logf(" โ€ข Total files created: %d", len(fileIds)) + t.Logf(" โ€ข File size: %dKB each", fileSize/1024) + t.Logf(" โ€ข Hardcoded cookie: 0x12345678") +} + +// TestFileDeletion tests deleting exactly 300 files from volume 1 +func TestFileDeletion(t *testing.T) { + if testing.Short() { + t.Skip("Skipping file deletion test in short mode") + } + + // Set up test cluster + cluster, cleanup := setupTestCluster(t) + defer cleanup() + + // Wait for cluster to be ready + require.NoError(t, waitForClusterReady()) + t.Logf("Test cluster ready with master at %s", cluster.masterAddress) + + // First generate some files to delete + const targetVolumeId = needle.VolumeId(1) + const fileCount = 600 + const fileSize = 100 * 1024 // 100KB files + + t.Logf("Pre-generating files for deletion test...") + fileIds := generateFilesToVolume1(t, fileCount, fileSize) + t.Logf("Pre-generated %d files for deletion test", len(fileIds)) + + // Delete exactly 300 files from the volume + const deleteCount = 300 + deletedFileIds := deleteSpecificFilesFromVolume(t, fileIds, deleteCount) + t.Logf("Deleted exactly %d files from volume %d", len(deletedFileIds), targetVolumeId) + t.Logf("๐Ÿ—‘๏ธ Sample deleted file IDs: %v", deletedFileIds[:5]) // Show first 5 deleted file IDs + + // Summary + t.Logf("๐Ÿ“Š File Deletion Summary:") + t.Logf(" โ€ข Volume ID: %d", targetVolumeId) + t.Logf(" โ€ข Files available for deletion: %d", len(fileIds)) + t.Logf(" โ€ข Files deleted: %d", len(deletedFileIds)) + t.Logf(" โ€ข Files remaining: %d", len(fileIds)-len(deletedFileIds)) + t.Logf(" โ€ข Deletion success rate: %.1f%%", float64(len(deletedFileIds))/float64(deleteCount)*100) +} + +// generateFilesToVolume1 creates 600 files of 100KB each targeting volume 1 with hardcoded cookie +func generateFilesToVolume1(t *testing.T, fileCount int, fileSize int) []string { + const targetVolumeId = needle.VolumeId(1) + const hardcodedCookie = uint32(0x12345678) // Hardcoded cookie for volume 1 files + + var fileIds []string + t.Logf("Starting generation of %d files of %dKB each targeting volume %d", fileCount, fileSize/1024, targetVolumeId) + + for i := 0; i < fileCount; i++ { + // Generate file content + fileData := make([]byte, fileSize) + rand.Read(fileData) + + // Generate file ID targeting volume 1 with hardcoded cookie + // Use high needle key values to avoid collisions with assigned IDs + needleKey := uint64(i) + 0x2000000 // Start from a high offset for volume 1 + generatedFid := needle.NewFileId(targetVolumeId, needleKey, hardcodedCookie) + + // Upload directly to volume 1 + err := uploadFileToVolumeDirectly(t, generatedFid, fileData) + require.NoError(t, err) + + fileIds = append(fileIds, generatedFid.String()) + + // Log progress for first few files and every 50th file + if i < 5 || (i+1)%50 == 0 { + t.Logf("โœ… Generated file %d/%d targeting volume %d: %s", i+1, fileCount, targetVolumeId, generatedFid.String()) + } + } + + t.Logf("โœ… Successfully generated %d files targeting volume %d with hardcoded cookie 0x%08x", len(fileIds), targetVolumeId, hardcodedCookie) + return fileIds +} + +// uploadFileToVolume uploads file data to an assigned volume server +func uploadFileToVolume(serverUrl, fid string, data []byte) error { + uploadUrl := fmt.Sprintf("http://%s/%s", serverUrl, fid) + + req, err := http.NewRequest("PUT", uploadUrl, bytes.NewReader(data)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/octet-stream") + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// uploadFileToVolumeDirectly uploads a file using a generated file ID +func uploadFileToVolumeDirectly(t *testing.T, fid *needle.FileId, data []byte) error { + // Find the volume server hosting this volume + locations, found := findVolumeLocations(fid.VolumeId) + if !found { + return fmt.Errorf("volume %d not found", fid.VolumeId) + } + + // Upload to the first available location + serverUrl := locations[0] + uploadUrl := fmt.Sprintf("http://%s/%s", serverUrl, fid.String()) + + req, err := http.NewRequest("PUT", uploadUrl, bytes.NewReader(data)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/octet-stream") + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("direct upload failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// findVolumeLocations finds the server locations for a given volume using HTTP lookup +func findVolumeLocations(volumeId needle.VolumeId) ([]string, bool) { + // Query master for volume locations using HTTP API + resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:9333/dir/lookup?volumeId=%d", volumeId)) + if err != nil { + return nil, false + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, false + } + + // Parse JSON response + type LookupResult struct { + VolumeId string `json:"volumeId"` + Locations []struct { + Url string `json:"url"` + PublicUrl string `json:"publicUrl"` + } `json:"locations"` + Error string `json:"error"` + } + + var result LookupResult + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + // Fallback to default locations for testing + return []string{"127.0.0.1:8080"}, true + } + + if result.Error != "" { + return nil, false + } + + var serverUrls []string + for _, location := range result.Locations { + // Convert Docker container hostnames to localhost with mapped ports + url := convertDockerHostnameToLocalhost(location.Url) + serverUrls = append(serverUrls, url) + } + + if len(serverUrls) == 0 { + // Fallback to default for testing + return []string{"127.0.0.1:8080"}, true + } + + return serverUrls, true +} + +// convertDockerHostnameToLocalhost converts Docker container hostnames to localhost with mapped ports +func convertDockerHostnameToLocalhost(dockerUrl string) string { + // Map Docker container hostnames to localhost ports + hostPortMap := map[string]string{ + "volume1:8080": "127.0.0.1:8080", + "volume2:8080": "127.0.0.1:8081", + "volume3:8080": "127.0.0.1:8082", + "volume4:8080": "127.0.0.1:8083", + "volume5:8080": "127.0.0.1:8084", + "volume6:8080": "127.0.0.1:8085", + } + + if localhost, exists := hostPortMap[dockerUrl]; exists { + return localhost + } + + // If not in map, return as-is (might be already localhost) + return dockerUrl +} + +// deleteSpecificFilesFromVolume deletes exactly the specified number of files from the volume +func deleteSpecificFilesFromVolume(t *testing.T, fileIds []string, deleteCount int) []string { + var deletedFileIds []string + successfulDeletions := 0 + + if deleteCount > len(fileIds) { + deleteCount = len(fileIds) + } + + t.Logf("๐Ÿ—‘๏ธ Starting deletion of exactly %d files out of %d total files", deleteCount, len(fileIds)) + + for i := 0; i < deleteCount; i++ { + fileId := fileIds[i] + + // Parse file ID to get volume server location + fid, err := needle.ParseFileIdFromString(fileId) + if err != nil { + t.Logf("Failed to parse file ID %s: %v", fileId, err) + continue + } + + // Find volume server hosting this file + locations, found := findVolumeLocations(fid.VolumeId) + if !found { + t.Logf("Volume locations not found for file %s", fileId) + continue + } + + // Delete file from volume server + deleteUrl := fmt.Sprintf("http://%s/%s", locations[0], fileId) + req, err := http.NewRequest("DELETE", deleteUrl, nil) + if err != nil { + t.Logf("Failed to create delete request for file %s: %v", fileId, err) + continue + } + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + t.Logf("Failed to delete file %s: %v", fileId, err) + continue + } + resp.Body.Close() + + if resp.StatusCode == http.StatusAccepted || resp.StatusCode == http.StatusOK { + successfulDeletions++ + deletedFileIds = append(deletedFileIds, fileId) + // Log progress for first few files and every 25th deletion + if i < 5 || (i+1)%25 == 0 { + t.Logf("๐Ÿ—‘๏ธ Deleted file %d/%d: %s (status: %d)", i+1, deleteCount, fileId, resp.StatusCode) + } + } else { + t.Logf("โŒ Delete failed for file %s with status %d", fileId, resp.StatusCode) + } + } + + t.Logf("โœ… Deletion summary: %d files deleted successfully out of %d attempted", successfulDeletions, deleteCount) + return deletedFileIds +} + +// Helper functions for test setup + +func setupTestCluster(t *testing.T) (*TestCluster, func()) { + // Create test cluster similar to existing integration tests + // This is a simplified version - in practice would start actual servers + cluster := &TestCluster{ + masterAddress: "127.0.0.1:9333", + volumeServers: []string{ + "127.0.0.1:8080", + "127.0.0.1:8081", + "127.0.0.1:8082", + "127.0.0.1:8083", + "127.0.0.1:8084", + "127.0.0.1:8085", + }, + } + + cleanup := func() { + // Cleanup cluster resources + t.Logf("Cleaning up test cluster") + } + + return cluster, cleanup +} + +func waitForClusterReady() error { + // Wait for test cluster to be ready + // In practice, this would ping the servers and wait for them to respond + time.Sleep(2 * time.Second) + return nil +} + +type TestCluster struct { + masterAddress string + volumeServers []string +}