You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
529 lines
25 KiB
529 lines
25 KiB
# Makefile for S3 SSE Integration Tests
|
|
# This Makefile provides targets for running comprehensive S3 Server-Side Encryption tests
|
|
|
|
# Default values
|
|
SEAWEEDFS_BINARY ?= weed
|
|
S3_PORT ?= 8333
|
|
FILER_PORT ?= 8888
|
|
VOLUME_PORT ?= 8080
|
|
MASTER_PORT ?= 9333
|
|
TEST_TIMEOUT ?= 15m
|
|
BUCKET_PREFIX ?= test-sse-
|
|
ACCESS_KEY ?= some_access_key1
|
|
SECRET_KEY ?= some_secret_key1
|
|
VOLUME_MAX_SIZE_MB ?= 50
|
|
VOLUME_MAX_COUNT ?= 100
|
|
|
|
# SSE-KMS configuration
|
|
KMS_KEY_ID ?= test-key-123
|
|
KMS_TYPE ?= local
|
|
OPENBAO_ADDR ?= http://127.0.0.1:8200
|
|
OPENBAO_TOKEN ?= root-token-for-testing
|
|
DOCKER_COMPOSE ?= docker-compose
|
|
|
|
# Test directory
|
|
TEST_DIR := $(shell pwd)
|
|
SEAWEEDFS_ROOT := $(shell cd ../../../ && pwd)
|
|
|
|
# Colors for output
|
|
RED := \033[0;31m
|
|
GREEN := \033[0;32m
|
|
YELLOW := \033[1;33m
|
|
NC := \033[0m # No Color
|
|
|
|
.PHONY: all test clean start-seaweedfs stop-seaweedfs stop-seaweedfs-safe start-seaweedfs-ci check-binary build-weed help help-extended test-with-server test-quick-with-server test-metadata-persistence setup-openbao test-with-kms test-ssekms-integration clean-kms start-full-stack stop-full-stack
|
|
|
|
all: test-basic
|
|
|
|
# Build SeaweedFS binary (GitHub Actions compatible)
|
|
build-weed:
|
|
@echo "Building SeaweedFS binary..."
|
|
@cd $(SEAWEEDFS_ROOT)/weed && go install -buildvcs=false
|
|
@echo "✅ SeaweedFS binary built successfully"
|
|
|
|
help:
|
|
@echo "SeaweedFS S3 SSE Integration Tests"
|
|
@echo ""
|
|
@echo "Available targets:"
|
|
@echo " test-basic - Run basic S3 put/get tests first"
|
|
@echo " test - Run all S3 SSE integration tests"
|
|
@echo " test-ssec - Run SSE-C tests only"
|
|
@echo " test-ssekms - Run SSE-KMS tests only"
|
|
@echo " test-copy - Run SSE copy operation tests"
|
|
@echo " test-multipart - Run SSE multipart upload tests"
|
|
@echo " test-errors - Run SSE error condition tests"
|
|
@echo " benchmark - Run SSE performance benchmarks"
|
|
@echo " KMS Integration:"
|
|
@echo " setup-openbao - Set up OpenBao KMS for testing"
|
|
@echo " test-with-kms - Run full SSE integration with real KMS"
|
|
@echo " test-ssekms-integration - Run SSE-KMS with OpenBao only"
|
|
@echo " start-full-stack - Start SeaweedFS + OpenBao with Docker"
|
|
@echo " stop-full-stack - Stop Docker services"
|
|
@echo " clean-kms - Clean up KMS test environment"
|
|
@echo " start-seaweedfs - Start SeaweedFS server for testing"
|
|
@echo " stop-seaweedfs - Stop SeaweedFS server"
|
|
@echo " clean - Clean up test artifacts"
|
|
@echo " check-binary - Check if SeaweedFS binary exists"
|
|
@echo ""
|
|
@echo "Configuration:"
|
|
@echo " SEAWEEDFS_BINARY=$(SEAWEEDFS_BINARY)"
|
|
@echo " S3_PORT=$(S3_PORT)"
|
|
@echo " FILER_PORT=$(FILER_PORT)"
|
|
@echo " VOLUME_PORT=$(VOLUME_PORT)"
|
|
@echo " MASTER_PORT=$(MASTER_PORT)"
|
|
@echo " TEST_TIMEOUT=$(TEST_TIMEOUT)"
|
|
@echo " VOLUME_MAX_SIZE_MB=$(VOLUME_MAX_SIZE_MB)"
|
|
|
|
check-binary:
|
|
@if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1; then \
|
|
echo "$(RED)Error: SeaweedFS binary '$(SEAWEEDFS_BINARY)' not found in PATH$(NC)"; \
|
|
echo "Please build SeaweedFS first by running 'make' in the root directory"; \
|
|
exit 1; \
|
|
fi
|
|
@echo "$(GREEN)SeaweedFS binary found: $$(which $(SEAWEEDFS_BINARY))$(NC)"
|
|
|
|
start-seaweedfs: check-binary
|
|
@echo "$(YELLOW)Starting SeaweedFS server for SSE testing...$(NC)"
|
|
@# Use port-based cleanup for consistency and safety
|
|
@echo "Cleaning up any existing processes..."
|
|
@lsof -ti :$(MASTER_PORT) | xargs -r kill -TERM || true
|
|
@lsof -ti :$(VOLUME_PORT) | xargs -r kill -TERM || true
|
|
@lsof -ti :$(FILER_PORT) | xargs -r kill -TERM || true
|
|
@lsof -ti :$(S3_PORT) | xargs -r kill -TERM || true
|
|
@sleep 2
|
|
|
|
# Create necessary directories
|
|
@mkdir -p /tmp/seaweedfs-test-sse-master
|
|
@mkdir -p /tmp/seaweedfs-test-sse-volume
|
|
@mkdir -p /tmp/seaweedfs-test-sse-filer
|
|
|
|
# Start master server with volume size limit and explicit gRPC port
|
|
@nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -port.grpc=$$(( $(MASTER_PORT) + 10000 )) -mdir=/tmp/seaweedfs-test-sse-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 > /tmp/seaweedfs-sse-master.log 2>&1 &
|
|
@sleep 3
|
|
|
|
# Start volume server with master HTTP port and increased capacity
|
|
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -mserver=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-sse-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 > /tmp/seaweedfs-sse-volume.log 2>&1 &
|
|
@sleep 5
|
|
|
|
# Start filer server (using standard SeaweedFS gRPC port convention: HTTP port + 10000)
|
|
@nohup $(SEAWEEDFS_BINARY) filer -port=$(FILER_PORT) -port.grpc=$$(( $(FILER_PORT) + 10000 )) -master=127.0.0.1:$(MASTER_PORT) -dataCenter=defaultDataCenter -ip=127.0.0.1 > /tmp/seaweedfs-sse-filer.log 2>&1 &
|
|
@sleep 3
|
|
|
|
# Create S3 configuration with SSE-KMS support
|
|
@printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}],"kms":{"type":"%s","configs":{"keyId":"%s","encryptionContext":{},"bucketKey":false}}}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" "$(KMS_TYPE)" "$(KMS_KEY_ID)" > /tmp/seaweedfs-sse-s3.json
|
|
|
|
# Start S3 server with KMS configuration
|
|
@nohup $(SEAWEEDFS_BINARY) s3 -port=$(S3_PORT) -filer=127.0.0.1:$(FILER_PORT) -config=/tmp/seaweedfs-sse-s3.json -ip.bind=127.0.0.1 > /tmp/seaweedfs-sse-s3.log 2>&1 &
|
|
@sleep 5
|
|
|
|
# Wait for S3 service to be ready
|
|
@echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)"
|
|
@for i in $$(seq 1 30); do \
|
|
if curl -s -f http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \
|
|
echo "$(GREEN)S3 service is ready$(NC)"; \
|
|
break; \
|
|
fi; \
|
|
echo "Waiting for S3 service... ($$i/30)"; \
|
|
sleep 1; \
|
|
done
|
|
|
|
# Additional wait for filer gRPC to be ready
|
|
@echo "$(YELLOW)Waiting for filer gRPC to be ready...$(NC)"
|
|
@sleep 2
|
|
@echo "$(GREEN)SeaweedFS server started successfully for SSE testing$(NC)"
|
|
@echo "Master: http://localhost:$(MASTER_PORT)"
|
|
@echo "Volume: http://localhost:$(VOLUME_PORT)"
|
|
@echo "Filer: http://localhost:$(FILER_PORT)"
|
|
@echo "S3: http://localhost:$(S3_PORT)"
|
|
@echo "Volume Max Size: $(VOLUME_MAX_SIZE_MB)MB"
|
|
@echo "SSE-KMS Support: Enabled"
|
|
|
|
stop-seaweedfs:
|
|
@echo "$(YELLOW)Stopping SeaweedFS server...$(NC)"
|
|
@# Use port-based cleanup for consistency and safety
|
|
@lsof -ti :$(MASTER_PORT) | xargs -r kill -TERM || true
|
|
@lsof -ti :$(VOLUME_PORT) | xargs -r kill -TERM || true
|
|
@lsof -ti :$(FILER_PORT) | xargs -r kill -TERM || true
|
|
@lsof -ti :$(S3_PORT) | xargs -r kill -TERM || true
|
|
@sleep 2
|
|
@echo "$(GREEN)SeaweedFS server stopped$(NC)"
|
|
|
|
# CI-safe server stop that's more conservative
|
|
stop-seaweedfs-safe:
|
|
@echo "$(YELLOW)Safely stopping SeaweedFS server...$(NC)"
|
|
@# Use port-based cleanup which is safer in CI
|
|
@if command -v lsof >/dev/null 2>&1; then \
|
|
echo "Using lsof for port-based cleanup..."; \
|
|
lsof -ti :$(MASTER_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
|
|
lsof -ti :$(VOLUME_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
|
|
lsof -ti :$(FILER_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
|
|
lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
|
|
else \
|
|
echo "lsof not available, using netstat approach..."; \
|
|
netstat -tlnp 2>/dev/null | grep :$(MASTER_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \
|
|
netstat -tlnp 2>/dev/null | grep :$(VOLUME_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \
|
|
netstat -tlnp 2>/dev/null | grep :$(FILER_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \
|
|
netstat -tlnp 2>/dev/null | grep :$(S3_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \
|
|
fi
|
|
@sleep 2
|
|
@echo "$(GREEN)SeaweedFS server safely stopped$(NC)"
|
|
|
|
clean:
|
|
@echo "$(YELLOW)Cleaning up SSE test artifacts...$(NC)"
|
|
@rm -rf /tmp/seaweedfs-test-sse-*
|
|
@rm -f /tmp/seaweedfs-sse-*.log
|
|
@rm -f /tmp/seaweedfs-sse-s3.json
|
|
@echo "$(GREEN)SSE test cleanup completed$(NC)"
|
|
|
|
test-basic: check-binary
|
|
@echo "$(YELLOW)Running basic S3 SSE integration tests...$(NC)"
|
|
@$(MAKE) start-seaweedfs-ci
|
|
@sleep 5
|
|
@echo "$(GREEN)Starting basic SSE tests...$(NC)"
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic" ./test/s3/sse || (echo "$(RED)Basic SSE tests failed$(NC)" && $(MAKE) stop-seaweedfs-safe && exit 1)
|
|
@$(MAKE) stop-seaweedfs-safe
|
|
@echo "$(GREEN)Basic SSE tests completed successfully!$(NC)"
|
|
|
|
test: test-basic
|
|
@echo "$(YELLOW)Running all S3 SSE integration tests...$(NC)"
|
|
@$(MAKE) start-seaweedfs-ci
|
|
@sleep 5
|
|
@echo "$(GREEN)Starting comprehensive SSE tests...$(NC)"
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestSSE.*Integration" ./test/s3/sse || (echo "$(RED)SSE tests failed$(NC)" && $(MAKE) stop-seaweedfs-safe && exit 1)
|
|
@$(MAKE) stop-seaweedfs-safe
|
|
@echo "$(GREEN)All SSE integration tests completed successfully!$(NC)"
|
|
|
|
test-ssec: check-binary
|
|
@echo "$(YELLOW)Running SSE-C integration tests...$(NC)"
|
|
@$(MAKE) start-seaweedfs-ci
|
|
@sleep 5
|
|
@echo "$(GREEN)Starting SSE-C tests...$(NC)"
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestSSEC.*Integration" ./test/s3/sse || (echo "$(RED)SSE-C tests failed$(NC)" && $(MAKE) stop-seaweedfs-safe && exit 1)
|
|
@$(MAKE) stop-seaweedfs-safe
|
|
@echo "$(GREEN)SSE-C tests completed successfully!$(NC)"
|
|
|
|
test-ssekms: check-binary
|
|
@echo "$(YELLOW)Running SSE-KMS integration tests...$(NC)"
|
|
@$(MAKE) start-seaweedfs-ci
|
|
@sleep 5
|
|
@echo "$(GREEN)Starting SSE-KMS tests...$(NC)"
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestSSEKMS.*Integration" ./test/s3/sse || (echo "$(RED)SSE-KMS tests failed$(NC)" && $(MAKE) stop-seaweedfs-safe && exit 1)
|
|
@$(MAKE) stop-seaweedfs-safe
|
|
@echo "$(GREEN)SSE-KMS tests completed successfully!$(NC)"
|
|
|
|
test-copy: check-binary
|
|
@echo "$(YELLOW)Running SSE copy operation tests...$(NC)"
|
|
@$(MAKE) start-seaweedfs-ci
|
|
@sleep 5
|
|
@echo "$(GREEN)Starting SSE copy tests...$(NC)"
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run ".*CopyIntegration" ./test/s3/sse || (echo "$(RED)SSE copy tests failed$(NC)" && $(MAKE) stop-seaweedfs-safe && exit 1)
|
|
@$(MAKE) stop-seaweedfs-safe
|
|
@echo "$(GREEN)SSE copy tests completed successfully!$(NC)"
|
|
|
|
test-multipart: check-binary
|
|
@echo "$(YELLOW)Running SSE multipart upload tests...$(NC)"
|
|
@$(MAKE) start-seaweedfs-ci
|
|
@sleep 5
|
|
@echo "$(GREEN)Starting SSE multipart tests...$(NC)"
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestSSEMultipartUploadIntegration" ./test/s3/sse || (echo "$(RED)SSE multipart tests failed$(NC)" && $(MAKE) stop-seaweedfs-safe && exit 1)
|
|
@$(MAKE) stop-seaweedfs-safe
|
|
@echo "$(GREEN)SSE multipart tests completed successfully!$(NC)"
|
|
|
|
test-errors: check-binary
|
|
@echo "$(YELLOW)Running SSE error condition tests...$(NC)"
|
|
@$(MAKE) start-seaweedfs-ci
|
|
@sleep 5
|
|
@echo "$(GREEN)Starting SSE error tests...$(NC)"
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestSSEErrorConditions" ./test/s3/sse || (echo "$(RED)SSE error tests failed$(NC)" && $(MAKE) stop-seaweedfs-safe && exit 1)
|
|
@$(MAKE) stop-seaweedfs-safe
|
|
@echo "$(GREEN)SSE error tests completed successfully!$(NC)"
|
|
|
|
test-quick: check-binary
|
|
@echo "$(YELLOW)Running quick SSE tests...$(NC)"
|
|
@$(MAKE) start-seaweedfs-ci
|
|
@sleep 5
|
|
@echo "$(GREEN)Starting quick SSE tests...$(NC)"
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=5m -run "TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic" ./test/s3/sse || (echo "$(RED)Quick SSE tests failed$(NC)" && $(MAKE) stop-seaweedfs-safe && exit 1)
|
|
@$(MAKE) stop-seaweedfs-safe
|
|
@echo "$(GREEN)Quick SSE tests completed successfully!$(NC)"
|
|
|
|
benchmark: check-binary
|
|
@echo "$(YELLOW)Running SSE performance benchmarks...$(NC)"
|
|
@$(MAKE) start-seaweedfs-ci
|
|
@sleep 5
|
|
@echo "$(GREEN)Starting SSE benchmarks...$(NC)"
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=30m -bench=. -run=Benchmark ./test/s3/sse || (echo "$(RED)SSE benchmarks failed$(NC)" && $(MAKE) stop-seaweedfs-safe && exit 1)
|
|
@$(MAKE) stop-seaweedfs-safe
|
|
@echo "$(GREEN)SSE benchmarks completed!$(NC)"
|
|
|
|
# Debug targets
|
|
debug-logs:
|
|
@echo "$(YELLOW)=== Master Log ===$(NC)"
|
|
@tail -n 50 /tmp/seaweedfs-sse-master.log || echo "No master log found"
|
|
@echo "$(YELLOW)=== Volume Log ===$(NC)"
|
|
@tail -n 50 /tmp/seaweedfs-sse-volume.log || echo "No volume log found"
|
|
@echo "$(YELLOW)=== Filer Log ===$(NC)"
|
|
@tail -n 50 /tmp/seaweedfs-sse-filer.log || echo "No filer log found"
|
|
@echo "$(YELLOW)=== S3 Log ===$(NC)"
|
|
@tail -n 50 /tmp/seaweedfs-sse-s3.log || echo "No S3 log found"
|
|
|
|
debug-status:
|
|
@echo "$(YELLOW)=== Process Status ===$(NC)"
|
|
@ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found"
|
|
@echo "$(YELLOW)=== Port Status ===$(NC)"
|
|
@netstat -an | grep -E "($(MASTER_PORT)|$(VOLUME_PORT)|$(FILER_PORT)|$(S3_PORT))" || echo "No ports in use"
|
|
|
|
# Manual test targets for development
|
|
manual-start: start-seaweedfs
|
|
@echo "$(GREEN)SeaweedFS with SSE support is now running for manual testing$(NC)"
|
|
@echo "You can now run SSE tests manually or use S3 clients to test SSE functionality"
|
|
@echo "Run 'make manual-stop' when finished"
|
|
|
|
manual-stop: stop-seaweedfs clean
|
|
|
|
# CI/CD targets
|
|
ci-test: test-quick
|
|
|
|
# Stress test
|
|
stress: check-binary
|
|
@echo "$(YELLOW)Running SSE stress tests...$(NC)"
|
|
@$(MAKE) start-seaweedfs-ci
|
|
@sleep 5
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=60m -run="TestSSE.*Integration" -count=5 ./test/s3/sse || (echo "$(RED)SSE stress tests failed$(NC)" && $(MAKE) stop-seaweedfs-safe && exit 1)
|
|
@$(MAKE) stop-seaweedfs-safe
|
|
@echo "$(GREEN)SSE stress tests completed!$(NC)"
|
|
|
|
# Performance test with various data sizes
|
|
perf: check-binary
|
|
@echo "$(YELLOW)Running SSE performance tests with various data sizes...$(NC)"
|
|
@$(MAKE) start-seaweedfs-ci
|
|
@sleep 5
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=60m -run=".*VariousDataSizes" ./test/s3/sse || (echo "$(RED)SSE performance tests failed$(NC)" && $(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe && exit 1)
|
|
@$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe
|
|
@echo "$(GREEN)SSE performance tests completed!$(NC)"
|
|
|
|
# Test specific scenarios that would catch the metadata bug
|
|
test-metadata-persistence: check-binary
|
|
@echo "$(YELLOW)Running SSE metadata persistence tests (would catch filer metadata bugs)...$(NC)"
|
|
@$(MAKE) start-seaweedfs-ci
|
|
@sleep 5
|
|
@echo "$(GREEN)Testing that SSE metadata survives full PUT/GET cycle...$(NC)"
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestSSECIntegrationBasic" ./test/s3/sse || (echo "$(RED)SSE metadata persistence tests failed$(NC)" && $(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe && exit 1)
|
|
@$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe
|
|
@echo "$(GREEN)SSE metadata persistence tests completed successfully!$(NC)"
|
|
@echo "$(GREEN)✅ These tests would have caught the filer metadata storage bug!$(NC)"
|
|
|
|
# GitHub Actions compatible test-with-server target that handles server lifecycle
|
|
test-with-server: build-weed
|
|
@echo "🚀 Starting SSE integration tests with automated server management..."
|
|
@echo "Starting SeaweedFS cluster..."
|
|
@# Use the CI-safe startup directly without aggressive cleanup
|
|
@if $(MAKE) start-seaweedfs-ci > weed-test.log 2>&1; then \
|
|
echo "✅ SeaweedFS cluster started successfully"; \
|
|
echo "Running SSE integration tests..."; \
|
|
trap '$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe || true' EXIT; \
|
|
if [ -n "$(TEST_PATTERN)" ]; then \
|
|
echo "🔍 Running tests matching pattern: $(TEST_PATTERN)"; \
|
|
cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" ./test/s3/sse || exit 1; \
|
|
else \
|
|
echo "🔍 Running all SSE integration tests"; \
|
|
cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestSSE.*Integration" ./test/s3/sse || exit 1; \
|
|
fi; \
|
|
echo "✅ All tests completed successfully"; \
|
|
$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe || true; \
|
|
else \
|
|
echo "❌ Failed to start SeaweedFS cluster"; \
|
|
echo "=== Server startup logs ==="; \
|
|
tail -100 weed-test.log 2>/dev/null || echo "No startup log available"; \
|
|
echo "=== System information ==="; \
|
|
ps aux | grep -E "weed|make" | grep -v grep || echo "No relevant processes found"; \
|
|
exit 1; \
|
|
fi
|
|
|
|
# CI-safe server startup that avoids process conflicts
|
|
start-seaweedfs-ci: check-binary
|
|
@echo "$(YELLOW)Starting SeaweedFS server for CI testing...$(NC)"
|
|
|
|
# Create necessary directories
|
|
@mkdir -p /tmp/seaweedfs-test-sse-master
|
|
@mkdir -p /tmp/seaweedfs-test-sse-volume
|
|
@mkdir -p /tmp/seaweedfs-test-sse-filer
|
|
|
|
# Clean up any old server logs
|
|
@rm -f /tmp/seaweedfs-sse-*.log || true
|
|
|
|
# Start master server with volume size limit and explicit gRPC port
|
|
@echo "Starting master server..."
|
|
@nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -port.grpc=$$(( $(MASTER_PORT) + 10000 )) -mdir=/tmp/seaweedfs-test-sse-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 > /tmp/seaweedfs-sse-master.log 2>&1 &
|
|
@sleep 3
|
|
|
|
# Start volume server with master HTTP port and increased capacity
|
|
@echo "Starting volume server..."
|
|
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -mserver=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-sse-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 > /tmp/seaweedfs-sse-volume.log 2>&1 &
|
|
@sleep 5
|
|
|
|
# Create S3 JSON configuration with KMS (Local provider) and basic identity for embedded S3
|
|
@sed -e 's/ACCESS_KEY_PLACEHOLDER/$(ACCESS_KEY)/g' \
|
|
-e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \
|
|
s3-config-template.json > /tmp/seaweedfs-s3.json
|
|
|
|
# Start filer server with embedded S3 using the JSON config (with verbose logging)
|
|
@echo "Starting filer server with embedded S3..."
|
|
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 nohup $(SEAWEEDFS_BINARY) filer -port=$(FILER_PORT) -port.grpc=$$(( $(FILER_PORT) + 10000 )) -master=127.0.0.1:$(MASTER_PORT) -dataCenter=defaultDataCenter -ip=127.0.0.1 -s3 -s3.port=$(S3_PORT) -s3.config=/tmp/seaweedfs-s3.json > /tmp/seaweedfs-sse-filer.log 2>&1 &
|
|
@sleep 5
|
|
|
|
# Wait for S3 service to be ready - use port-based checking for reliability
|
|
@echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)"
|
|
@for i in $$(seq 1 20); do \
|
|
if netstat -an 2>/dev/null | grep -q ":$(S3_PORT).*LISTEN" || \
|
|
ss -an 2>/dev/null | grep -q ":$(S3_PORT).*LISTEN" || \
|
|
lsof -i :$(S3_PORT) >/dev/null 2>&1; then \
|
|
echo "$(GREEN)S3 service is listening on port $(S3_PORT)$(NC)"; \
|
|
sleep 1; \
|
|
break; \
|
|
fi; \
|
|
if [ $$i -eq 20 ]; then \
|
|
echo "$(RED)S3 service failed to start within 20 seconds$(NC)"; \
|
|
echo "=== Detailed Logs ==="; \
|
|
echo "Master log:"; tail -30 /tmp/seaweedfs-sse-master.log || true; \
|
|
echo "Volume log:"; tail -30 /tmp/seaweedfs-sse-volume.log || true; \
|
|
echo "Filer log:"; tail -30 /tmp/seaweedfs-sse-filer.log || true; \
|
|
echo "=== Port Status ==="; \
|
|
netstat -an 2>/dev/null | grep ":$(S3_PORT)" || \
|
|
ss -an 2>/dev/null | grep ":$(S3_PORT)" || \
|
|
echo "No port listening on $(S3_PORT)"; \
|
|
echo "=== Process Status ==="; \
|
|
ps aux | grep -E "weed.*(filer|s3).*$(S3_PORT)" | grep -v grep || echo "No S3 process found"; \
|
|
exit 1; \
|
|
fi; \
|
|
echo "Waiting for S3 service... ($$i/20)"; \
|
|
sleep 1; \
|
|
done
|
|
|
|
# Additional wait for filer gRPC to be ready
|
|
@echo "$(YELLOW)Waiting for filer gRPC to be ready...$(NC)"
|
|
@sleep 2
|
|
@echo "$(GREEN)SeaweedFS server started successfully for SSE testing$(NC)"
|
|
@echo "Master: http://localhost:$(MASTER_PORT)"
|
|
@echo "Volume: http://localhost:$(VOLUME_PORT)"
|
|
@echo "Filer: http://localhost:$(FILER_PORT)"
|
|
@echo "S3: http://localhost:$(S3_PORT)"
|
|
@echo "Volume Max Size: $(VOLUME_MAX_SIZE_MB)MB"
|
|
@echo "SSE-KMS Support: Enabled"
|
|
|
|
# GitHub Actions compatible quick test subset
|
|
test-quick-with-server: build-weed
|
|
@echo "🚀 Starting quick SSE tests with automated server management..."
|
|
@trap 'make stop-seaweedfs-safe || true' EXIT; \
|
|
echo "Starting SeaweedFS cluster..."; \
|
|
if make start-seaweedfs-ci > weed-test.log 2>&1; then \
|
|
echo "✅ SeaweedFS cluster started successfully"; \
|
|
echo "Running quick SSE integration tests..."; \
|
|
cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration" ./test/s3/sse || exit 1; \
|
|
echo "✅ Quick tests completed successfully"; \
|
|
make stop-seaweedfs-safe || true; \
|
|
else \
|
|
echo "❌ Failed to start SeaweedFS cluster"; \
|
|
echo "=== Server startup logs ==="; \
|
|
tail -50 weed-test.log; \
|
|
exit 1; \
|
|
fi
|
|
|
|
# Help target - extended version
|
|
help-extended:
|
|
@echo "Available targets:"
|
|
@echo " test - Run all SSE integration tests (requires running server)"
|
|
@echo " test-with-server - Run all tests with automatic server management (GitHub Actions compatible)"
|
|
@echo " test-quick-with-server - Run quick tests with automatic server management"
|
|
@echo " test-ssec - Run only SSE-C tests"
|
|
@echo " test-ssekms - Run only SSE-KMS tests"
|
|
@echo " test-copy - Run only copy operation tests"
|
|
@echo " test-multipart - Run only multipart upload tests"
|
|
@echo " benchmark - Run performance benchmarks"
|
|
@echo " perf - Run performance tests with various data sizes"
|
|
@echo " test-metadata-persistence - Test metadata persistence (catches filer bugs)"
|
|
@echo " build-weed - Build SeaweedFS binary"
|
|
@echo " check-binary - Check if SeaweedFS binary exists"
|
|
@echo " start-seaweedfs - Start SeaweedFS cluster"
|
|
@echo " start-seaweedfs-ci - Start SeaweedFS cluster (CI-safe version)"
|
|
@echo " stop-seaweedfs - Stop SeaweedFS cluster"
|
|
@echo " stop-seaweedfs-safe - Stop SeaweedFS cluster (CI-safe version)"
|
|
@echo " clean - Clean up test artifacts"
|
|
@echo " debug-logs - Show recent logs from all services"
|
|
@echo ""
|
|
@echo "Environment Variables:"
|
|
@echo " ACCESS_KEY - S3 access key (default: some_access_key1)"
|
|
@echo " SECRET_KEY - S3 secret key (default: some_secret_key1)"
|
|
@echo " KMS_KEY_ID - KMS key ID for SSE-KMS (default: test-key-123)"
|
|
@echo " KMS_TYPE - KMS type (default: local)"
|
|
@echo " VOLUME_MAX_SIZE_MB - Volume maximum size in MB (default: 50)"
|
|
@echo " TEST_TIMEOUT - Test timeout (default: 15m)"
|
|
|
|
####################################################
|
|
# KMS Integration Testing with OpenBao
|
|
####################################################
|
|
|
|
setup-openbao:
|
|
@echo "$(YELLOW)Setting up OpenBao for SSE-KMS testing...$(NC)"
|
|
@$(DOCKER_COMPOSE) up -d openbao
|
|
@sleep 10
|
|
@echo "$(YELLOW)Configuring OpenBao...$(NC)"
|
|
@OPENBAO_ADDR=$(OPENBAO_ADDR) OPENBAO_TOKEN=$(OPENBAO_TOKEN) ./setup_openbao_sse.sh
|
|
@echo "$(GREEN)✅ OpenBao setup complete!$(NC)"
|
|
|
|
start-full-stack: setup-openbao
|
|
@echo "$(YELLOW)Starting full SeaweedFS + KMS stack...$(NC)"
|
|
@$(DOCKER_COMPOSE) up -d
|
|
@echo "$(YELLOW)Waiting for services to be ready...$(NC)"
|
|
@sleep 15
|
|
@echo "$(GREEN)✅ Full stack running!$(NC)"
|
|
@echo "OpenBao: $(OPENBAO_ADDR)"
|
|
@echo "S3 API: http://localhost:$(S3_PORT)"
|
|
|
|
stop-full-stack:
|
|
@echo "$(YELLOW)Stopping full stack...$(NC)"
|
|
@$(DOCKER_COMPOSE) down
|
|
@echo "$(GREEN)✅ Full stack stopped$(NC)"
|
|
|
|
test-with-kms: start-full-stack
|
|
@echo "$(YELLOW)Running SSE integration tests with real KMS...$(NC)"
|
|
@sleep 5 # Extra time for KMS initialization
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/s3/sse -run "SSE.*Integration" || (echo "$(RED)Tests failed$(NC)" && make stop-full-stack && exit 1)
|
|
@echo "$(GREEN)✅ All KMS integration tests passed!$(NC)"
|
|
@make stop-full-stack
|
|
|
|
test-ssekms-integration: start-full-stack
|
|
@echo "$(YELLOW)Running SSE-KMS integration tests with OpenBao...$(NC)"
|
|
@sleep 5 # Extra time for KMS initialization
|
|
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/s3/sse -run "TestSSEKMS.*Integration" || (echo "$(RED)SSE-KMS tests failed$(NC)" && make stop-full-stack && exit 1)
|
|
@echo "$(GREEN)✅ SSE-KMS integration tests passed!$(NC)"
|
|
@make stop-full-stack
|
|
|
|
clean-kms:
|
|
@echo "$(YELLOW)Cleaning up KMS test environment...$(NC)"
|
|
@$(DOCKER_COMPOSE) down -v --remove-orphans || true
|
|
@docker system prune -f || true
|
|
@echo "$(GREEN)✅ KMS environment cleaned up!$(NC)"
|
|
|
|
status-kms:
|
|
@echo "$(YELLOW)KMS Environment Status:$(NC)"
|
|
@$(DOCKER_COMPOSE) ps
|
|
@echo ""
|
|
@echo "$(YELLOW)OpenBao Health:$(NC)"
|
|
@curl -s $(OPENBAO_ADDR)/v1/sys/health | jq '.' || echo "OpenBao not accessible"
|
|
@echo ""
|
|
@echo "$(YELLOW)S3 API Status:$(NC)"
|
|
@curl -s http://localhost:$(S3_PORT) || echo "S3 API not accessible"
|
|
|
|
# Quick test with just basic KMS functionality
|
|
test-kms-quick: setup-openbao
|
|
@echo "$(YELLOW)Running quick KMS functionality test...$(NC)"
|
|
@cd ../../../test/kms && make dev-test
|
|
@echo "$(GREEN)✅ Quick KMS test passed!$(NC)"
|
|
|
|
# Development targets
|
|
dev-kms: setup-openbao
|
|
@echo "$(GREEN)Development environment ready$(NC)"
|
|
@echo "OpenBao: $(OPENBAO_ADDR)"
|
|
@echo "Token: $(OPENBAO_TOKEN)"
|
|
@echo "Use 'make test-ssekms-integration' to run tests"
|