Chris Lu
2 days ago
32 changed files with 1198 additions and 175 deletions
-
6.github/workflows/container_latest.yml
-
6.github/workflows/container_release1.yml
-
6.github/workflows/container_release2.yml
-
4.github/workflows/container_release3.yml
-
4.github/workflows/container_release4.yml
-
4.github/workflows/container_release5.yml
-
4.github/workflows/container_release_unified.yml
-
54.github/workflows/s3-go-tests.yml
-
6.github/workflows/spark-integration-tests.yml
-
3docker/Dockerfile.go_build
-
4docker/entrypoint.sh
-
18go.mod
-
78go.sum
-
20test/kafka/go.mod
-
40test/kafka/go.sum
-
321test/s3/tagging/Makefile
-
53test/s3/tagging/README.md
-
446test/s3/tagging/s3_tagging_test.go
-
27weed/admin/dash/admin_server.go
-
109weed/admin/dash/user_management_test.go
-
1weed/command/admin.go
-
18weed/command/scaffold/filer.toml
-
6weed/filer/redis2/redis_cluster_store.go
-
4weed/filer/redis2/redis_sentinel_store.go
-
26weed/filer/redis2/redis_store.go
-
30weed/filer/redis2/universal_redis_store.go
-
6weed/filer/redis_lua/redis_cluster_store.go
-
4weed/filer/redis_lua/redis_sentinel_store.go
-
6weed/filer/redis_lua/redis_store.go
-
18weed/filer/redis_lua/universal_redis_store.go
-
23weed/s3api/s3api_object_handlers_put.go
-
14weed/shell/command_volume_check_disk.go
@ -0,0 +1,321 @@ |
|||
# S3 Object Tagging Tests
|
|||
# Tests for GitHub issue #7589: S3 object Tags query comes back empty
|
|||
|
|||
.PHONY: help build-weed setup-server start-server stop-server test-tagging test-tagging-quick test-tagging-comprehensive test-all clean logs check-deps test-with-server |
|||
|
|||
# Configuration
|
|||
WEED_BINARY := ../../../weed/weed_binary |
|||
S3_PORT := 8006 |
|||
MASTER_PORT := 9338 |
|||
VOLUME_PORT := 8085 |
|||
FILER_PORT := 8893 |
|||
TEST_TIMEOUT := 10m |
|||
TEST_PATTERN := TestObjectTaggingOnUpload|TestPutObjectTaggingAPI|TestDeleteObjectTagging|TestTag |
|||
|
|||
# Default target
|
|||
help: |
|||
@echo "S3 Object Tagging Tests Makefile" |
|||
@echo "" |
|||
@echo "Available targets:" |
|||
@echo " help - Show this help message" |
|||
@echo " build-weed - Build the SeaweedFS binary" |
|||
@echo " check-deps - Check dependencies and build binary if needed" |
|||
@echo " start-server - Start SeaweedFS server for testing" |
|||
@echo " stop-server - Stop SeaweedFS server" |
|||
@echo " test-tagging - Run all tagging tests" |
|||
@echo " test-tagging-quick - Run core tagging tests only" |
|||
@echo " test-tagging-comprehensive - Run comprehensive tagging tests" |
|||
@echo " test-with-server - Start server, run tests, stop server" |
|||
@echo " logs - Show server logs" |
|||
@echo " clean - Clean up test artifacts and stop server" |
|||
@echo " health-check - Check if server is accessible" |
|||
@echo "" |
|||
@echo "Configuration:" |
|||
@echo " S3_PORT=${S3_PORT}" |
|||
@echo " TEST_TIMEOUT=${TEST_TIMEOUT}" |
|||
|
|||
# Build the SeaweedFS binary
|
|||
build-weed: |
|||
@echo "Building SeaweedFS binary..." |
|||
@cd ../../../weed && go build -o weed_binary . |
|||
@chmod +x $(WEED_BINARY) |
|||
@echo "✅ SeaweedFS binary built at $(WEED_BINARY)" |
|||
|
|||
check-deps: build-weed |
|||
@echo "Checking dependencies..." |
|||
@echo "🔍 DEBUG: Checking Go installation..." |
|||
@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1) |
|||
@echo "🔍 DEBUG: Go version: $$(go version)" |
|||
@echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)..." |
|||
@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1) |
|||
@echo "🔍 DEBUG: Binary size: $$(ls -lh $(WEED_BINARY) | awk '{print $$5}')" |
|||
@echo "🔍 DEBUG: Binary permissions: $$(ls -la $(WEED_BINARY) | awk '{print $$1}')" |
|||
@echo "🔍 DEBUG: Checking Go module dependencies..." |
|||
@go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1) |
|||
@go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1) |
|||
@echo "✅ All dependencies are available" |
|||
|
|||
# Start SeaweedFS server for testing
|
|||
start-server: check-deps |
|||
@echo "Starting SeaweedFS server..." |
|||
@echo "🔍 DEBUG: Current working directory: $$(pwd)" |
|||
@echo "🔍 DEBUG: Checking for existing weed processes..." |
|||
@ps aux | grep weed | grep -v grep || echo "No existing weed processes found" |
|||
@echo "🔍 DEBUG: Cleaning up any existing PID file..." |
|||
@rm -f weed-server.pid |
|||
@echo "🔍 DEBUG: Checking for port conflicts..." |
|||
@if netstat -tlnp 2>/dev/null | grep $(S3_PORT) >/dev/null; then \
|
|||
echo "⚠️ Port $(S3_PORT) is already in use, trying to find the process..."; \
|
|||
netstat -tlnp 2>/dev/null | grep $(S3_PORT) || true; \
|
|||
else \
|
|||
echo "✅ Port $(S3_PORT) is available"; \
|
|||
fi |
|||
@echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)" |
|||
@ls -la $(WEED_BINARY) || (echo "❌ Binary not found!" && exit 1) |
|||
@echo "🔍 DEBUG: Checking config file at ../../../docker/compose/s3.json" |
|||
@ls -la ../../../docker/compose/s3.json || echo "⚠️ Config file not found, continuing without it" |
|||
@echo "🔍 DEBUG: Creating volume directory..." |
|||
@mkdir -p ./test-volume-data |
|||
@echo "🔍 DEBUG: Launching SeaweedFS server in background..." |
|||
@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 -dir=./test-volume-data -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 -volume.max=100 -volume.preStopSeconds=1 -master.port=$(MASTER_PORT) -volume.port=$(VOLUME_PORT) -filer.port=$(FILER_PORT) -s3.port=$(S3_PORT) -metricsPort=9329 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -master.peers=none" |
|||
@$(WEED_BINARY) server \
|
|||
-filer \
|
|||
-filer.maxMB=64 \
|
|||
-s3 \
|
|||
-ip.bind 0.0.0.0 \
|
|||
-dir=./test-volume-data \
|
|||
-master.raftHashicorp \
|
|||
-master.electionTimeout 1s \
|
|||
-master.volumeSizeLimitMB=100 \
|
|||
-volume.max=100 \
|
|||
-volume.preStopSeconds=1 \
|
|||
-master.port=$(MASTER_PORT) \
|
|||
-volume.port=$(VOLUME_PORT) \
|
|||
-filer.port=$(FILER_PORT) \
|
|||
-s3.port=$(S3_PORT) \
|
|||
-metricsPort=9329 \
|
|||
-s3.allowEmptyFolder=false \
|
|||
-s3.allowDeleteBucketNotEmpty=true \
|
|||
-s3.config=../../../docker/compose/s3.json \
|
|||
-master.peers=none \
|
|||
> weed-test.log 2>&1 & echo $$! > weed-server.pid |
|||
@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')" |
|||
@echo "🔍 DEBUG: Checking if PID is still running..." |
|||
@sleep 2 |
|||
@if [ -f weed-server.pid ]; then \
|
|||
SERVER_PID=$$(cat weed-server.pid); \
|
|||
ps -p $$SERVER_PID || echo "⚠️ Server PID $$SERVER_PID not found after 2 seconds"; \
|
|||
else \
|
|||
echo "⚠️ PID file not found"; \
|
|||
fi |
|||
@echo "🔍 DEBUG: Waiting for server to start (up to 90 seconds)..." |
|||
@for i in $$(seq 1 90); do \
|
|||
echo "🔍 DEBUG: Attempt $$i/90 - checking port $(S3_PORT)"; \
|
|||
if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
|
|||
echo "✅ SeaweedFS server started successfully on port $(S3_PORT) after $$i seconds"; \
|
|||
exit 0; \
|
|||
fi; \
|
|||
if [ $$i -eq 5 ]; then \
|
|||
echo "🔍 DEBUG: After 5 seconds, checking process and logs..."; \
|
|||
ps aux | grep weed | grep -v grep || echo "No weed processes found"; \
|
|||
if [ -f weed-test.log ]; then \
|
|||
echo "=== First server logs ==="; \
|
|||
head -20 weed-test.log; \
|
|||
fi; \
|
|||
fi; \
|
|||
if [ $$i -eq 15 ]; then \
|
|||
echo "🔍 DEBUG: After 15 seconds, checking port bindings..."; \
|
|||
netstat -tlnp 2>/dev/null | grep $(S3_PORT) || echo "Port $(S3_PORT) not bound"; \
|
|||
netstat -tlnp 2>/dev/null | grep $(MASTER_PORT) || echo "Port $(MASTER_PORT) not bound"; \
|
|||
netstat -tlnp 2>/dev/null | grep $(VOLUME_PORT) || echo "Port $(VOLUME_PORT) not bound"; \
|
|||
fi; \
|
|||
if [ $$i -eq 30 ]; then \
|
|||
echo "⚠️ Server taking longer than expected (30s), checking logs..."; \
|
|||
if [ -f weed-test.log ]; then \
|
|||
echo "=== Recent server logs ==="; \
|
|||
tail -20 weed-test.log; \
|
|||
fi; \
|
|||
fi; \
|
|||
sleep 1; \
|
|||
done; \
|
|||
echo "❌ Server failed to start within 90 seconds"; \
|
|||
echo "🔍 DEBUG: Final process check:"; \
|
|||
ps aux | grep weed | grep -v grep || echo "No weed processes found"; \
|
|||
echo "🔍 DEBUG: Final port check:"; \
|
|||
netstat -tlnp 2>/dev/null | grep -E "($(S3_PORT)|$(MASTER_PORT)|$(VOLUME_PORT))" || echo "No ports bound"; \
|
|||
echo "=== Full server logs ==="; \
|
|||
if [ -f weed-test.log ]; then \
|
|||
cat weed-test.log; \
|
|||
else \
|
|||
echo "No log file found"; \
|
|||
fi; \
|
|||
exit 1 |
|||
|
|||
# Stop SeaweedFS server
|
|||
stop-server: |
|||
@echo "Stopping SeaweedFS server..." |
|||
@if [ -f weed-server.pid ]; then \
|
|||
SERVER_PID=$$(cat weed-server.pid); \
|
|||
echo "Killing server PID $$SERVER_PID"; \
|
|||
if ps -p $$SERVER_PID >/dev/null 2>&1; then \
|
|||
kill -TERM $$SERVER_PID 2>/dev/null || true; \
|
|||
sleep 2; \
|
|||
if ps -p $$SERVER_PID >/dev/null 2>&1; then \
|
|||
echo "Process still running, sending KILL signal..."; \
|
|||
kill -KILL $$SERVER_PID 2>/dev/null || true; \
|
|||
sleep 1; \
|
|||
fi; \
|
|||
else \
|
|||
echo "Process $$SERVER_PID not found (already stopped)"; \
|
|||
fi; \
|
|||
rm -f weed-server.pid; \
|
|||
else \
|
|||
echo "No PID file found, checking for running processes..."; \
|
|||
echo "⚠️ Skipping automatic process cleanup to avoid CI issues"; \
|
|||
echo "Note: Any remaining weed processes should be cleaned up by the CI environment"; \
|
|||
fi |
|||
@echo "✅ SeaweedFS server stopped" |
|||
|
|||
# Show server logs
|
|||
logs: |
|||
@if test -f weed-test.log; then \
|
|||
echo "=== SeaweedFS Server Logs ==="; \
|
|||
tail -f weed-test.log; \
|
|||
else \
|
|||
echo "No log file found. Server may not be running."; \
|
|||
fi |
|||
|
|||
# Core tagging tests (basic functionality)
|
|||
test-tagging-quick: check-deps |
|||
@echo "Running core tagging tests..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTaggingOnUpload|TestPutObjectTaggingAPI" . |
|||
@echo "✅ Core tagging tests completed" |
|||
|
|||
# All tagging tests (comprehensive)
|
|||
test-tagging: check-deps |
|||
@echo "Running all tagging tests..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" . |
|||
@echo "✅ All tagging tests completed" |
|||
|
|||
# Comprehensive tagging tests (all features)
|
|||
test-tagging-comprehensive: check-deps |
|||
@echo "Running comprehensive tagging tests..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) . |
|||
@echo "✅ Comprehensive tagging tests completed" |
|||
|
|||
# All tests without server management
|
|||
test-tagging-simple: check-deps |
|||
@echo "Running tagging tests (assuming server is already running)..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) . |
|||
@echo "✅ All tagging tests completed" |
|||
|
|||
# Start server, run tests, stop server
|
|||
test-with-server: start-server |
|||
@echo "Running tagging tests with managed server..." |
|||
@sleep 5 # Give server time to fully start |
|||
@make test-tagging-comprehensive || (echo "Tests failed, stopping server..." && make stop-server && exit 1) |
|||
@make stop-server |
|||
@echo "✅ All tests completed with managed server" |
|||
|
|||
# Health check
|
|||
health-check: |
|||
@echo "Checking server health..." |
|||
@if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
|
|||
echo "✅ Server is accessible on port $(S3_PORT)"; \
|
|||
else \
|
|||
echo "❌ Server is not accessible on port $(S3_PORT)"; \
|
|||
exit 1; \
|
|||
fi |
|||
|
|||
# Clean up
|
|||
clean: |
|||
@echo "Cleaning up test artifacts..." |
|||
@make stop-server |
|||
@rm -f weed-test.log |
|||
@rm -f weed-server.pid |
|||
@rm -rf ./test-volume-data |
|||
@rm -f tagging.test |
|||
@go clean -testcache |
|||
@echo "✅ Cleanup completed" |
|||
|
|||
# Individual test targets for specific functionality
|
|||
test-upload: |
|||
@echo "Running upload tagging tests..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTaggingOnUpload" . |
|||
|
|||
test-special-chars: |
|||
@echo "Running special characters tagging tests..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTaggingOnUploadWithSpecialCharacters" . |
|||
|
|||
test-api: |
|||
@echo "Running API tagging tests..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestPutObjectTaggingAPI" . |
|||
|
|||
test-get: |
|||
@echo "Running get tagging tests..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestGetObjectTaggingAPI" . |
|||
|
|||
test-delete: |
|||
@echo "Running delete tagging tests..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestDeleteObjectTaggingAPI" . |
|||
|
|||
# Development targets
|
|||
dev-start: start-server |
|||
@echo "Development server started. Access S3 API at http://localhost:$(S3_PORT)" |
|||
@echo "To stop: make stop-server" |
|||
|
|||
dev-test: check-deps |
|||
@echo "Running tests in development mode..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectTaggingOnUpload" . |
|||
|
|||
# CI targets
|
|||
ci-test: check-deps |
|||
@echo "Running tests in CI mode..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) -race . |
|||
|
|||
# All targets
|
|||
test-all: test-tagging test-tagging-comprehensive |
|||
@echo "✅ All tagging tests completed" |
|||
|
|||
# Benchmark targets
|
|||
benchmark-tagging: |
|||
@echo "Running tagging performance benchmarks..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) -bench=. -benchmem . |
|||
|
|||
# Coverage targets
|
|||
coverage: |
|||
@echo "Running tests with coverage..." |
|||
@go test -v -timeout=$(TEST_TIMEOUT) -coverprofile=coverage.out . |
|||
@go tool cover -html=coverage.out -o coverage.html |
|||
@echo "Coverage report generated: coverage.html" |
|||
|
|||
# Format and lint
|
|||
fmt: |
|||
@echo "Formatting Go code..." |
|||
@go fmt . |
|||
|
|||
lint: |
|||
@echo "Running linter..." |
|||
@golint . || echo "golint not available, skipping..." |
|||
|
|||
# Install dependencies for development
|
|||
install-deps: |
|||
@echo "Installing Go dependencies..." |
|||
@go mod tidy |
|||
@go mod download |
|||
|
|||
# Show current configuration
|
|||
show-config: |
|||
@echo "Current configuration:" |
|||
@echo " WEED_BINARY: $(WEED_BINARY)" |
|||
@echo " S3_PORT: $(S3_PORT)" |
|||
@echo " TEST_TIMEOUT: $(TEST_TIMEOUT)" |
|||
@echo " TEST_PATTERN: $(TEST_PATTERN)" |
|||
|
|||
# Legacy targets for backward compatibility
|
|||
test: test-with-server |
|||
test-verbose: test-tagging-comprehensive |
|||
test-single: test-upload |
|||
test-clean: clean |
|||
build: check-deps |
|||
setup: check-deps |
|||
@ -0,0 +1,53 @@ |
|||
# S3 Object Tagging Tests |
|||
|
|||
This directory contains tests for S3 object tagging functionality. |
|||
|
|||
## Issue Reference |
|||
|
|||
These tests were created to verify the fix for [GitHub Issue #7589](https://github.com/seaweedfs/seaweedfs/issues/7589): |
|||
**S3 object Tags query comes back empty** |
|||
|
|||
## Problem Description |
|||
|
|||
When uploading an object with tags using the `X-Amz-Tagging` header, the tags were not being stored. |
|||
When querying the object tagging with `GetObjectTagging`, the response was empty. |
|||
|
|||
This was a regression between SeaweedFS 4.00 and 4.01. |
|||
|
|||
## Root Cause |
|||
|
|||
The `putToFiler` function in `s3api_object_handlers_put.go` was not parsing the `X-Amz-Tagging` header |
|||
and storing the tags in the entry's Extended metadata. The code was only copying user metadata |
|||
(headers starting with `X-Amz-Meta-`) but not object tags. |
|||
|
|||
## Fix |
|||
|
|||
Added tag parsing logic to `putToFiler` that: |
|||
1. Reads the `X-Amz-Tagging` header |
|||
2. Parses it using `url.ParseQuery()` for proper URL decoding |
|||
3. Stores each tag with the prefix `X-Amz-Tagging-` in the entry's Extended metadata |
|||
|
|||
## Running Tests |
|||
|
|||
```bash |
|||
# Run all tagging tests |
|||
cd test/s3/tagging |
|||
make test |
|||
|
|||
# Run specific test |
|||
make test-upload |
|||
|
|||
# Or using go test directly |
|||
go test -v ./... |
|||
``` |
|||
|
|||
## Test Cases |
|||
|
|||
1. **TestObjectTaggingOnUpload** - Basic test for tags sent during object upload |
|||
2. **TestObjectTaggingOnUploadWithSpecialCharacters** - Tests URL-encoded tag values |
|||
3. **TestObjectTaggingOnUploadWithEmptyValue** - Tests tags with empty values |
|||
4. **TestPutObjectTaggingAPI** - Tests the PutObjectTagging API separately |
|||
5. **TestDeleteObjectTagging** - Tests tag deletion |
|||
6. **TestTagsNotPreservedAfterObjectOverwrite** - Verifies AWS S3 behavior on overwrite |
|||
7. **TestMaximumNumberOfTags** - Tests storing the maximum 10 tags |
|||
8. **TestTagCountHeader** - Tests the x-amz-tagging-count header in HeadObject |
|||
@ -0,0 +1,446 @@ |
|||
package tagging |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"os" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/aws/aws-sdk-go-v2/aws" |
|||
"github.com/aws/aws-sdk-go-v2/config" |
|||
"github.com/aws/aws-sdk-go-v2/credentials" |
|||
"github.com/aws/aws-sdk-go-v2/service/s3" |
|||
"github.com/aws/aws-sdk-go-v2/service/s3/types" |
|||
"github.com/stretchr/testify/assert" |
|||
"github.com/stretchr/testify/require" |
|||
) |
|||
|
|||
// S3TestConfig holds configuration for S3 tests
|
|||
type S3TestConfig struct { |
|||
Endpoint string |
|||
AccessKey string |
|||
SecretKey string |
|||
Region string |
|||
BucketPrefix string |
|||
UseSSL bool |
|||
SkipVerifySSL bool |
|||
} |
|||
|
|||
// getDefaultConfig returns a fresh instance of the default test configuration
|
|||
func getDefaultConfig() *S3TestConfig { |
|||
endpoint := os.Getenv("S3_ENDPOINT") |
|||
if endpoint == "" { |
|||
endpoint = "http://localhost:8333" // Default SeaweedFS S3 port
|
|||
} |
|||
accessKey := os.Getenv("S3_ACCESS_KEY") |
|||
if accessKey == "" { |
|||
accessKey = "some_access_key1" |
|||
} |
|||
secretKey := os.Getenv("S3_SECRET_KEY") |
|||
if secretKey == "" { |
|||
secretKey = "some_secret_key1" |
|||
} |
|||
return &S3TestConfig{ |
|||
Endpoint: endpoint, |
|||
AccessKey: accessKey, |
|||
SecretKey: secretKey, |
|||
Region: "us-east-1", |
|||
BucketPrefix: "test-tagging-", |
|||
UseSSL: false, |
|||
SkipVerifySSL: true, |
|||
} |
|||
} |
|||
|
|||
// getS3Client creates an AWS S3 client for testing
|
|||
func getS3Client(t *testing.T) *s3.Client { |
|||
defaultConfig := getDefaultConfig() |
|||
cfg, err := config.LoadDefaultConfig(context.TODO(), |
|||
config.WithRegion(defaultConfig.Region), |
|||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( |
|||
defaultConfig.AccessKey, |
|||
defaultConfig.SecretKey, |
|||
"", |
|||
)), |
|||
config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc( |
|||
func(service, region string, options ...interface{}) (aws.Endpoint, error) { |
|||
return aws.Endpoint{ |
|||
URL: defaultConfig.Endpoint, |
|||
SigningRegion: defaultConfig.Region, |
|||
}, nil |
|||
})), |
|||
) |
|||
require.NoError(t, err) |
|||
|
|||
client := s3.NewFromConfig(cfg, func(o *s3.Options) { |
|||
o.UsePathStyle = true |
|||
}) |
|||
return client |
|||
} |
|||
|
|||
// createTestBucket creates a test bucket with a unique name
|
|||
func createTestBucket(t *testing.T, client *s3.Client) string { |
|||
defaultConfig := getDefaultConfig() |
|||
bucketName := fmt.Sprintf("%s%d", defaultConfig.BucketPrefix, time.Now().UnixNano()) |
|||
|
|||
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ |
|||
Bucket: aws.String(bucketName), |
|||
}) |
|||
require.NoError(t, err) |
|||
|
|||
// Wait for bucket metadata to be fully processed
|
|||
time.Sleep(50 * time.Millisecond) |
|||
|
|||
return bucketName |
|||
} |
|||
|
|||
// cleanupTestBucket removes the test bucket and all its contents
|
|||
func cleanupTestBucket(t *testing.T, client *s3.Client, bucketName string) { |
|||
// First, delete all objects in the bucket
|
|||
listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ |
|||
Bucket: aws.String(bucketName), |
|||
}) |
|||
if err == nil { |
|||
for _, obj := range listResp.Contents { |
|||
_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: obj.Key, |
|||
}) |
|||
if err != nil { |
|||
t.Logf("Warning: failed to delete object %s: %v", *obj.Key, err) |
|||
} |
|||
} |
|||
} |
|||
|
|||
// Then delete the bucket
|
|||
_, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{ |
|||
Bucket: aws.String(bucketName), |
|||
}) |
|||
if err != nil { |
|||
t.Logf("Warning: failed to delete bucket %s: %v", bucketName, err) |
|||
} |
|||
} |
|||
|
|||
// TestObjectTaggingOnUpload tests that tags sent during object upload (via X-Amz-Tagging header)
|
|||
// are properly stored and can be retrieved. This is the fix for GitHub issue #7589.
|
|||
func TestObjectTaggingOnUpload(t *testing.T) { |
|||
client := getS3Client(t) |
|||
bucketName := createTestBucket(t, client) |
|||
defer cleanupTestBucket(t, client, bucketName) |
|||
|
|||
objectKey := "test-object-with-tags" |
|||
objectContent := "Hello, World!" |
|||
|
|||
// Put object with tags using the Tagging parameter (X-Amz-Tagging header)
|
|||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
Body: strings.NewReader(objectContent), |
|||
Tagging: aws.String("env=production&team=platform"), |
|||
}) |
|||
require.NoError(t, err, "Should be able to put object with tags") |
|||
|
|||
// Get the tags back
|
|||
tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
}) |
|||
require.NoError(t, err, "Should be able to get object tags") |
|||
|
|||
// Verify tags were stored correctly
|
|||
require.Len(t, tagResp.TagSet, 2, "Should have 2 tags") |
|||
|
|||
// Build a map for easier assertion
|
|||
tagMap := make(map[string]string) |
|||
for _, tag := range tagResp.TagSet { |
|||
tagMap[*tag.Key] = *tag.Value |
|||
} |
|||
|
|||
assert.Equal(t, "production", tagMap["env"], "env tag should be 'production'") |
|||
assert.Equal(t, "platform", tagMap["team"], "team tag should be 'platform'") |
|||
} |
|||
|
|||
// TestObjectTaggingOnUploadWithSpecialCharacters tests tags with URL-encoded characters
|
|||
func TestObjectTaggingOnUploadWithSpecialCharacters(t *testing.T) { |
|||
client := getS3Client(t) |
|||
bucketName := createTestBucket(t, client) |
|||
defer cleanupTestBucket(t, client, bucketName) |
|||
|
|||
objectKey := "test-object-with-special-tags" |
|||
objectContent := "Hello, World!" |
|||
|
|||
// Put object with tags containing special characters
|
|||
// AWS SDK will URL-encode these automatically
|
|||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
Body: strings.NewReader(objectContent), |
|||
Tagging: aws.String("timestamp=2025-07-16 14:40:39&path=/tmp/file.txt"), |
|||
}) |
|||
require.NoError(t, err, "Should be able to put object with special character tags") |
|||
|
|||
// Get the tags back
|
|||
tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
}) |
|||
require.NoError(t, err, "Should be able to get object tags") |
|||
|
|||
// Verify tags were stored and URL-decoded correctly
|
|||
require.Len(t, tagResp.TagSet, 2, "Should have 2 tags") |
|||
|
|||
tagMap := make(map[string]string) |
|||
for _, tag := range tagResp.TagSet { |
|||
tagMap[*tag.Key] = *tag.Value |
|||
} |
|||
|
|||
assert.Equal(t, "2025-07-16 14:40:39", tagMap["timestamp"], "timestamp tag should be decoded correctly") |
|||
assert.Equal(t, "/tmp/file.txt", tagMap["path"], "path tag should be decoded correctly") |
|||
} |
|||
|
|||
// TestObjectTaggingOnUploadWithEmptyValue tests tags with empty values
|
|||
func TestObjectTaggingOnUploadWithEmptyValue(t *testing.T) { |
|||
client := getS3Client(t) |
|||
bucketName := createTestBucket(t, client) |
|||
defer cleanupTestBucket(t, client, bucketName) |
|||
|
|||
objectKey := "test-object-with-empty-tag" |
|||
objectContent := "Hello, World!" |
|||
|
|||
// Put object with a tag that has an empty value
|
|||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
Body: strings.NewReader(objectContent), |
|||
Tagging: aws.String("marker=&env=dev"), |
|||
}) |
|||
require.NoError(t, err, "Should be able to put object with empty tag value") |
|||
|
|||
// Get the tags back
|
|||
tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
}) |
|||
require.NoError(t, err, "Should be able to get object tags") |
|||
|
|||
// Verify tags were stored correctly
|
|||
require.Len(t, tagResp.TagSet, 2, "Should have 2 tags") |
|||
|
|||
tagMap := make(map[string]string) |
|||
for _, tag := range tagResp.TagSet { |
|||
tagMap[*tag.Key] = *tag.Value |
|||
} |
|||
|
|||
assert.Equal(t, "", tagMap["marker"], "marker tag should have empty value") |
|||
assert.Equal(t, "dev", tagMap["env"], "env tag should be 'dev'") |
|||
} |
|||
|
|||
// TestPutObjectTaggingAPI tests the PutObjectTagging API separately from upload
|
|||
func TestPutObjectTaggingAPI(t *testing.T) { |
|||
client := getS3Client(t) |
|||
bucketName := createTestBucket(t, client) |
|||
defer cleanupTestBucket(t, client, bucketName) |
|||
|
|||
objectKey := "test-object-for-tagging-api" |
|||
objectContent := "Hello, World!" |
|||
|
|||
// First, put object without tags
|
|||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
Body: strings.NewReader(objectContent), |
|||
}) |
|||
require.NoError(t, err, "Should be able to put object without tags") |
|||
|
|||
// Get tags - should be empty
|
|||
tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
}) |
|||
require.NoError(t, err, "Should be able to get object tags") |
|||
assert.Len(t, tagResp.TagSet, 0, "Should have no tags initially") |
|||
|
|||
// Now add tags using PutObjectTagging API
|
|||
_, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
Tagging: &types.Tagging{ |
|||
TagSet: []types.Tag{ |
|||
{Key: aws.String("env"), Value: aws.String("staging")}, |
|||
{Key: aws.String("version"), Value: aws.String("1.0")}, |
|||
}, |
|||
}, |
|||
}) |
|||
require.NoError(t, err, "Should be able to put object tags via API") |
|||
|
|||
// Get tags - should now have the tags
|
|||
tagResp, err = client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
}) |
|||
require.NoError(t, err, "Should be able to get object tags after PutObjectTagging") |
|||
require.Len(t, tagResp.TagSet, 2, "Should have 2 tags") |
|||
|
|||
tagMap := make(map[string]string) |
|||
for _, tag := range tagResp.TagSet { |
|||
tagMap[*tag.Key] = *tag.Value |
|||
} |
|||
|
|||
assert.Equal(t, "staging", tagMap["env"], "env tag should be 'staging'") |
|||
assert.Equal(t, "1.0", tagMap["version"], "version tag should be '1.0'") |
|||
} |
|||
|
|||
// TestDeleteObjectTagging tests the DeleteObjectTagging API
|
|||
func TestDeleteObjectTagging(t *testing.T) { |
|||
client := getS3Client(t) |
|||
bucketName := createTestBucket(t, client) |
|||
defer cleanupTestBucket(t, client, bucketName) |
|||
|
|||
objectKey := "test-object-for-delete-tags" |
|||
objectContent := "Hello, World!" |
|||
|
|||
// Put object with tags
|
|||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
Body: strings.NewReader(objectContent), |
|||
Tagging: aws.String("env=production"), |
|||
}) |
|||
require.NoError(t, err, "Should be able to put object with tags") |
|||
|
|||
// Verify tags exist
|
|||
tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
}) |
|||
require.NoError(t, err, "Should be able to get object tags") |
|||
require.Len(t, tagResp.TagSet, 1, "Should have 1 tag") |
|||
|
|||
// Delete tags
|
|||
_, err = client.DeleteObjectTagging(context.TODO(), &s3.DeleteObjectTaggingInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
}) |
|||
require.NoError(t, err, "Should be able to delete object tags") |
|||
|
|||
// Verify tags are deleted
|
|||
tagResp, err = client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
}) |
|||
require.NoError(t, err, "Should be able to get object tags after deletion") |
|||
assert.Len(t, tagResp.TagSet, 0, "Should have no tags after deletion") |
|||
} |
|||
|
|||
// TestTagsNotPreservedAfterObjectOverwrite tests that tags are NOT preserved when an object is overwritten
|
|||
// This matches AWS S3 behavior where overwriting an object replaces all metadata including tags
|
|||
func TestTagsNotPreservedAfterObjectOverwrite(t *testing.T) { |
|||
client := getS3Client(t) |
|||
bucketName := createTestBucket(t, client) |
|||
defer cleanupTestBucket(t, client, bucketName) |
|||
|
|||
objectKey := "test-object-overwrite-tags" |
|||
objectContent := "Original content" |
|||
|
|||
// Put object with tags
|
|||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
Body: strings.NewReader(objectContent), |
|||
Tagging: aws.String("original=true"), |
|||
}) |
|||
require.NoError(t, err, "Should be able to put object with tags") |
|||
|
|||
// Verify original tags exist
|
|||
tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
}) |
|||
require.NoError(t, err, "Should be able to get object tags") |
|||
require.Len(t, tagResp.TagSet, 1, "Should have 1 tag") |
|||
assert.Equal(t, "original", *tagResp.TagSet[0].Key) |
|||
|
|||
// Overwrite the object WITHOUT tags
|
|||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
Body: strings.NewReader("New content"), |
|||
}) |
|||
require.NoError(t, err, "Should be able to overwrite object") |
|||
|
|||
// Tags should be gone after overwrite (matches AWS S3 behavior)
|
|||
tagResp, err = client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
}) |
|||
require.NoError(t, err, "Should be able to get object tags after overwrite") |
|||
assert.Len(t, tagResp.TagSet, 0, "Tags should be cleared after object overwrite") |
|||
} |
|||
|
|||
// TestMaximumNumberOfTags tests that we can store the maximum 10 tags per object
|
|||
func TestMaximumNumberOfTags(t *testing.T) { |
|||
client := getS3Client(t) |
|||
bucketName := createTestBucket(t, client) |
|||
defer cleanupTestBucket(t, client, bucketName) |
|||
|
|||
objectKey := "test-object-max-tags" |
|||
objectContent := "Hello, World!" |
|||
|
|||
// Build 10 tags (S3 max)
|
|||
tags := []string{} |
|||
for i := 1; i <= 10; i++ { |
|||
tags = append(tags, fmt.Sprintf("key%d=value%d", i, i)) |
|||
} |
|||
tagging := strings.Join(tags, "&") |
|||
|
|||
// Put object with 10 tags
|
|||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
Body: strings.NewReader(objectContent), |
|||
Tagging: aws.String(tagging), |
|||
}) |
|||
require.NoError(t, err, "Should be able to put object with 10 tags") |
|||
|
|||
// Get the tags back
|
|||
tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
}) |
|||
require.NoError(t, err, "Should be able to get object tags") |
|||
assert.Len(t, tagResp.TagSet, 10, "Should have 10 tags") |
|||
} |
|||
|
|||
// TestTagCountHeader tests that the x-amz-tagging-count header is returned in HeadObject
|
|||
func TestTagCountHeader(t *testing.T) { |
|||
client := getS3Client(t) |
|||
bucketName := createTestBucket(t, client) |
|||
defer cleanupTestBucket(t, client, bucketName) |
|||
|
|||
objectKey := "test-object-tag-count" |
|||
objectContent := "Hello, World!" |
|||
|
|||
// Put object with tags
|
|||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
Body: strings.NewReader(objectContent), |
|||
Tagging: aws.String("env=prod&team=backend&version=2.0"), |
|||
}) |
|||
require.NoError(t, err, "Should be able to put object with tags") |
|||
|
|||
// Head object to get tag count
|
|||
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String(objectKey), |
|||
}) |
|||
require.NoError(t, err, "Should be able to head object") |
|||
|
|||
// Check tag count header
|
|||
if headResp.TagCount != nil { |
|||
assert.Equal(t, int32(3), *headResp.TagCount, "Tag count should be 3") |
|||
} else { |
|||
t.Log("Warning: TagCount header not returned - this may be expected depending on implementation") |
|||
} |
|||
} |
|||
@ -0,0 +1,109 @@ |
|||
package dash |
|||
|
|||
import ( |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/credential" |
|||
_ "github.com/seaweedfs/seaweedfs/weed/credential/filer_etc" // Import to register filer_etc store
|
|||
"github.com/seaweedfs/seaweedfs/weed/pb" |
|||
"google.golang.org/grpc" |
|||
) |
|||
|
|||
// TestFilerAddressFunctionInterface tests that the filer_etc store
|
|||
// implements the correct SetFilerAddressFunc interface (issue #7575)
|
|||
func TestFilerAddressFunctionInterface(t *testing.T) { |
|||
// Create credential manager with filer_etc store
|
|||
credentialManager, err := credential.NewCredentialManagerWithDefaults("") |
|||
if err != nil { |
|||
t.Fatalf("Failed to initialize credential manager: %v", err) |
|||
} |
|||
|
|||
store := credentialManager.GetStore() |
|||
if store == nil { |
|||
t.Fatal("Credential store is nil") |
|||
} |
|||
|
|||
// Check if store is filer_etc type
|
|||
if store.GetName() != credential.StoreTypeFilerEtc { |
|||
t.Skipf("Skipping test - store is not filer_etc (got: %s)", store.GetName()) |
|||
} |
|||
|
|||
// Check if store implements SetFilerAddressFunc interface
|
|||
// This is the critical check for bug #7575
|
|||
filerFuncSetter, ok := store.(interface { |
|||
SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption) |
|||
}) |
|||
if !ok { |
|||
t.Fatal("FilerEtcStore does not implement SetFilerAddressFunc interface - bug #7575") |
|||
} |
|||
|
|||
// Verify we can call the method without panic
|
|||
mockFilerAddress := pb.ServerAddress("localhost:8888") |
|||
filerFuncSetter.SetFilerAddressFunc(func() pb.ServerAddress { |
|||
return mockFilerAddress |
|||
}, grpc.WithInsecure()) |
|||
|
|||
t.Log("FilerEtcStore correctly implements SetFilerAddressFunc interface") |
|||
} |
|||
|
|||
// TestGenerateAccessKey tests the access key generation function
|
|||
func TestGenerateAccessKey(t *testing.T) { |
|||
key1 := generateAccessKey() |
|||
key2 := generateAccessKey() |
|||
|
|||
// Check length
|
|||
if len(key1) != 20 { |
|||
t.Errorf("Expected access key length 20, got %d", len(key1)) |
|||
} |
|||
|
|||
// Check uniqueness
|
|||
if key1 == key2 { |
|||
t.Error("Generated access keys should be unique") |
|||
} |
|||
|
|||
// Check character set
|
|||
for _, c := range key1 { |
|||
if !((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { |
|||
t.Errorf("Access key contains invalid character: %c", c) |
|||
} |
|||
} |
|||
} |
|||
|
|||
// TestGenerateSecretKey tests the secret key generation function
|
|||
func TestGenerateSecretKey(t *testing.T) { |
|||
key1 := generateSecretKey() |
|||
key2 := generateSecretKey() |
|||
|
|||
// Check length (base64 encoding of 30 bytes = 40 characters)
|
|||
if len(key1) != 40 { |
|||
t.Errorf("Expected secret key length 40, got %d", len(key1)) |
|||
} |
|||
|
|||
// Check uniqueness
|
|||
if key1 == key2 { |
|||
t.Error("Generated secret keys should be unique") |
|||
} |
|||
} |
|||
|
|||
// TestGenerateAccountId tests the account ID generation function
|
|||
func TestGenerateAccountId(t *testing.T) { |
|||
id1 := generateAccountId() |
|||
id2 := generateAccountId() |
|||
|
|||
// Check length
|
|||
if len(id1) != 12 { |
|||
t.Errorf("Expected account ID length 12, got %d", len(id1)) |
|||
} |
|||
|
|||
// Check that it's a number
|
|||
for _, c := range id1 { |
|||
if c < '0' || c > '9' { |
|||
t.Errorf("Account ID contains non-digit character: %c", c) |
|||
} |
|||
} |
|||
|
|||
// Check uniqueness (they should usually be different)
|
|||
if id1 == id2 { |
|||
t.Log("Warning: Generated account IDs are the same (rare but possible)") |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue