Chris Lu
3 weeks ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 767 additions and 41 deletions
-
85.github/workflows/s3-filer-group-tests.yml
-
166test/s3/filer_group/Makefile
-
99test/s3/filer_group/README.md
-
332test/s3/filer_group/s3_filer_group_test.go
-
8test/s3/filer_group/test_config.json
-
111weed/admin/dash/admin_server.go
-
6weed/s3api/s3_constants/buckets.go
-
1weed/s3api/s3_objectlock/object_lock_check.go
@ -0,0 +1,85 @@ |
|||
name: "S3 Filer Group Tests" |
|||
|
|||
on: |
|||
pull_request: |
|||
|
|||
concurrency: |
|||
group: ${{ github.head_ref }}/s3-filer-group-tests |
|||
cancel-in-progress: true |
|||
|
|||
permissions: |
|||
contents: read |
|||
|
|||
defaults: |
|||
run: |
|||
working-directory: weed |
|||
|
|||
jobs: |
|||
s3-filer-group-tests: |
|||
name: S3 Filer Group Integration Tests |
|||
runs-on: ubuntu-22.04 |
|||
timeout-minutes: 20 |
|||
|
|||
steps: |
|||
- name: Check out code |
|||
uses: actions/checkout@v6 |
|||
|
|||
- name: Set up Go |
|||
uses: actions/setup-go@v6 |
|||
with: |
|||
go-version-file: 'go.mod' |
|||
id: go |
|||
|
|||
- name: Install SeaweedFS |
|||
run: | |
|||
go install -buildvcs=false |
|||
|
|||
- name: Run S3 Filer Group Tests |
|||
timeout-minutes: 15 |
|||
working-directory: test/s3/filer_group |
|||
run: | |
|||
set -x |
|||
echo "=== System Information ===" |
|||
uname -a |
|||
free -h |
|||
df -h |
|||
echo "=== Starting Filer Group Tests ===" |
|||
|
|||
# Run tests with automatic server management |
|||
# The test-with-server target handles server startup/shutdown automatically |
|||
# Server is started with -filer.group=testgroup |
|||
make test-with-server || { |
|||
echo "❌ Filer group tests failed, checking logs..." |
|||
if [ -f weed-test.log ]; then |
|||
echo "=== Server logs ===" |
|||
tail -100 weed-test.log |
|||
fi |
|||
echo "=== Process information ===" |
|||
ps aux | grep -E "(weed|test)" || true |
|||
exit 1 |
|||
} |
|||
|
|||
- name: Show server logs on failure |
|||
if: failure() |
|||
working-directory: test/s3/filer_group |
|||
run: | |
|||
echo "=== Server Logs ===" |
|||
if [ -f weed-test.log ]; then |
|||
echo "Last 100 lines of server logs:" |
|||
tail -100 weed-test.log |
|||
else |
|||
echo "No server log file found" |
|||
fi |
|||
|
|||
echo "=== Test Environment ===" |
|||
ps aux | grep -E "(weed|test)" || true |
|||
netstat -tlnp | grep -E "(8333|9333|8080)" || true |
|||
|
|||
- name: Upload test logs on failure |
|||
if: failure() |
|||
uses: actions/upload-artifact@v5 |
|||
with: |
|||
name: s3-filer-group-test-logs |
|||
path: test/s3/filer_group/weed-test*.log |
|||
retention-days: 3 |
|||
|
|||
@ -0,0 +1,166 @@ |
|||
# S3 Filer Group Test Makefile
|
|||
# This Makefile provides targets for running S3 filer group integration tests
|
|||
# These tests verify that S3 bucket operations work correctly when a filer group is configured.
|
|||
|
|||
.PHONY: help build-weed check-deps start-server stop-server test test-with-server clean logs health-check |
|||
|
|||
# Configuration
|
|||
WEED_BINARY := ../../../weed/weed_binary |
|||
S3_PORT := 8333 |
|||
MASTER_PORT := 9333 |
|||
MASTER_GRPC_PORT := 19333 |
|||
VOLUME_PORT := 8080 |
|||
FILER_PORT := 8888 |
|||
TEST_TIMEOUT := 10m |
|||
TEST_PATTERN := Test |
|||
|
|||
# Filer group configuration
|
|||
FILER_GROUP := testgroup |
|||
|
|||
# Default target
|
|||
help: |
|||
@echo "S3 Filer Group Test Makefile" |
|||
@echo "" |
|||
@echo "Available targets:" |
|||
@echo " help - Show this help message" |
|||
@echo " build-weed - Build the SeaweedFS binary" |
|||
@echo " check-deps - Check dependencies and build binary if needed" |
|||
@echo " start-server - Start SeaweedFS server with filer group" |
|||
@echo " stop-server - Stop SeaweedFS server" |
|||
@echo " test - Run filer group tests (server must be running)" |
|||
@echo " test-with-server - Start server, run tests, stop server" |
|||
@echo " logs - Show server logs" |
|||
@echo " clean - Clean up test artifacts and stop server" |
|||
@echo " health-check - Check if server is accessible" |
|||
@echo "" |
|||
@echo "Configuration:" |
|||
@echo " S3_PORT=${S3_PORT}" |
|||
@echo " MASTER_GRPC_PORT=${MASTER_GRPC_PORT}" |
|||
@echo " FILER_GROUP=${FILER_GROUP}" |
|||
@echo " TEST_TIMEOUT=${TEST_TIMEOUT}" |
|||
|
|||
# Build the SeaweedFS binary
|
|||
build-weed: |
|||
@echo "Building SeaweedFS binary..." |
|||
@cd ../../../weed && go build -o weed_binary . |
|||
@chmod +x $(WEED_BINARY) |
|||
@echo "✅ SeaweedFS binary built at $(WEED_BINARY)" |
|||
|
|||
check-deps: build-weed |
|||
@echo "Checking dependencies..." |
|||
@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1) |
|||
@echo "Go version: $$(go version)" |
|||
@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1) |
|||
@go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1) |
|||
@go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1) |
|||
@echo "✅ All dependencies are available" |
|||
|
|||
# Start SeaweedFS server with filer group configured
|
|||
start-server: check-deps |
|||
@echo "Starting SeaweedFS server with filer group: $(FILER_GROUP)..." |
|||
@rm -f weed-server.pid |
|||
@mkdir -p ./test-volume-data |
|||
@if netstat -tlnp 2>/dev/null | grep $(S3_PORT) >/dev/null; then \
|
|||
echo "⚠️ Port $(S3_PORT) is already in use"; \
|
|||
exit 1; \
|
|||
fi |
|||
@echo "Launching SeaweedFS server with filer group $(FILER_GROUP)..." |
|||
@$(WEED_BINARY) server \
|
|||
-debug \
|
|||
-s3 \
|
|||
-s3.port=$(S3_PORT) \
|
|||
-s3.allowDeleteBucketNotEmpty=true \
|
|||
-s3.config=../../../docker/compose/s3.json \
|
|||
-filer \
|
|||
-filer.maxMB=64 \
|
|||
-filer.filerGroup=$(FILER_GROUP) \
|
|||
-master.volumeSizeLimitMB=50 \
|
|||
-volume.max=100 \
|
|||
-dir=./test-volume-data \
|
|||
-volume.preStopSeconds=1 \
|
|||
-metricsPort=9325 \
|
|||
> weed-test.log 2>&1 & echo $$! > weed-server.pid |
|||
@echo "Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')" |
|||
@echo "Waiting for server to start (up to 90 seconds)..." |
|||
@for i in $$(seq 1 90); do \
|
|||
if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
|
|||
echo "✅ SeaweedFS server started successfully on port $(S3_PORT) with filer group $(FILER_GROUP)"; \
|
|||
exit 0; \
|
|||
fi; \
|
|||
if [ $$i -eq 30 ]; then \
|
|||
echo "⚠️ Server taking longer than expected (30s), checking logs..."; \
|
|||
if [ -f weed-test.log ]; then \
|
|||
tail -20 weed-test.log; \
|
|||
fi; \
|
|||
fi; \
|
|||
sleep 1; \
|
|||
done; \
|
|||
echo "❌ Server failed to start within 90 seconds"; \
|
|||
if [ -f weed-test.log ]; then \
|
|||
cat weed-test.log; \
|
|||
fi; \
|
|||
exit 1 |
|||
|
|||
# Stop SeaweedFS server
|
|||
stop-server: |
|||
@echo "Stopping SeaweedFS server..." |
|||
@if [ -f weed-server.pid ]; then \
|
|||
SERVER_PID=$$(cat weed-server.pid); \
|
|||
echo "Killing server PID $$SERVER_PID"; \
|
|||
if ps -p $$SERVER_PID >/dev/null 2>&1; then \
|
|||
kill -TERM $$SERVER_PID 2>/dev/null || true; \
|
|||
sleep 2; \
|
|||
if ps -p $$SERVER_PID >/dev/null 2>&1; then \
|
|||
echo "Process still running, sending KILL signal..."; \
|
|||
kill -KILL $$SERVER_PID 2>/dev/null || true; \
|
|||
sleep 1; \
|
|||
fi; \
|
|||
else \
|
|||
echo "Process $$SERVER_PID not found (already stopped)"; \
|
|||
fi; \
|
|||
rm -f weed-server.pid; \
|
|||
else \
|
|||
echo "No PID file found"; \
|
|||
fi |
|||
@echo "✅ SeaweedFS server stopped" |
|||
|
|||
# Show server logs
|
|||
logs: |
|||
@if test -f weed-test.log; then \
|
|||
echo "=== SeaweedFS Server Logs ==="; \
|
|||
tail -f weed-test.log; \
|
|||
else \
|
|||
echo "No log file found. Server may not be running."; \
|
|||
fi |
|||
|
|||
# Run filer group tests (assumes server is already running)
|
|||
test: check-deps |
|||
@echo "Running filer group S3 tests..." |
|||
@FILER_GROUP=$(FILER_GROUP) S3_ENDPOINT=http://localhost:$(S3_PORT) MASTER_ADDRESS=localhost:$(MASTER_GRPC_PORT) \
|
|||
go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" . |
|||
@echo "✅ Filer group tests completed" |
|||
|
|||
# Run tests with automatic server management
|
|||
test-with-server: start-server |
|||
@echo "Server started successfully, running filer group tests..." |
|||
@echo "Test pattern: $(TEST_PATTERN)" |
|||
@echo "Test timeout: $(TEST_TIMEOUT)" |
|||
@trap "$(MAKE) stop-server" EXIT; \
|
|||
$(MAKE) test || (echo "❌ Tests failed, showing server logs:" && echo "=== Last 50 lines of server logs ===" && tail -50 weed-test.log && echo "=== End of server logs ===" && exit 1) |
|||
@$(MAKE) stop-server |
|||
@echo "✅ Tests completed and server stopped" |
|||
|
|||
# Clean up test artifacts
|
|||
clean: |
|||
@echo "Cleaning up test artifacts..." |
|||
@$(MAKE) stop-server |
|||
@rm -f weed-test*.log weed-server.pid |
|||
@rm -rf test-volume-data/ |
|||
@go clean -testcache |
|||
@echo "✅ Cleanup completed" |
|||
|
|||
# Quick health check
|
|||
health-check: |
|||
@echo "Running health check..." |
|||
@curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1 && echo "✅ S3 API is accessible" || echo "❌ S3 API is not accessible" |
|||
@curl -s http://localhost:9325/metrics >/dev/null 2>&1 && echo "✅ Metrics endpoint is accessible" || echo "❌ Metrics endpoint is not accessible" |
|||
@ -0,0 +1,99 @@ |
|||
# Filer Group S3 Tests |
|||
|
|||
These tests verify that S3 bucket operations work correctly when a filer group is configured. |
|||
|
|||
## Background |
|||
|
|||
When SeaweedFS is configured with a filer group (via `-filer.group` option), collections are named with the filer group prefix: |
|||
|
|||
```text |
|||
Collection name = {filerGroup}_{bucketName} |
|||
``` |
|||
|
|||
For example, with filer group `mygroup` and bucket `mybucket`, the collection will be named `mygroup_mybucket`. |
|||
|
|||
## Issue Being Tested |
|||
|
|||
This test suite was created to verify the fix for a bug where: |
|||
- The admin UI was using just the bucket name when deleting collections |
|||
- This caused collection deletion to fail when a filer group was configured |
|||
- After bucket deletion via admin UI, the collection data would be orphaned |
|||
|
|||
## Running the Tests |
|||
|
|||
### Prerequisites |
|||
|
|||
1. SeaweedFS servers must be running with a filer group configured |
|||
2. The S3 gateway must be accessible |
|||
3. Master server must be accessible for collection verification |
|||
|
|||
### Quick Start |
|||
|
|||
```bash |
|||
# Set environment variables |
|||
export FILER_GROUP=testgroup |
|||
export S3_ENDPOINT=http://localhost:8333 |
|||
export MASTER_ADDRESS=localhost:9333 |
|||
|
|||
# Run tests |
|||
go test -v ./... |
|||
``` |
|||
|
|||
### Using the Makefile |
|||
|
|||
```bash |
|||
# Start test servers with filer group configured |
|||
make start-servers FILER_GROUP=testgroup |
|||
|
|||
# Run tests |
|||
make test |
|||
|
|||
# Stop servers |
|||
make stop-servers |
|||
|
|||
# Or run full test cycle |
|||
make full-test |
|||
``` |
|||
|
|||
### Configuration |
|||
|
|||
Tests can be configured via: |
|||
|
|||
1. Environment variables: |
|||
- `FILER_GROUP`: The filer group name (required for tests to run) |
|||
- `S3_ENDPOINT`: S3 API endpoint (default: `http://localhost:8333`) |
|||
- `MASTER_ADDRESS`: Master server address (default: localhost:9333) |
|||
|
|||
2. `test_config.json` file |
|||
|
|||
## Test Cases |
|||
|
|||
### TestFilerGroupCollectionNaming |
|||
Verifies that when a bucket is created and objects are uploaded: |
|||
1. The collection is created with the correct filer group prefix |
|||
2. Bucket deletion removes the correctly-named collection |
|||
|
|||
### TestBucketDeletionWithFilerGroup |
|||
Specifically tests that bucket deletion via S3 API correctly deletes |
|||
the collection when filer group is configured. |
|||
|
|||
### TestMultipleBucketsWithFilerGroup |
|||
Tests creating and deleting multiple buckets to ensure the filer group |
|||
prefix is correctly applied and removed for all buckets. |
|||
|
|||
## Expected Behavior |
|||
|
|||
With filer group `testgroup`: |
|||
|
|||
| Bucket Name | Expected Collection Name | |
|||
|-------------|-------------------------| |
|||
| mybucket | testgroup_mybucket | |
|||
| test-123 | testgroup_test-123 | |
|||
|
|||
Without filer group: |
|||
|
|||
| Bucket Name | Expected Collection Name | |
|||
|-------------|-------------------------| |
|||
| mybucket | mybucket | |
|||
| test-123 | test-123 | |
|||
|
|||
@ -0,0 +1,332 @@ |
|||
package filer_group |
|||
|
|||
import ( |
|||
"context" |
|||
"encoding/json" |
|||
"fmt" |
|||
"os" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/aws/aws-sdk-go-v2/aws" |
|||
"github.com/aws/aws-sdk-go-v2/config" |
|||
"github.com/aws/aws-sdk-go-v2/credentials" |
|||
"github.com/aws/aws-sdk-go-v2/service/s3" |
|||
"github.com/stretchr/testify/assert" |
|||
"github.com/stretchr/testify/require" |
|||
"google.golang.org/grpc" |
|||
"google.golang.org/grpc/credentials/insecure" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb" |
|||
) |
|||
|
|||
// TestConfig holds configuration for filer group S3 tests
|
|||
type TestConfig struct { |
|||
S3Endpoint string `json:"s3_endpoint"` |
|||
MasterAddress string `json:"master_address"` |
|||
AccessKey string `json:"access_key"` |
|||
SecretKey string `json:"secret_key"` |
|||
Region string `json:"region"` |
|||
FilerGroup string `json:"filer_group"` |
|||
} |
|||
|
|||
var testConfig = &TestConfig{ |
|||
S3Endpoint: "http://localhost:8333", |
|||
MasterAddress: "localhost:19333", // gRPC port = 10000 + master HTTP port (9333)
|
|||
AccessKey: "some_access_key1", |
|||
SecretKey: "some_secret_key1", |
|||
Region: "us-east-1", |
|||
FilerGroup: "testgroup", // Expected filer group for these tests
|
|||
} |
|||
|
|||
func init() { |
|||
// Load config from file if exists
|
|||
if data, err := os.ReadFile("test_config.json"); err == nil { |
|||
if err := json.Unmarshal(data, testConfig); err != nil { |
|||
// Log but don't fail - env vars can still override
|
|||
fmt.Fprintf(os.Stderr, "Warning: failed to parse test_config.json: %v\n", err) |
|||
} |
|||
} |
|||
|
|||
// Override from environment variables
|
|||
if v := os.Getenv("S3_ENDPOINT"); v != "" { |
|||
testConfig.S3Endpoint = v |
|||
} |
|||
if v := os.Getenv("MASTER_ADDRESS"); v != "" { |
|||
testConfig.MasterAddress = v |
|||
} |
|||
if v := os.Getenv("FILER_GROUP"); v != "" { |
|||
testConfig.FilerGroup = v |
|||
} |
|||
} |
|||
|
|||
func getS3Client(t *testing.T) *s3.Client { |
|||
cfg, err := config.LoadDefaultConfig(context.TODO(), |
|||
config.WithRegion(testConfig.Region), |
|||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( |
|||
testConfig.AccessKey, |
|||
testConfig.SecretKey, |
|||
"", |
|||
)), |
|||
) |
|||
require.NoError(t, err) |
|||
|
|||
return s3.NewFromConfig(cfg, func(o *s3.Options) { |
|||
o.BaseEndpoint = aws.String(testConfig.S3Endpoint) |
|||
o.UsePathStyle = true |
|||
}) |
|||
} |
|||
|
|||
func getMasterClient(t *testing.T) master_pb.SeaweedClient { |
|||
conn, err := grpc.NewClient(testConfig.MasterAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) |
|||
require.NoError(t, err) |
|||
t.Cleanup(func() { conn.Close() }) |
|||
return master_pb.NewSeaweedClient(conn) |
|||
} |
|||
|
|||
func getNewBucketName() string { |
|||
return fmt.Sprintf("filergroup-test-%d", time.Now().UnixNano()) |
|||
} |
|||
|
|||
// getExpectedCollectionName returns the expected collection name for a bucket
|
|||
// When filer group is configured, collections are named: {filerGroup}_{bucketName}
|
|||
func getExpectedCollectionName(bucketName string) string { |
|||
if testConfig.FilerGroup != "" { |
|||
return fmt.Sprintf("%s_%s", testConfig.FilerGroup, bucketName) |
|||
} |
|||
return bucketName |
|||
} |
|||
|
|||
// listAllCollections returns a list of all collection names from the master
|
|||
func listAllCollections(t *testing.T, masterClient master_pb.SeaweedClient) []string { |
|||
collectionResp, err := masterClient.CollectionList(context.Background(), &master_pb.CollectionListRequest{ |
|||
IncludeNormalVolumes: true, |
|||
IncludeEcVolumes: true, |
|||
}) |
|||
if err != nil { |
|||
t.Logf("Warning: failed to list collections: %v", err) |
|||
return nil |
|||
} |
|||
var names []string |
|||
for _, c := range collectionResp.Collections { |
|||
names = append(names, c.Name) |
|||
} |
|||
return names |
|||
} |
|||
|
|||
// collectionExists checks if a collection exists in the master
|
|||
func collectionExists(t *testing.T, masterClient master_pb.SeaweedClient, collectionName string) bool { |
|||
for _, name := range listAllCollections(t, masterClient) { |
|||
if name == collectionName { |
|||
return true |
|||
} |
|||
} |
|||
return false |
|||
} |
|||
|
|||
// waitForCollectionExists waits for a collection to exist using polling
|
|||
func waitForCollectionExists(t *testing.T, masterClient master_pb.SeaweedClient, collectionName string) { |
|||
var lastCollections []string |
|||
success := assert.Eventually(t, func() bool { |
|||
lastCollections = listAllCollections(t, masterClient) |
|||
for _, name := range lastCollections { |
|||
if name == collectionName { |
|||
return true |
|||
} |
|||
} |
|||
return false |
|||
}, 10*time.Second, 200*time.Millisecond) |
|||
if !success { |
|||
t.Fatalf("collection %s should be created; existing collections: %v", collectionName, lastCollections) |
|||
} |
|||
} |
|||
|
|||
// waitForCollectionDeleted waits for a collection to be deleted using polling
|
|||
func waitForCollectionDeleted(t *testing.T, masterClient master_pb.SeaweedClient, collectionName string) { |
|||
require.Eventually(t, func() bool { |
|||
return !collectionExists(t, masterClient, collectionName) |
|||
}, 10*time.Second, 200*time.Millisecond, "collection %s should be deleted", collectionName) |
|||
} |
|||
|
|||
// TestFilerGroupCollectionNaming verifies that when a filer group is configured,
|
|||
// collections are created with the correct prefix ({filerGroup}_{bucketName})
|
|||
func TestFilerGroupCollectionNaming(t *testing.T) { |
|||
if testConfig.FilerGroup == "" { |
|||
t.Skip("Skipping test: FILER_GROUP not configured. Set FILER_GROUP environment variable or configure in test_config.json") |
|||
} |
|||
|
|||
s3Client := getS3Client(t) |
|||
masterClient := getMasterClient(t) |
|||
bucketName := getNewBucketName() |
|||
expectedCollection := getExpectedCollectionName(bucketName) |
|||
|
|||
t.Logf("Testing with filer group: %s", testConfig.FilerGroup) |
|||
t.Logf("Bucket name: %s", bucketName) |
|||
t.Logf("Expected collection name: %s", expectedCollection) |
|||
|
|||
// Create bucket
|
|||
_, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{ |
|||
Bucket: aws.String(bucketName), |
|||
}) |
|||
require.NoError(t, err, "CreateBucket should succeed") |
|||
|
|||
// Upload an object to trigger collection creation
|
|||
_, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String("test-object"), |
|||
Body: strings.NewReader("test content"), |
|||
}) |
|||
require.NoError(t, err, "PutObject should succeed") |
|||
|
|||
// Wait for collection to be visible using polling
|
|||
waitForCollectionExists(t, masterClient, expectedCollection) |
|||
|
|||
// Verify collection exists with correct name
|
|||
require.True(t, collectionExists(t, masterClient, expectedCollection), |
|||
"Collection %s should exist (filer group prefix applied)", expectedCollection) |
|||
|
|||
// Cleanup: delete object and bucket
|
|||
_, err = s3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String("test-object"), |
|||
}) |
|||
require.NoError(t, err, "DeleteObject should succeed") |
|||
|
|||
_, err = s3Client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{ |
|||
Bucket: aws.String(bucketName), |
|||
}) |
|||
require.NoError(t, err, "DeleteBucket should succeed") |
|||
|
|||
// Wait for collection to be deleted using polling
|
|||
waitForCollectionDeleted(t, masterClient, expectedCollection) |
|||
|
|||
t.Log("SUCCESS: Collection naming with filer group works correctly") |
|||
} |
|||
|
|||
// TestBucketDeletionWithFilerGroup verifies that bucket deletion correctly
|
|||
// deletes the collection when filer group is configured
|
|||
func TestBucketDeletionWithFilerGroup(t *testing.T) { |
|||
if testConfig.FilerGroup == "" { |
|||
t.Skip("Skipping test: FILER_GROUP not configured") |
|||
} |
|||
|
|||
s3Client := getS3Client(t) |
|||
masterClient := getMasterClient(t) |
|||
bucketName := getNewBucketName() |
|||
expectedCollection := getExpectedCollectionName(bucketName) |
|||
|
|||
// Create bucket and add an object
|
|||
_, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{ |
|||
Bucket: aws.String(bucketName), |
|||
}) |
|||
require.NoError(t, err) |
|||
|
|||
_, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String("test-object"), |
|||
Body: strings.NewReader("test content"), |
|||
}) |
|||
require.NoError(t, err) |
|||
|
|||
// Wait for collection to be created using polling
|
|||
waitForCollectionExists(t, masterClient, expectedCollection) |
|||
|
|||
// Verify collection exists before deletion
|
|||
require.True(t, collectionExists(t, masterClient, expectedCollection), |
|||
"Collection should exist before bucket deletion") |
|||
|
|||
// Delete object first
|
|||
_, err = s3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ |
|||
Bucket: aws.String(bucketName), |
|||
Key: aws.String("test-object"), |
|||
}) |
|||
require.NoError(t, err) |
|||
|
|||
// Delete bucket
|
|||
_, err = s3Client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{ |
|||
Bucket: aws.String(bucketName), |
|||
}) |
|||
require.NoError(t, err, "DeleteBucket should succeed") |
|||
|
|||
// Wait for collection to be deleted using polling
|
|||
waitForCollectionDeleted(t, masterClient, expectedCollection) |
|||
|
|||
// Verify collection was deleted
|
|||
require.False(t, collectionExists(t, masterClient, expectedCollection), |
|||
"Collection %s should be deleted after bucket deletion", expectedCollection) |
|||
|
|||
t.Log("SUCCESS: Bucket deletion with filer group correctly deletes collection") |
|||
} |
|||
|
|||
// TestMultipleBucketsWithFilerGroup tests creating and deleting multiple buckets
|
|||
func TestMultipleBucketsWithFilerGroup(t *testing.T) { |
|||
if testConfig.FilerGroup == "" { |
|||
t.Skip("Skipping test: FILER_GROUP not configured") |
|||
} |
|||
|
|||
s3Client := getS3Client(t) |
|||
masterClient := getMasterClient(t) |
|||
|
|||
buckets := make([]string, 3) |
|||
for i := 0; i < 3; i++ { |
|||
buckets[i] = getNewBucketName() |
|||
} |
|||
|
|||
// Create all buckets and add objects
|
|||
for _, bucket := range buckets { |
|||
_, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{ |
|||
Bucket: aws.String(bucket), |
|||
}) |
|||
require.NoError(t, err) |
|||
|
|||
_, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{ |
|||
Bucket: aws.String(bucket), |
|||
Key: aws.String("test-object"), |
|||
Body: strings.NewReader("test content"), |
|||
}) |
|||
require.NoError(t, err) |
|||
} |
|||
|
|||
// Wait for all collections to be created using polling
|
|||
for _, bucket := range buckets { |
|||
expectedCollection := getExpectedCollectionName(bucket) |
|||
waitForCollectionExists(t, masterClient, expectedCollection) |
|||
} |
|||
|
|||
// Verify all collections exist with correct naming
|
|||
for _, bucket := range buckets { |
|||
expectedCollection := getExpectedCollectionName(bucket) |
|||
require.True(t, collectionExists(t, masterClient, expectedCollection), |
|||
"Collection %s should exist for bucket %s", expectedCollection, bucket) |
|||
} |
|||
|
|||
// Delete all buckets
|
|||
for _, bucket := range buckets { |
|||
_, err := s3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ |
|||
Bucket: aws.String(bucket), |
|||
Key: aws.String("test-object"), |
|||
}) |
|||
require.NoError(t, err) |
|||
|
|||
_, err = s3Client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{ |
|||
Bucket: aws.String(bucket), |
|||
}) |
|||
require.NoError(t, err) |
|||
} |
|||
|
|||
// Wait for all collections to be deleted using polling
|
|||
for _, bucket := range buckets { |
|||
expectedCollection := getExpectedCollectionName(bucket) |
|||
waitForCollectionDeleted(t, masterClient, expectedCollection) |
|||
} |
|||
|
|||
// Verify all collections are deleted
|
|||
for _, bucket := range buckets { |
|||
expectedCollection := getExpectedCollectionName(bucket) |
|||
require.False(t, collectionExists(t, masterClient, expectedCollection), |
|||
"Collection %s should be deleted for bucket %s", expectedCollection, bucket) |
|||
} |
|||
|
|||
t.Log("SUCCESS: Multiple bucket operations with filer group work correctly") |
|||
} |
|||
@ -0,0 +1,8 @@ |
|||
{ |
|||
"s3_endpoint": "http://localhost:8333", |
|||
"master_address": "localhost:9333", |
|||
"access_key": "some_access_key1", |
|||
"secret_key": "some_secret_key1", |
|||
"region": "us-east-1", |
|||
"filer_group": "testgroup" |
|||
} |
|||
@ -0,0 +1,6 @@ |
|||
package s3_constants |
|||
|
|||
const ( |
|||
// DefaultBucketsPath is the default path for S3 buckets in the filer
|
|||
DefaultBucketsPath = "/buckets" |
|||
) |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue