diff --git a/.github/workflows/s3-mutation-regression-tests.yml b/.github/workflows/s3-mutation-regression-tests.yml new file mode 100644 index 000000000..22960e550 --- /dev/null +++ b/.github/workflows/s3-mutation-regression-tests.yml @@ -0,0 +1,133 @@ +name: "S3 Mutation Regression Tests" + +on: + pull_request: + paths: + - 'weed/s3api/**' + - 'test/s3/delete/**' + - 'test/s3/distributed_lock/**' + - 'test/s3/versioning/**' + - 'test/volume_server/framework/**' + - 'docker/compose/s3.json' + - '.github/workflows/s3-mutation-regression-tests.yml' + +concurrency: + group: ${{ github.head_ref }}/s3-mutation-regression-tests + cancel-in-progress: true + +permissions: + contents: read + +jobs: + s3-versioning-regressions: + name: S3 Versioning Regression Tests + runs-on: ubuntu-22.04 + timeout-minutes: 25 + + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + + - name: Run S3 versioning regression tests + timeout-minutes: 20 + working-directory: test/s3/versioning + run: | + set -x + make test-with-server TEST_PATTERN="TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker" + + - name: Show server logs on failure + if: failure() + working-directory: test/s3/versioning + run: | + echo "=== Server Logs ===" + if [ -f weed-test.log ]; then + tail -100 weed-test.log + fi + + - name: Upload versioning logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-versioning-regression-logs + path: test/s3/versioning/weed-test*.log + retention-days: 3 + + s3-delete-regressions: + name: S3 Delete Regression Tests + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + + - name: Run S3 delete regression tests + timeout-minutes: 15 + working-directory: test/s3/delete + run: | + set -x + make test-with-server + + - name: Show server logs on failure + if: failure() + working-directory: test/s3/delete + run: | + echo "=== Server Logs ===" + if [ -f weed-test.log ]; then + tail -100 weed-test.log + fi + + - name: Upload delete logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-delete-regression-logs + path: test/s3/delete/weed-test*.log + retention-days: 3 + + s3-distributed-lock-regressions: + name: S3 Distributed Lock Regression Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + + - name: Build SeaweedFS + run: | + go build -o weed/weed -buildvcs=false ./weed + + - name: Run distributed lock regressions + timeout-minutes: 25 + env: + TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp + S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1" + WEED_BINARY: ${{ github.workspace }}/weed/weed + run: | + set -x + mkdir -p "$TMPDIR" + go test -v -count=1 -timeout=20m ./test/s3/distributed_lock + + - name: Upload distributed lock logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-distributed-lock-regression-logs + path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_* + retention-days: 3 diff --git a/test/s3/delete/Makefile b/test/s3/delete/Makefile new file mode 100644 index 000000000..a3a00a409 --- /dev/null +++ b/test/s3/delete/Makefile @@ -0,0 +1,98 @@ +# S3 delete regression test harness + +.PHONY: all test help build-weed check-deps start-server stop-server test-delete test-with-server clean logs + +WEED_BINARY := ../../../weed/weed_binary +S3_PORT := 8333 +TEST_TIMEOUT := 10m +SERVER_DIR := ./test-volume-data +S3_CONFIG := ./test_s3.json + +.DEFAULT_GOAL := help + +all: test + +test: test-with-server + +help: + @echo "S3 delete regression test harness" + @echo "" + @echo "Available targets:" + @echo " build-weed - Build the SeaweedFS binary" + @echo " start-server - Start SeaweedFS with S3 enabled" + @echo " stop-server - Stop the test server" + @echo " test-delete - Run delete regression tests" + @echo " test-with-server - Start server, run tests, stop server" + @echo " logs - Show server logs" + @echo " clean - Remove test artifacts" + +build-weed: + @echo "Building SeaweedFS binary..." + @cd ../../../weed && go build -o weed_binary . + @chmod +x $(WEED_BINARY) + +check-deps: build-weed + @command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1) + @test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1) + @test -f $(S3_CONFIG) || (echo "S3 config not found at $(S3_CONFIG)" && exit 1) + +start-server: check-deps + @echo "Starting SeaweedFS server for delete regression tests..." + @rm -f weed-server.pid + @mkdir -p $(SERVER_DIR) + @$(WEED_BINARY) mini \ + -dir=$(SERVER_DIR) \ + -s3 \ + -s3.port=$(S3_PORT) \ + -s3.config=$(S3_CONFIG) \ + -master.peers=none \ + > weed-test.log 2>&1 & echo $$! > weed-server.pid + @for i in $$(seq 1 60); do \ + if curl -sS -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \ + echo "SeaweedFS S3 server is ready on port $(S3_PORT)"; \ + exit 0; \ + fi; \ + sleep 1; \ + done; \ + echo "SeaweedFS S3 server failed to start"; \ + echo "=== Server logs ==="; \ + test -f weed-test.log && cat weed-test.log || true; \ + exit 1 + +stop-server: + @echo "Stopping SeaweedFS server..." + @if [ -f weed-server.pid ]; then \ + SERVER_PID=$$(cat weed-server.pid); \ + kill -TERM $$SERVER_PID 2>/dev/null || true; \ + for i in $$(seq 1 20); do \ + if ! kill -0 $$SERVER_PID >/dev/null 2>&1; then \ + break; \ + fi; \ + sleep 0.1; \ + done; \ + if kill -0 $$SERVER_PID >/dev/null 2>&1; then \ + kill -KILL $$SERVER_PID 2>/dev/null || true; \ + fi; \ + rm -f weed-server.pid; \ + fi + +test-delete: check-deps + @echo "Running delete regression tests..." + @go test -v -timeout=$(TEST_TIMEOUT) . + +test-with-server: start-server + @trap "$(MAKE) stop-server" EXIT; \ + $(MAKE) test-delete || (echo "=== Last 100 lines of server logs ===" && tail -100 weed-test.log && exit 1) + @$(MAKE) stop-server + +logs: + @if test -f weed-test.log; then \ + tail -f weed-test.log; \ + else \ + echo "No server log file found."; \ + fi + +clean: + @$(MAKE) stop-server + @rm -f weed-test.log weed-server.pid + @rm -rf $(SERVER_DIR) diff --git a/test/s3/delete/test_s3.json b/test/s3/delete/test_s3.json new file mode 100644 index 000000000..1f9b46000 --- /dev/null +++ b/test/s3/delete/test_s3.json @@ -0,0 +1,20 @@ +{ + "identities": [ + { + "name": "admin", + "credentials": [ + { + "accessKey": "admin", + "secretKey": "admin" + } + ], + "actions": [ + "Admin", + "Read", + "List", + "Tagging", + "Write" + ] + } + ] +} diff --git a/test/s3/distributed_lock/distributed_lock_test.go b/test/s3/distributed_lock/distributed_lock_test.go index 31dd0f181..63adec185 100644 --- a/test/s3/distributed_lock/distributed_lock_test.go +++ b/test/s3/distributed_lock/distributed_lock_test.go @@ -74,33 +74,31 @@ func runConditionalPutRace(t *testing.T, clients []s3RaceClient, bucket, key str t.Helper() start := make(chan struct{}) - results := make(chan putAttemptResult, len(clients)*2) + results := make(chan putAttemptResult, len(clients)) var wg sync.WaitGroup for _, client := range clients { - for attempt := 0; attempt < 2; attempt++ { - wg.Add(1) - body := fmt.Sprintf("%s-attempt-%d", client.name, attempt) - go func(client s3RaceClient, body string) { - defer wg.Done() - <-start - - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - _, err := client.client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - IfNoneMatch: aws.String("*"), - Body: bytes.NewReader([]byte(body)), - }) - results <- putAttemptResult{ - clientName: client.name, - body: body, - err: err, - } - }(client, body) - } + wg.Add(1) + body := fmt.Sprintf("%s-race", client.name) + go func(client s3RaceClient, body string) { + defer wg.Done() + <-start + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + _, err := client.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + IfNoneMatch: aws.String("*"), + Body: bytes.NewReader([]byte(body)), + }) + results <- putAttemptResult{ + clientName: client.name, + body: body, + err: err, + } + }(client, body) } close(start) @@ -127,7 +125,7 @@ func runConditionalPutRace(t *testing.T, clients []s3RaceClient, bucket, key str require.Empty(t, unexpectedErrors, "unexpected race errors") require.Equal(t, 1, successes, "exactly one write should win") - require.Equal(t, len(clients)*2-1, preconditionFailures, "all losing writes should fail with 412") + require.Equal(t, len(clients)-1, preconditionFailures, "all losing writes should fail with 412") object, err := clients[0].client.GetObject(context.Background(), &s3.GetObjectInput{ Bucket: aws.String(bucket), diff --git a/test/s3/versioning/Makefile b/test/s3/versioning/Makefile index 7e939f90e..7884f9e7b 100644 --- a/test/s3/versioning/Makefile +++ b/test/s3/versioning/Makefile @@ -1,7 +1,7 @@ # S3 API Test Makefile # This Makefile provides comprehensive targets for running S3 versioning tests -.PHONY: help build-weed setup-server start-server stop-server test-versioning test-versioning-quick test-versioning-comprehensive test-all clean logs check-deps +.PHONY: all test help build-weed setup-server start-server stop-server test-versioning test-versioning-quick test-versioning-regressions test-versioning-comprehensive test-all clean logs check-deps # Configuration WEED_BINARY := ../../../weed/weed_binary @@ -12,6 +12,12 @@ FILER_PORT := 8888 TEST_TIMEOUT := 10m TEST_PATTERN := TestVersioning +.DEFAULT_GOAL := help + +all: test + +test: test-with-server + # Default target help: @echo "S3 API Test Makefile" @@ -25,6 +31,7 @@ help: @echo " stop-server - Stop SeaweedFS server" @echo " test-versioning - Run all versioning tests" @echo " test-versioning-quick - Run core versioning tests only" + @echo " test-versioning-regressions - Run targeted versioning regression tests" @echo " test-versioning-simple - Run tests without server management" @echo " test-versioning-comprehensive - Run comprehensive versioning tests" @echo " test-all - Run all S3 API tests" @@ -179,6 +186,12 @@ test-versioning-quick: check-deps @go test -v -timeout=$(TEST_TIMEOUT) -run "TestBucketListReturnDataVersioning|TestVersioningBasicWorkflow|TestVersioningDeleteMarkers" . @echo "✅ Core versioning tests completed" +# Targeted regressions for conditional mutation, multipart, copy, and suspended delete flows. +test-versioning-regressions: check-deps + @echo "Running targeted S3 versioning regression tests..." + @go test -v -timeout=$(TEST_TIMEOUT) -run "TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker" . + @echo "✅ Targeted versioning regression tests completed" + # All versioning tests test-versioning: check-deps @echo "Running all S3 versioning tests..." @@ -348,4 +361,4 @@ compare-python-tests: @echo "Python equivalent: test_bucket_list_return_data_versioning" @echo "" @echo "Running Go version..." - @time go test -v -run "TestBucketListReturnDataVersioning" . 2>&1 | grep -E "(PASS|FAIL|took)" \ No newline at end of file + @time go test -v -run "TestBucketListReturnDataVersioning" . 2>&1 | grep -E "(PASS|FAIL|took)"