diff --git a/.github/workflows/s3-tables-tests.yml b/.github/workflows/s3-tables-tests.yml index df365956d..b6e9e69f7 100644 --- a/.github/workflows/s3-tables-tests.yml +++ b/.github/workflows/s3-tables-tests.yml @@ -328,6 +328,141 @@ jobs: path: test/s3tables/catalog_risingwave/test-output.log retention-days: 3 + sts-integration-tests: + name: STS Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + id: go + + - name: Set up Docker + uses: docker/setup-buildx-action@v3 + + - name: Pre-pull Python image + run: docker pull python:3 + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Run STS Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/sts_integration + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting STS Integration Tests ===" + + # Run STS integration tests + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "STS integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/sts_integration + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v6 + with: + name: sts-integration-test-logs + path: test/s3tables/sts_integration/test-output.log + retention-days: 3 + + lakekeeper-integration-tests: + name: Lakekeeper Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + id: go + + - name: Set up Docker + uses: docker/setup-buildx-action@v3 + + - name: Pre-pull Python image + run: docker pull python:3 + + - name: Pre-pull LocalStack image (if needed) + run: docker pull localstack/localstack:latest || true + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Run Lakekeeper Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/lakekeeper + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting Lakekeeper Integration Tests ===" + + # Run Lakekeeper integration tests + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "Lakekeeper integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/lakekeeper + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v6 + with: + name: lakekeeper-integration-test-logs + path: test/s3tables/lakekeeper/test-output.log + retention-days: 3 + s3-tables-build-verification: name: S3 Tables Build Verification runs-on: ubuntu-22.04 diff --git a/.github/workflows/volume-server-integration-tests.yml b/.github/workflows/volume-server-integration-tests.yml new file mode 100644 index 000000000..5c278d13d --- /dev/null +++ b/.github/workflows/volume-server-integration-tests.yml @@ -0,0 +1,122 @@ +name: "Volume Server Integration Tests" + +on: + pull_request: + branches: [ master ] + paths: + - 'test/volume_server/**' + - 'weed/server/**' + - 'weed/storage/**' + - 'weed/pb/volume_server.proto' + - 'weed/pb/volume_server_pb/**' + - '.github/workflows/volume-server-integration-tests.yml' + push: + branches: [ master, main ] + paths: + - 'test/volume_server/**' + - 'weed/server/**' + - 'weed/storage/**' + - 'weed/pb/volume_server.proto' + - 'weed/pb/volume_server_pb/**' + - '.github/workflows/volume-server-integration-tests.yml' + +concurrency: + group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests + cancel-in-progress: true + +permissions: + contents: read + +env: + GO_VERSION: '1.24' + TEST_TIMEOUT: '30m' + +jobs: + volume-server-integration-tests: + name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }}) + runs-on: ubuntu-22.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + test-type: [grpc, http] + shard: [1, 2, 3] + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Build SeaweedFS binary + run: | + cd weed + go build -o weed . + chmod +x weed + ./weed version + + - name: Run volume server integration tests + env: + WEED_BINARY: ${{ github.workspace }}/weed/weed + run: | + if [ "${{ matrix.test-type }}" == "grpc" ]; then + if [ "${{ matrix.shard }}" == "1" ]; then + TEST_PATTERN="^Test[A-H]" + elif [ "${{ matrix.shard }}" == "2" ]; then + TEST_PATTERN="^Test[I-S]" + else + TEST_PATTERN="^Test[T-Z]" + fi + else + if [ "${{ matrix.shard }}" == "1" ]; then + TEST_PATTERN="^Test[A-G]" + elif [ "${{ matrix.shard }}" == "2" ]; then + TEST_PATTERN="^Test[H-R]" + else + TEST_PATTERN="^Test[S-Z]" + fi + fi + echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..." + go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}" + + - name: Collect logs on failure + if: failure() + run: | + mkdir -p /tmp/volume-server-it-logs + find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-it-logs/ \; || true + + - name: Archive logs on failure + if: failure() + uses: actions/upload-artifact@v6 + with: + name: volume-server-integration-test-logs + path: /tmp/volume-server-it-logs/ + if-no-files-found: warn + retention-days: 7 + + - name: Test summary + if: always() + run: | + if [ "${{ matrix.test-type }}" == "grpc" ]; then + if [ "${{ matrix.shard }}" == "1" ]; then + TEST_PATTERN="^Test[A-H]" + elif [ "${{ matrix.shard }}" == "2" ]; then + TEST_PATTERN="^Test[I-S]" + else + TEST_PATTERN="^Test[T-Z]" + fi + else + if [ "${{ matrix.shard }}" == "1" ]; then + TEST_PATTERN="^Test[A-G]" + elif [ "${{ matrix.shard }}" == "2" ]; then + TEST_PATTERN="^Test[H-R]" + else + TEST_PATTERN="^Test[S-Z]" + fi + fi + echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY" + echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY" + echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY" diff --git a/k8s/charts/seaweedfs/templates/admin/admin-service.yaml b/k8s/charts/seaweedfs/templates/admin/admin-service.yaml index 8dc801b34..8686926ac 100644 --- a/k8s/charts/seaweedfs/templates/admin/admin-service.yaml +++ b/k8s/charts/seaweedfs/templates/admin/admin-service.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ printf "%s-admin" (include "seaweedfs.fullname" .) | trunc 63 | trimSuffix "-" }} + name: {{ include "seaweedfs.componentName" (list . "admin") }} namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} diff --git a/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml b/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml index 672d9aee9..23617df56 100644 --- a/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml @@ -134,7 +134,7 @@ spec: {{- if .Values.worker.adminServer }} -admin={{ .Values.worker.adminServer }} \ {{- else }} - -admin={{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }}{{ if .Values.admin.grpcPort }}.{{ .Values.admin.grpcPort }}{{ end }} \ + -admin={{ template "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }}{{ if .Values.admin.grpcPort }}.{{ .Values.admin.grpcPort }}{{ end }} \ {{- end }} -capabilities={{ .Values.worker.capabilities }} \ -maxConcurrent={{ .Values.worker.maxConcurrent }} \ diff --git a/test/s3/iam/s3_sts_credential_prefix_test.go b/test/s3/iam/s3_sts_credential_prefix_test.go new file mode 100644 index 000000000..b8736d21f --- /dev/null +++ b/test/s3/iam/s3_sts_credential_prefix_test.go @@ -0,0 +1,82 @@ +package iam + +import ( + "encoding/xml" + "io" + "net/http" + "net/url" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSTSTemporaryCredentialPrefix verifies that STS temporary credentials use ASIA prefix +// This test ensures AWS compatibility - temporary credentials should use ASIA, not AKIA +func TestSTSTemporaryCredentialPrefix(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + if !isSTSEndpointRunning(t) { + t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint) + } + + // Use test credentials from environment or fall back to defaults + accessKey := os.Getenv("STS_TEST_ACCESS_KEY") + if accessKey == "" { + accessKey = "admin" + } + secretKey := os.Getenv("STS_TEST_SECRET_KEY") + if secretKey == "" { + secretKey = "admin" + } + + t.Run("assume_role_returns_asia_prefix", func(t *testing.T) { + resp, err := callSTSAPIWithSigV4(t, url.Values{ + "Action": {"AssumeRole"}, + "Version": {"2011-06-15"}, + "RoleArn": {"arn:aws:iam::role/admin"}, + "RoleSessionName": {"asia-prefix-test"}, + }, accessKey, secretKey) + require.NoError(t, err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + if resp.StatusCode != http.StatusOK { + t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body)) + t.Skip("AssumeRole not fully implemented yet") + } + + var stsResp AssumeRoleTestResponse + err = xml.Unmarshal(body, &stsResp) + require.NoError(t, err, "Failed to parse response: %s", string(body)) + + creds := stsResp.Result.Credentials + require.NotEmpty(t, creds.AccessKeyId, "AccessKeyId should not be empty") + + // Verify ASIA prefix for temporary credentials + assert.True(t, strings.HasPrefix(creds.AccessKeyId, "ASIA"), + "Temporary credentials must use ASIA prefix (not AKIA for permanent keys), got: %s", creds.AccessKeyId) + + // Verify it's NOT using AKIA (permanent credentials) + assert.False(t, strings.HasPrefix(creds.AccessKeyId, "AKIA"), + "Temporary credentials must NOT use AKIA prefix (that's for permanent IAM keys), got: %s", creds.AccessKeyId) + + // Verify format: ASIA + 16 hex characters = 20 chars total + assert.Equal(t, 20, len(creds.AccessKeyId), + "Access key ID should be 20 characters (ASIA + 16 hex chars), got: %s", creds.AccessKeyId) + + t.Logf("✓ Temporary credentials correctly use ASIA prefix: %s", creds.AccessKeyId) + }) + + t.Run("assume_role_with_web_identity_returns_asia_prefix", func(t *testing.T) { + // This test would require OIDC setup, so we'll skip it for now + // but the same ASIA prefix validation should apply + t.Skip("AssumeRoleWithWebIdentity requires OIDC provider setup") + }) +} diff --git a/test/s3tables/catalog_risingwave/risingwave_dml_test.go b/test/s3tables/catalog_risingwave/risingwave_dml_test.go new file mode 100644 index 000000000..bf1566075 --- /dev/null +++ b/test/s3tables/catalog_risingwave/risingwave_dml_test.go @@ -0,0 +1,219 @@ +package catalog_risingwave + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestRisingWaveIcebergDML(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + env := NewTestEnvironment(t) + defer env.Cleanup(t) + + if !env.dockerAvailable { + t.Skip("Docker not available, skipping RisingWave integration test") + } + + t.Log(">>> Starting SeaweedFS...") + env.StartSeaweedFS(t) + t.Log(">>> SeaweedFS started.") + + tableBucket := "iceberg-tables" + t.Logf(">>> Creating table bucket: %s", tableBucket) + createTableBucket(t, env, tableBucket) + + t.Log(">>> Starting RisingWave...") + env.StartRisingWave(t) + t.Log(">>> RisingWave started.") + + // Create Iceberg namespace + createIcebergNamespace(t, env, "default") + + icebergUri := env.dockerIcebergEndpoint() + s3Endpoint := env.dockerS3Endpoint() + + // 1. Test INSERT (Append-only) + t.Run("TestInsert", func(t *testing.T) { + tableName := "test_insert_" + randomString(6) + createIcebergTable(t, env, tableBucket, "default", tableName) + + rwTableName := "rw_insert_" + randomString(6) + runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("CREATE TABLE %s (id int, name varchar);", rwTableName)) + + sinkName := "test_sink_insert_" + randomString(6) + createSinkSql := fmt.Sprintf(` +CREATE SINK %s FROM %s +WITH ( + connector = 'iceberg', + catalog.type = 'rest', + catalog.uri = '%s', + catalog.name = 'default', + database.name = 'default', + table.name = '%s', + warehouse.path = 's3://%s', + s3.endpoint = '%s', + s3.region = 'us-east-1', + s3.access.key = '%s', + s3.secret.key = '%s', + s3.path.style.access = 'true', + catalog.rest.sigv4_enabled = 'true', + catalog.rest.signing_region = 'us-east-1', + catalog.rest.signing_name = 's3', + type = 'append-only', + force_append_only = 'true' +);`, sinkName, rwTableName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey) + + t.Logf(">>> Creating sink %s...", sinkName) + runRisingWaveSQL(t, env.postgresSidecar, createSinkSql) + + t.Log(">>> Inserting into RisingWave table...") + runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("INSERT INTO %s VALUES (1, 'Alice'), (2, 'Bob');", rwTableName)) + runRisingWaveSQL(t, env.postgresSidecar, "FLUSH;") + + // Verify with Source + sourceName := "test_source_insert_" + randomString(6) + createSourceSql := fmt.Sprintf(` +CREATE SOURCE %s WITH ( + connector = 'iceberg', + catalog.type = 'rest', + catalog.uri = '%s', + catalog.name = 'default', + database.name = 'default', + table.name = '%s', + warehouse.path = 's3://%s', + s3.endpoint = '%s', + s3.region = 'us-east-1', + s3.access.key = '%s', + s3.secret.key = '%s', + s3.path.style.access = 'true', + catalog.rest.sigv4_enabled = 'true', + catalog.rest.signing_region = 'us-east-1', + catalog.rest.signing_name = 's3' +);`, sourceName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey) + + runRisingWaveSQL(t, env.postgresSidecar, createSourceSql) + + t.Log(">>> Selecting from source to verify INSERT...") + verifyQuery(t, env, sourceName, "1 | Alice", "2 | Bob") + }) + + // 2. Test UPSERT (Update/Delete) + t.Run("TestUpsert", func(t *testing.T) { + tableName := "test_upsert_" + randomString(6) + // We need a table with PK for upsert to work effectively in RW logic, + // effectively maps to Iceberg v2 table. + createIcebergTable(t, env, tableBucket, "default", tableName) + + rwTableName := "rw_upsert_" + randomString(6) + runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("CREATE TABLE %s (id int PRIMARY KEY, name varchar);", rwTableName)) + + sinkName := "test_sink_upsert_" + randomString(6) + createSinkSql := fmt.Sprintf(` +CREATE SINK %s FROM %s +WITH ( + connector = 'iceberg', + catalog.type = 'rest', + catalog.uri = '%s', + catalog.name = 'default', + database.name = 'default', + table.name = '%s', + warehouse.path = 's3://%s', + s3.endpoint = '%s', + s3.region = 'us-east-1', + s3.access.key = '%s', + s3.secret.key = '%s', + s3.path.style.access = 'true', + catalog.rest.sigv4_enabled = 'true', + catalog.rest.signing_region = 'us-east-1', + catalog.rest.signing_name = 's3', + type = 'upsert', -- Upsert mode + primary_key = 'id' +);`, sinkName, rwTableName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey) + + t.Logf(">>> Creating upsert sink %s...", sinkName) + runRisingWaveSQL(t, env.postgresSidecar, createSinkSql) + + t.Log(">>> Inserting initial data...") + runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("INSERT INTO %s VALUES (1, 'Charlie'), (2, 'Dave');", rwTableName)) + runRisingWaveSQL(t, env.postgresSidecar, "FLUSH;") + + // Update 1, Delete 2 + t.Log(">>> Updating and Deleting data...") + runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("UPDATE %s SET name = 'Charles' WHERE id = 1;", rwTableName)) + runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("DELETE FROM %s WHERE id = 2;", rwTableName)) + runRisingWaveSQL(t, env.postgresSidecar, "FLUSH;") + + // Verify with Source + sourceName := "test_source_upsert_" + randomString(6) + createSourceSql := fmt.Sprintf(` +CREATE SOURCE %s WITH ( + connector = 'iceberg', + catalog.type = 'rest', + catalog.uri = '%s', + catalog.name = 'default', + database.name = 'default', + table.name = '%s', + warehouse.path = 's3://%s', + s3.endpoint = '%s', + s3.region = 'us-east-1', + s3.access.key = '%s', + s3.secret.key = '%s', + s3.path.style.access = 'true', + catalog.rest.sigv4_enabled = 'true', + catalog.rest.signing_region = 'us-east-1', + catalog.rest.signing_name = 's3' +);`, sourceName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey) + + runRisingWaveSQL(t, env.postgresSidecar, createSourceSql) + + t.Log(">>> Selecting from source to verify UPSERT...") + // Should see (1, 'Charles') and NOT (2, 'Dave') + verifyQuery(t, env, sourceName, "1 | Charles") + verifyQueryAbsence(t, env, sourceName, "2 | Dave") + }) +} + +func verifyQuery(t *testing.T, env *TestEnvironment, sourceName string, expectedSubstrings ...string) { + t.Helper() + var output string + for i := 0; i < 15; i++ { + output = runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("SELECT * FROM %s ORDER BY id;", sourceName)) + allFound := true + for _, s := range expectedSubstrings { + if !strings.Contains(output, s) { + allFound = false + break + } + } + if allFound { + return + } + time.Sleep(2 * time.Second) + } + t.Fatalf("Failed to find expected data %v in output:\n%s", expectedSubstrings, output) +} + +func verifyQueryAbsence(t *testing.T, env *TestEnvironment, sourceName string, unexpectedSubstrings ...string) { + t.Helper() + var output string + for i := 0; i < 15; i++ { + output = runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("SELECT * FROM %s ORDER BY id;", sourceName)) + noneFound := true + for _, s := range unexpectedSubstrings { + if strings.Contains(output, s) { + noneFound = false + break + } + } + if noneFound { + return + } + time.Sleep(2 * time.Second) + } + t.Fatalf("Found unexpected data %v in output:\n%s", unexpectedSubstrings, output) +} diff --git a/test/s3tables/lakekeeper/lakekeeper_test.go b/test/s3tables/lakekeeper/lakekeeper_test.go new file mode 100644 index 000000000..67cab1139 --- /dev/null +++ b/test/s3tables/lakekeeper/lakekeeper_test.go @@ -0,0 +1,341 @@ +package lakekeeper + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/s3tables/testutil" +) + +type TestEnvironment struct { + seaweedDir string + weedBinary string + dataDir string + bindIP string + s3Port int + s3GrpcPort int + masterPort int + masterGrpcPort int + filerPort int + filerGrpcPort int + volumePort int + volumeGrpcPort int + weedProcess *exec.Cmd + weedCancel context.CancelFunc + accessKey string + secretKey string +} + +func TestLakekeeperIntegration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + if !testutil.HasDocker() { + t.Skip("Docker not available, skipping Lakekeeper integration test") + } + + env := NewTestEnvironment(t) + defer env.Cleanup(t) + + fmt.Printf(">>> Starting SeaweedFS with Lakekeeper configuration...\n") + env.StartSeaweedFS(t) + fmt.Printf(">>> SeaweedFS started.\n") + + // Run python script in docker to test STS and S3 operations + runLakekeeperRepro(t, env) +} + +func NewTestEnvironment(t *testing.T) *TestEnvironment { + t.Helper() + + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Failed to get working directory: %v", err) + } + + seaweedDir := wd + for i := 0; i < 6; i++ { + if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil { + break + } + seaweedDir = filepath.Dir(seaweedDir) + } + + weedBinary := filepath.Join(seaweedDir, "weed", "weed") + if _, err := os.Stat(weedBinary); err != nil { + weedBinary = "weed" + if _, err := exec.LookPath(weedBinary); err != nil { + t.Skip("weed binary not found, skipping integration test") + } + } + + dataDir, err := os.MkdirTemp("", "seaweed-lakekeeper-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + + bindIP := testutil.FindBindIP() + + masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master") + volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume") + filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer") + s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3") + + return &TestEnvironment{ + seaweedDir: seaweedDir, + weedBinary: weedBinary, + dataDir: dataDir, + bindIP: bindIP, + s3Port: s3Port, + s3GrpcPort: s3GrpcPort, + masterPort: masterPort, + masterGrpcPort: masterGrpcPort, + filerPort: filerPort, + filerGrpcPort: filerGrpcPort, + volumePort: volumePort, + volumeGrpcPort: volumeGrpcPort, + accessKey: "admin", + secretKey: "admin", + } +} + +func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { + t.Helper() + + iamConfigPath := filepath.Join(env.dataDir, "iam.json") + // Note: signingKey must be base64 encoded for []byte JSON unmarshaling + iamConfig := fmt.Sprintf(`{ + "identities": [ + { + "name": "admin", + "credentials": [ + { + "accessKey": "%s", + "secretKey": "%s" + } + ], + "actions": ["Admin", "Read", "List", "Tagging", "Write"] + } + ], + "sts": { + "tokenDuration": "12h", + "maxSessionLength": "24h", + "issuer": "seaweedfs-sts", + "signingKey": "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz" + }, + "roles": [ + { + "roleName": "LakekeeperVendedRole", + "roleArn": "arn:aws:iam::000000000000:role/LakekeeperVendedRole", + "trustPolicy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "sts:AssumeRole" + } + ] + }, + "attachedPolicies": ["FullAccess"] + } + ], + "policies": [ + { + "name": "FullAccess", + "document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "*", + "Resource": "*" + } + ] + } + } + ] +}`, env.accessKey, env.secretKey) + + if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil { + t.Fatalf("Failed to create IAM config: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + env.weedCancel = cancel + + // Start weed mini with both S3 config (standard IAM) and IAM config (advanced IAM/STS) + cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini", + "-master.port", fmt.Sprintf("%d", env.masterPort), + "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), + "-volume.port", fmt.Sprintf("%d", env.volumePort), + "-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort), + "-filer.port", fmt.Sprintf("%d", env.filerPort), + "-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort), + "-s3.port", fmt.Sprintf("%d", env.s3Port), + "-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort), + "-s3.config", iamConfigPath, + "-s3.iam.config", iamConfigPath, + "-s3.iam.readOnly=false", + "-ip", env.bindIP, + "-ip.bind", "0.0.0.0", + "-dir", env.dataDir, + ) + cmd.Dir = env.dataDir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Start(); err != nil { + t.Fatalf("Failed to start SeaweedFS: %v", err) + } + env.weedProcess = cmd + + if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) { + t.Fatalf("S3 API failed to become ready") + } +} + +func (env *TestEnvironment) Cleanup(t *testing.T) { + t.Helper() + if env.weedCancel != nil { + env.weedCancel() + } + if env.weedProcess != nil { + time.Sleep(1 * time.Second) + _ = env.weedProcess.Wait() + } + if env.dataDir != "" { + _ = os.RemoveAll(env.dataDir) + } +} + +func runLakekeeperRepro(t *testing.T, env *TestEnvironment) { + t.Helper() + + scriptContent := fmt.Sprintf(` +import boto3 +import botocore.config +import botocore +from botocore.exceptions import ClientError +import os +import sys +import time +import logging + +# Enable botocore debug logging to see signature calculation +logging.basicConfig(level=logging.DEBUG) +botocore.session.get_session().set_debug_logger() + +print("Starting Lakekeeper repro test...") + +endpoint_url = "http://host.docker.internal:%d" +access_key = "%s" +secret_key = "%s" +region = "us-east-1" + +print(f"Connecting to {endpoint_url}") + +try: + config = botocore.config.Config( + retries={'max_attempts': 3} + ) + sts = boto3.client( + 'sts', + endpoint_url=endpoint_url, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region, + config=config + ) + + role_arn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole" + session_name = "lakekeeper-session" + + print(f"Calling AssumeRole on {role_arn} with POST body...") + + # Standard boto3 call sends parameters in POST body + response = sts.assume_role( + RoleArn=role_arn, + RoleSessionName=session_name + ) + + creds = response['Credentials'] + access_key_id = creds['AccessKeyId'] + secret_access_key = creds['SecretAccessKey'] + session_token = creds['SessionToken'] + + print(f"Success! Got credentials with prefix: {access_key_id[:4]}") + + if not access_key_id.startswith("ASIA"): + print(f"FAILED: Expected ASIA prefix, got {access_key_id}") + sys.exit(1) + + print("Verifying S3 operations with vended credentials...") + s3 = boto3.client( + 's3', + endpoint_url=endpoint_url, + aws_access_key_id=access_key_id, + aws_secret_access_key=secret_access_key, + aws_session_token=session_token, + region_name=region, + config=config + ) + + bucket = "lakekeeper-vended-bucket" + print(f"Creating bucket {bucket}...") + s3.create_bucket(Bucket=bucket) + + print("Listing buckets...") + response = s3.list_buckets() + buckets = [b['Name'] for b in response['Buckets']] + print(f"Found buckets: {buckets}") + + if bucket not in buckets: + print(f"FAILED: Bucket {bucket} not found in list") + sys.exit(1) + + print("SUCCESS: Lakekeeper flow verified!") + sys.exit(0) + +except Exception as e: + print(f"FAILED: {e}") + # Print more details if it is a ClientError + if hasattr(e, 'response'): + print(f"Response: {e.response}") + sys.exit(1) +`, env.s3Port, env.accessKey, env.secretKey) + + scriptPath := filepath.Join(env.dataDir, "lakekeeper_repro.py") + if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil { + t.Fatalf("Failed to write python script: %v", err) + } + + containerName := "seaweed-lakekeeper-client-" + fmt.Sprintf("%d", time.Now().UnixNano()) + + // Create a context with timeout for the docker run command + dockerCtx, dockerCancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer dockerCancel() + + cmd := exec.CommandContext(dockerCtx, "docker", "run", "--rm", + "--name", containerName, + "--add-host", "host.docker.internal:host-gateway", + "-v", fmt.Sprintf("%s:/work", env.dataDir), + "python:3", + "/bin/bash", "-c", "pip install boto3 && python /work/lakekeeper_repro.py", + ) + + output, err := cmd.CombinedOutput() + if err != nil { + if dockerCtx.Err() == context.DeadlineExceeded { + t.Fatalf("Lakekeeper repro client timed out after 5 minutes\nOutput:\n%s", string(output)) + } + t.Fatalf("Lakekeeper repro client failed: %v\nOutput:\n%s", err, string(output)) + } + t.Logf("Lakekeeper repro client output:\n%s", string(output)) +} diff --git a/test/s3tables/sts_integration/sts_integration_test.go b/test/s3tables/sts_integration/sts_integration_test.go new file mode 100644 index 000000000..33b78e06f --- /dev/null +++ b/test/s3tables/sts_integration/sts_integration_test.go @@ -0,0 +1,280 @@ +package sts_integration + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/s3tables/testutil" +) + +// TestEnvironment mirrors the one in trino_catalog_test.go but simplified +type TestEnvironment struct { + seaweedDir string + weedBinary string + dataDir string + bindIP string + s3Port int + s3GrpcPort int + masterPort int + masterGrpcPort int + filerPort int + filerGrpcPort int + volumePort int + volumeGrpcPort int + weedProcess *exec.Cmd + weedCancel context.CancelFunc + dockerAvailable bool + accessKey string + secretKey string +} + +func TestSTSIntegration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + env := NewTestEnvironment(t) + defer env.Cleanup(t) + + if !env.dockerAvailable { + t.Skip("Docker not available, skipping STS integration test") + } + + fmt.Printf(">>> Starting SeaweedFS...\n") + env.StartSeaweedFS(t) + fmt.Printf(">>> SeaweedFS started.\n") + + // Run python script in docker to test STS + runPythonSTSClient(t, env) +} + +func NewTestEnvironment(t *testing.T) *TestEnvironment { + t.Helper() + + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Failed to get working directory: %v", err) + } + + seaweedDir := wd + for i := 0; i < 6; i++ { + if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil { + break + } + seaweedDir = filepath.Dir(seaweedDir) + } + + weedBinary := filepath.Join(seaweedDir, "weed", "weed") + info, err := os.Stat(weedBinary) + if err != nil || info.IsDir() { + weedBinary = "weed" + if _, err := exec.LookPath(weedBinary); err != nil { + t.Skip("weed binary not found, skipping integration test") + } + } + + if !testutil.HasDocker() { + t.Skip("Docker not available, skipping integration test") + } + + // Create a unique temporary directory for this test run + dataDir, err := os.MkdirTemp("", "seaweed-sts-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + // The Cleanup method will remove this directory, so no need for defer here. + + bindIP := testutil.FindBindIP() + + masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master") + volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume") + filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer") + s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3") // Changed to use testutil.MustFreePortPair + + return &TestEnvironment{ + seaweedDir: seaweedDir, + weedBinary: weedBinary, + dataDir: dataDir, + bindIP: bindIP, + s3Port: s3Port, + s3GrpcPort: s3GrpcPort, + masterPort: masterPort, + masterGrpcPort: masterGrpcPort, + filerPort: filerPort, + filerGrpcPort: filerGrpcPort, + volumePort: volumePort, + volumeGrpcPort: volumeGrpcPort, + dockerAvailable: testutil.HasDocker(), + accessKey: "admin", // Matching default in testutil.WriteIAMConfig + secretKey: "admin", + } +} + +func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { + t.Helper() + + // Create IAM config file + iamConfigPath, err := testutil.WriteIAMConfig(env.dataDir, env.accessKey, env.secretKey) + if err != nil { + t.Fatalf("Failed to create IAM config: %v", err) + } + + // Create empty security.toml + securityToml := filepath.Join(env.dataDir, "security.toml") + if err := os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644); err != nil { + t.Fatalf("Failed to create security.toml: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + env.weedCancel = cancel + + cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + "-master.port", fmt.Sprintf("%d", env.masterPort), + "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), + "-volume.port", fmt.Sprintf("%d", env.volumePort), + "-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort), + "-filer.port", fmt.Sprintf("%d", env.filerPort), + "-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort), + "-s3.port", fmt.Sprintf("%d", env.s3Port), + "-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort), + "-s3.config", iamConfigPath, + "-ip", env.bindIP, + "-ip.bind", "0.0.0.0", + "-dir", env.dataDir, + ) + cmd.Dir = env.dataDir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Start(); err != nil { + t.Fatalf("Failed to start SeaweedFS: %v", err) + } + env.weedProcess = cmd + + // Wait for S3 API to be ready + if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) { + t.Fatalf("S3 API failed to become ready") + } +} + +func (env *TestEnvironment) Start(t *testing.T) { + if !testutil.HasDocker() { + t.Skip("Docker not available") + } +} + +func (env *TestEnvironment) Cleanup(t *testing.T) { + t.Helper() + if env.weedCancel != nil { + env.weedCancel() + } + if env.weedProcess != nil { + time.Sleep(1 * time.Second) + _ = env.weedProcess.Wait() + } + if env.dataDir != "" { + _ = os.RemoveAll(env.dataDir) + } +} + +func runPythonSTSClient(t *testing.T, env *TestEnvironment) { + t.Helper() + + // Write python script to temp dir + scriptContent := fmt.Sprintf(` +import boto3 +import botocore.config +from botocore.exceptions import ClientError +import os +import sys + +print("Starting STS test...") + +endpoint_url = "http://host.docker.internal:%d" +access_key = "%s" +secret_key = "%s" +region = "us-east-1" + +print(f"Connecting to {endpoint_url} with key {access_key}") + +try: + config = botocore.config.Config( + retries={'max_attempts': 0} + ) + sts = boto3.client( + 'sts', + endpoint_url=endpoint_url, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region, + config=config + ) + + role_arn = "arn:aws:iam::000000000000:role/test-role" + session_name = "test-session" + + print(f"Calling AssumeRole on {role_arn}") + + # This call typically sends parameters in POST body by default in boto3 + response = sts.assume_role( + RoleArn=role_arn, + RoleSessionName=session_name + ) + + print("Success! Got credentials:") + print(response['Credentials']) + +except ClientError as e: + # Print available keys for debugging if needed + # print(e.response.keys()) + + response_meta = e.response.get('ResponseMetadata', {}) + http_code = response_meta.get('HTTPStatusCode') + + error_data = e.response.get('Error', {}) + error_code = error_data.get('Code', 'Unknown') + + print(f"Got error: {http_code} {error_code}") + + # We expect 503 ServiceUnavailable because stsHandlers is nil in weed mini + # This confirms the request was routed to STS handler logic (UnifiedPostHandler) + # instead of IAM handler (which would return 403 AccessDenied or 501 NotImplemented) + if http_code == 503: + print("SUCCESS: Got expected 503 Service Unavailable (STS not configured)") + sys.exit(0) + + print(f"FAILED: Unexpected error {e}") + sys.exit(1) +except Exception as e: + print(f"FAILED: {e}") + sys.exit(1) +`, env.s3Port, env.accessKey, env.secretKey) + + scriptPath := filepath.Join(env.dataDir, "sts_test.py") + if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil { + t.Fatalf("Failed to write python script: %v", err) + } + + containerName := "seaweed-sts-client-" + fmt.Sprintf("%d", time.Now().UnixNano()) + + cmd := exec.Command("docker", "run", "--rm", + "--name", containerName, + "--add-host", "host.docker.internal:host-gateway", + "-v", fmt.Sprintf("%s:/work", env.dataDir), + "python:3", + "/bin/bash", "-c", "pip install boto3 && python /work/sts_test.py", + ) + + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Python STS client failed: %v\nOutput:\n%s", err, string(output)) + } + t.Logf("Python STS client output:\n%s", string(output)) +} + +// Helpers copied from trino_catalog_test.go diff --git a/test/s3tables/testutil/docker.go b/test/s3tables/testutil/docker.go new file mode 100644 index 000000000..eff2b2ac1 --- /dev/null +++ b/test/s3tables/testutil/docker.go @@ -0,0 +1,66 @@ +package testutil + +import ( + "context" + "net" + "net/http" + "os/exec" + "testing" + "time" +) + +func HasDocker() bool { + cmd := exec.Command("docker", "version") + return cmd.Run() == nil +} + +func MustFreePortPair(t *testing.T, name string) (int, int) { + httpPort, grpcPort, err := findAvailablePortPair() + if err != nil { + t.Fatalf("Failed to get free port pair for %s: %v", name, err) + } + return httpPort, grpcPort +} + +func findAvailablePortPair() (int, int, error) { + httpPort, err := GetFreePort() + if err != nil { + return 0, 0, err + } + grpcPort, err := GetFreePort() + if err != nil { + return 0, 0, err + } + return httpPort, grpcPort, nil +} + +func GetFreePort() (int, error) { + listener, err := net.Listen("tcp", "0.0.0.0:0") + if err != nil { + return 0, err + } + defer listener.Close() + return listener.Addr().(*net.TCPAddr).Port, nil +} + +func WaitForService(url string, timeout time.Duration) bool { + client := &http.Client{Timeout: 2 * time.Second} + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return false + case <-ticker.C: + resp, err := client.Get(url) + if err == nil { + resp.Body.Close() + return true + } + } + } +} diff --git a/test/volume_server/DEV_PLAN.md b/test/volume_server/DEV_PLAN.md new file mode 100644 index 000000000..73f81e758 --- /dev/null +++ b/test/volume_server/DEV_PLAN.md @@ -0,0 +1,1129 @@ +# Volume Server Integration Test Dev Plan (Go) + +## Goal +Create a Go integration test suite under `test/volume_server` that validates **drop-in behavior parity** for the Volume Server HTTP and gRPC APIs, so a Rust rewrite can be verified against the current Go behavior. + +## Hard Requirements +- Tests live under `test/volume_server`. +- Tests are written in Go. +- HTTP + gRPC APIs are both covered. +- Coverage targets execution-path parity (happy path + edge/failure/state variants), not only API reachability. +- During implementation, commit each logic change separately for reviewability. + +## Ground Truth (API Surface) +- HTTP handlers: + - `weed/server/volume_server.go` + - `weed/server/volume_server_handlers.go` + - `weed/server/volume_server_handlers_read.go` + - `weed/server/volume_server_handlers_write.go` + - `weed/server/volume_server_handlers_admin.go` + - `weed/server/common.go` (path parsing and range handling) +- gRPC service and handlers: + - `weed/pb/volume_server.proto` + - `weed/server/volume_grpc_*.go` + +## Proposed Test Directory Tree + +```text +test/volume_server/ + DEV_PLAN.md + README.md + Makefile + framework/ + cluster.go # test cluster lifecycle, ports, process control + volume_fixture.go # test data provisioning and seed volumes + http_client.go # request helpers, auth helpers, assertions + grpc_client.go # grpc dial + common call wrappers + fault_injection.go # kill/restart peers, network/error simulation hooks + matrix/ + config_profiles.go # runtime matrix profiles (jwt, readMode, public port split, etc.) + http/ + admin_test.go # /status, /healthz, /ui, static resources + read_test.go # GET/HEAD read behaviors + variations + write_test.go # POST/PUT write behaviors + variations + delete_test.go # DELETE behaviors + variations + cors_and_options_test.go + throttling_test.go # upload/download in-flight limit paths + auth_test.go # jwt/no-jwt/bad-jwt/fid mismatch + grpc/ + admin_lifecycle_test.go + vacuum_test.go + data_rw_test.go + copy_sync_test.go + tail_test.go + erasure_coding_test.go + tiering_test.go + remote_fetch_test.go + scrub_test.go + query_test.go + health_state_test.go + compatibility/ + golden_behavior_test.go # protocol-level parity checks for selected canonical flows +``` + +## Environment/Matrix Profiles (Execution Dimensions) +Each API should be exercised across the smallest set of profiles that still covers behavior divergence: + +- `P1`: Single volume server, no JWT, `readMode=proxy`, single HTTP port. +- `P2`: Public/admin port split (`port.public != port`) to verify public read-only behavior. +- `P3`: JWT enabled (`jwt.signing.key` and `jwt.signing.read.key`) with valid/invalid/missing tokens. +- `P4`: Replicated volume layout (>=2 volume servers) to cover proxy/redirect/replicate paths. +- `P5`: Erasure coding volumes present. +- `P6`: Remote tier backend configured. +- `P7`: Maintenance mode enabled. +- `P8`: Upload/download throttling limits enabled. + +## HTTP API Test Case Tree + +### 1. Admin and service endpoints +- [ ] `GET /status` + - [ ] baseline payload fields (`Version`, `DiskStatuses`, `Volumes`) + - [ ] response headers (`Server`, request ID) +- [ ] `GET /healthz` + - [ ] healthy -> `200` + - [ ] server stopping -> `503` + - [ ] heartbeat disabled -> `503` +- [ ] `GET /ui/index.html` + - [ ] enabled path renders page + - [ ] disabled/secured behavior matches current config +- [ ] static assets (`/favicon.ico`, `/seaweedfsstatic/*`) reachable + +### 2. Data read endpoints (`GET`/`HEAD` on `/...`) +- [ ] URL shape variants + - [ ] `/{vid},{fid}` + - [ ] `/{vid}/{fid}` + - [ ] `/{vid}/{fid}/{filename}` + - [ ] malformed vid/fid -> `400` +- [ ] auth variants + - [ ] no JWT required + - [ ] read JWT missing/invalid/fid-mismatch -> `401` + - [ ] valid JWT -> success +- [ ] locality/read mode variants + - [ ] local volume read success + - [ ] missing local volume with `readMode=local` -> `404` + - [ ] missing local volume with `readMode=proxy` -> proxied response + - [ ] missing local volume with `readMode=redirect` -> `301` + - [ ] proxied-loop guard (`proxied=true`) behavior +- [ ] object state variants + - [ ] not found/deleted -> `404` + - [ ] cookie mismatch -> `404` + - [ ] internal read failure -> `500` + - [ ] `readDeleted=true` behavior +- [ ] conditional headers + - [ ] `If-Modified-Since` -> `304` + - [ ] `If-None-Match` -> `304` + - [ ] normal ETag emission +- [ ] range handling + - [ ] full read (no `Range`) + - [ ] single range -> `206`, `Content-Range` + - [ ] multi-range -> `206`, multipart body + - [ ] invalid range -> `416` + - [ ] oversized-range-sum behavior parity +- [ ] content transformations + - [ ] compressed data + `Accept-Encoding=gzip` + - [ ] compressed data without accepted encoding (decompress path) + - [ ] image resize (`width`,`height`,`mode`) + - [ ] image crop (`crop_*`) +- [ ] chunk manifest behavior + - [ ] manifest auto-expansion path + - [ ] `cm=false` bypass path +- [ ] response header passthrough queries + - [ ] `response-*` overrides and `dl` content-disposition behavior +- [ ] `HEAD` parity + - [ ] headers same as `GET` minus body + - [ ] content-length behavior parity + +### 3. Data write endpoints (`PUT`/`POST` on `/...`) +- [ ] URL shape and parse validation +- [ ] write JWT variants (missing/invalid/mismatch) +- [ ] payload variants + - [ ] standard upload success -> `201` + - [ ] unchanged write -> `204` with ETag + - [ ] oversize file rejected (file-size limit) + - [ ] malformed multipart/form data -> `400` +- [ ] metadata variants (name/mime/pairs/md5 headers) +- [ ] replication path + - [ ] `type=replicate` bypasses upload throttling + - [ ] replication write failure behavior +- [ ] throttling paths + - [ ] limit disabled + - [ ] over limit wait then proceed + - [ ] timeout -> `429` + - [ ] canceled request -> `499` + +### 4. Data delete endpoint (`DELETE` on `/...`) +- [ ] normal volume delete success (`202` + size) +- [ ] non-existing needle (`404` size=0) +- [ ] EC volume delete path +- [ ] cookie mismatch -> `400` +- [ ] chunk manifest delete (children first) + - [ ] chunk delete success + - [ ] chunk delete failure -> `500` +- [ ] `ts` override behavior +- [ ] auth variants as in write path + +### 5. Method/CORS behavior +- [ ] private port `OPTIONS` -> allow `PUT,POST,GET,DELETE,OPTIONS` +- [ ] public port `OPTIONS` -> allow `GET,OPTIONS` +- [ ] CORS origin headers on requests with `Origin` +- [ ] unsupported method behavior parity (private and public ports) + +### 6. Concurrency-limit and replica fallback behavior +- [ ] download over-limit + replica available -> proxy/redirect fallback +- [ ] download over-limit + no replica -> wait/timeout/cancel outcomes +- [ ] upload/download inflight counters update and release (no leaks) + +## gRPC API Test Case Tree + +For each RPC below: cover baseline success, validation/argument errors, state preconditions (maintenance, missing volume), and stream interruption where applicable. + +### A. Admin/Lifecycle +- [ ] `DeleteCollection` + - [ ] existing collection + - [ ] non-existing collection idempotence/error parity +- [ ] `AllocateVolume` + - [ ] success + - [ ] maintenance mode reject + - [ ] duplicate/invalid allocation parity +- [ ] `VolumeMount` + - [ ] success + - [ ] missing/not-mountable volume +- [ ] `VolumeUnmount` + - [ ] success + - [ ] missing/not-mounted volume +- [ ] `VolumeDelete` + - [ ] `only_empty=true` and `false` + - [ ] maintenance mode reject +- [ ] `VolumeConfigure` + - [ ] success + - [ ] invalid replication string -> `resp.Error` path + - [ ] unmount failure path + - [ ] configure failure + remount rollback path + - [ ] mount failure path +- [ ] `VolumeMarkReadonly` + - [ ] success with `persist=false/true` + - [ ] volume not found + - [ ] notify-master failure (pre/post local transition) +- [ ] `VolumeMarkWritable` + - [ ] success + - [ ] volume not found + - [ ] notify-master failure +- [ ] `VolumeStatus` + - [ ] success + - [ ] volume not found + - [ ] data backend missing +- [ ] `VolumeServerStatus` + - [ ] payload completeness (`State`, `MemoryStatus`, disk statuses) +- [ ] `VolumeServerLeave` + - [ ] heartbeat stopped effect +- [ ] `GetState` +- [ ] `SetState` + - [ ] state transition success + - [ ] invalid update error path +- [ ] `Ping` + - [ ] target type: filer / volume / master + - [ ] unreachable target error wrapping + - [ ] unknown target type behavior parity + +### B. Vacuum / compaction +- [ ] `VacuumVolumeCheck` + - [ ] success + garbage ratio + - [ ] missing volume/error path +- [ ] `VacuumVolumeCompact` (stream) + - [ ] progress events emitted + - [ ] maintenance mode reject + - [ ] compact failure + - [ ] client stream receive interruption +- [ ] `VacuumVolumeCommit` + - [ ] success (readonly + size fields) + - [ ] maintenance mode reject + - [ ] commit failure +- [ ] `VacuumVolumeCleanup` + - [ ] success + - [ ] maintenance mode reject + - [ ] cleanup failure + +### C. Data read/write +- [ ] `ReadNeedleBlob` + - [ ] success + - [ ] missing volume + - [ ] invalid offset/size path +- [ ] `ReadNeedleMeta` + - [ ] success + - [ ] missing volume + - [ ] EC-only volume unsupported path + - [ ] read metadata failure +- [ ] `WriteNeedleBlob` + - [ ] success + - [ ] maintenance mode reject + - [ ] missing volume + - [ ] write failure +- [ ] `VolumeNeedleStatus` + - [ ] normal volume success + - [ ] EC volume success + - [ ] volume missing + - [ ] needle missing/read error + +### D. Batch and scan +- [ ] `BatchDelete` + - [ ] `skip_cookie_check=true/false` + - [ ] invalid fid parse + - [ ] not found + - [ ] cookie mismatch path + - [ ] chunk manifest reject path (`406` in result) + - [ ] regular and EC delete paths +- [ ] `ReadAllNeedles` (stream) + - [ ] multiple volumes success + - [ ] one missing volume abort behavior + +### E. Copy/sync/replication streams +- [ ] `VolumeSyncStatus` + - [ ] success + - [ ] missing volume +- [ ] `VolumeIncrementalCopy` (stream) + - [ ] data streamed from `since_ns` + - [ ] `isLastOne` no-data path + - [ ] missing volume/error path +- [ ] `VolumeCopy` (stream) + - [ ] full copy success (`.dat/.idx/.vif`) + mount + - [ ] existing destination volume delete-before-copy path + - [ ] source unavailable / read status failure + - [ ] no free location + - [ ] remote-dat-file branch + - [ ] copy integrity mismatch failures + - [ ] final progress/append timestamp behavior +- [ ] `ReadVolumeFileStatus` + - [ ] success field validation + - [ ] missing volume +- [ ] `CopyFile` (stream) + - [ ] normal volume path + - [ ] EC volume path + - [ ] compaction revision mismatch + - [ ] missing source file with ignore flag true/false + - [ ] zero-byte and stop-offset edge cases +- [ ] `ReceiveFile` (client stream) + - [ ] happy path regular volume + - [ ] happy path EC file target + - [ ] info-first protocol violation + - [ ] unknown message type + - [ ] maintenance mode reject + - [ ] write/create failure cleanup behavior + +### F. Tailing +- [ ] `VolumeTailSender` (stream) + - [ ] volume not found + - [ ] heartbeat chunks when no updates + - [ ] idle-timeout drain completion + - [ ] large needle chunking behavior +- [ ] `VolumeTailReceiver` + - [ ] success applies streamed writes + - [ ] destination volume missing + - [ ] source stream/connect failure + +### G. Erasure coding +- [ ] `VolumeEcShardsGenerate` + - [ ] success default config + - [ ] success with existing `.vif` EC config + - [ ] maintenance mode reject + - [ ] collection mismatch + - [ ] generate/write cleanup on failure +- [ ] `VolumeEcShardsRebuild` + - [ ] rebuild missing shards success + - [ ] no shards found path + - [ ] rebuild failures +- [ ] `VolumeEcShardsCopy` + - [ ] shard copy success + - [ ] `copy_ecx`/`copy_ecj`/`copy_vif` toggles + - [ ] explicit `disk_id` valid/invalid + - [ ] no-space/source-copy failure +- [ ] `VolumeEcShardsDelete` + - [ ] delete selected shard ids + - [ ] delete-last-shard cleanup (`.ecx/.ecj` + optional `.vif`) + - [ ] missing shard no-op parity +- [ ] `VolumeEcShardsMount` + - [ ] multi-shard success + - [ ] per-shard failure abort behavior +- [ ] `VolumeEcShardsUnmount` + - [ ] multi-shard success + - [ ] per-shard failure abort behavior +- [ ] `VolumeEcShardRead` (stream) + - [ ] success + - [ ] not-found volume/shard + - [ ] deleted file key returns `IsDeleted` + - [ ] chunked streaming for large reads +- [ ] `VolumeEcBlobDelete` + - [ ] delete existing blob + - [ ] already deleted idempotence + - [ ] locate failure +- [ ] `VolumeEcShardsToVolume` + - [ ] success path from EC -> normal + - [ ] missing EC volume/shard + - [ ] invalid data-shard config + - [ ] no-live-entries failed-precondition path + - [ ] write dat/idx failures +- [ ] `VolumeEcShardsInfo` + - [ ] success counts (including deleted) + - [ ] missing EC volume + - [ ] walk-index failure + +### H. Tiering and remote +- [ ] `VolumeTierMoveDatToRemote` (stream) + - [ ] success with progress events + - [ ] maintenance mode reject + - [ ] volume missing / collection mismatch + - [ ] destination backend missing + - [ ] destination exists already + - [ ] keep-local true/false branches +- [ ] `VolumeTierMoveDatFromRemote` (stream) + - [ ] success with progress events + - [ ] volume missing / collection mismatch + - [ ] already-local path + - [ ] backend missing/download failure + - [ ] keep-remote true/false branches +- [ ] `FetchAndWriteNeedle` + - [ ] success without replicas + - [ ] success with replica fanout + - [ ] maintenance mode reject + - [ ] missing volume + - [ ] remote client/read failure + - [ ] local write failure + - [ ] one replica write failure behavior + +### I. Query and scrub +- [ ] `Query` (stream) + - [ ] JSON input selection/filter success + - [ ] malformed fid parse failure + - [ ] read/cookie mismatch failure + - [ ] CSV-input current behavior parity +- [ ] `ScrubVolume` + - [ ] auto-select all volumes when request empty + - [ ] mode `INDEX` + - [ ] mode `LOCAL` (not-implemented detail reporting) + - [ ] mode `FULL` (not-implemented detail reporting) + - [ ] unsupported mode error +- [ ] `ScrubEcVolume` + - [ ] auto-select all EC volumes when request empty + - [ ] mode `INDEX` + - [ ] mode `LOCAL` + - [ ] mode `FULL` (not-implemented detail reporting) + - [ ] unsupported mode error + +## Phased Implementation Plan (Tracking) + +### Phase 0: Harness and scaffolding +- [x] Create `framework/` cluster bootstrap and teardown +- [x] Add profile-based environment builder (P1..P8) +- [x] Add common assertion helpers (HTTP and gRPC) +- [x] Add `README.md` with run instructions +- [x] Add `Makefile` targets (`test-volume-server`, profile filters) + +### Phase 1: HTTP parity suites +- [ ] Admin/status/health/UI/static +- [ ] Read path variants and headers/range/transforms +- [ ] Write/delete/auth/throttling/public-port behavior + +### Phase 2: gRPC parity suites (core) +- [x] Admin/lifecycle/state/ping +- [ ] Vacuum + batch + data rw +- [ ] Copy/sync/tail + +### Phase 3: gRPC parity suites (advanced) +- [ ] Erasure coding family +- [ ] Tiering + remote fetch +- [ ] Query + scrub + +### Phase 4: Compatibility hardening +- [ ] Golden behavior assertions for canonical flows +- [ ] Flake reduction and deterministic retries/timeouts +- [ ] CI runtime tuning and sharding + +## Commit Strategy (for implementation) +Use one commit per logical change set. Suggested sequence: + +1. `test(volume_server): add integration framework and cluster lifecycle` +2. `test(volume_server): add config profile matrix and test utilities` +3. `test(volume_server/http): add admin and health endpoint coverage` +4. `test(volume_server/http): add read-path matrix coverage` +5. `test(volume_server/http): add write/delete/auth/throttling coverage` +6. `test(volume_server/grpc): add admin lifecycle and state/ping coverage` +7. `test(volume_server/grpc): add vacuum batch and data rw coverage` +8. `test(volume_server/grpc): add copy sync and tail coverage` +9. `test(volume_server/grpc): add erasure coding coverage` +10. `test(volume_server/grpc): add tiering and remote fetch coverage` +11. `test(volume_server/grpc): add query and scrub coverage` +12. `test(volume_server): add compatibility golden scenarios and docs` + +## Progress Log +Update this section during implementation: + +- Date: 2026-02-12 +- Change: Added initial dev plan and project scaffold. +- APIs covered: Planning only. +- Profiles covered: Planning only. +- Gaps introduced/remaining: Implementation not started yet. +- Commit: `21c10a9ec` + +- Date: 2026-02-12 +- Change: Added integration harness, profile matrix, and auto-build support for missing `weed` binary. Master now starts with `-peers=none` and low `-volumeSizeLimitMB`. +- APIs covered: Harness only. +- Profiles covered: P1, P2, P3, P8 definitions in place. +- Gaps introduced/remaining: Full API case matrix still pending. +- Commit: `5e9c437e0`, `1be296139` + +- Date: 2026-02-12 +- Change: Added HTTP integration coverage for admin endpoints, method options, and upload/read/range/head/delete roundtrip. +- APIs covered: `/status`, `/healthz`, `/ui/index.html`, `OPTIONS /`, `POST/GET/HEAD/DELETE /{fid}`. +- Profiles covered: P1, P2. +- Gaps introduced/remaining: Remaining HTTP branch variants (JWT/proxy/redirect/throttling/etc.) still pending. +- Commit: `038e9161e`, `1df1e3812` + +- Date: 2026-02-12 +- Change: Added gRPC integration coverage for state/status/ping and admin lifecycle/maintenance checks. +- APIs covered: `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `AllocateVolume`, `VolumeStatus`, `VolumeMount`, `VolumeUnmount`, `VolumeDelete`. +- Profiles covered: P1. +- Gaps introduced/remaining: Remaining gRPC methods and advanced branches still pending. +- Commit: `9a9b8c500`, `3c562a64c` + +- Date: 2026-02-12 +- Change: Added HTTP cache/range branch tests. +- APIs covered: `GET /{fid}` with `If-None-Match` (`304`) and invalid `Range` (`416`). +- Profiles covered: P1. +- Gaps introduced/remaining: Remaining HTTP auth/proxy/redirect/throttling branches pending. +- Commit: `317346b51` + +- Date: 2026-02-12 +- Change: Added gRPC `BatchDelete` integration checks for invalid fid mapping and maintenance-mode rejection. +- APIs covered: `BatchDelete`. +- Profiles covered: P1. +- Gaps introduced/remaining: Remaining gRPC method families still pending. +- Commit: `7a8aed127` + +- Date: 2026-02-12 +- Change: Added gRPC integration tests for needle status, configure validation branch, ping volume-target branch, and leave/health interaction. +- APIs covered: `VolumeNeedleStatus`, `VolumeConfigure` (invalid replication response path), `Ping` (`volumeServer` target), `VolumeServerLeave`. +- Profiles covered: P1. +- Gaps introduced/remaining: Still pending large RPC groups (vacuum/copy/tail/ec/tiering/query/scrub). +- Commit: `59a571a10` + +- Date: 2026-02-12 +- Change: Added gRPC vacuum integration coverage for success/missing-volume and maintenance-mode rejection branches. +- APIs covered: `VacuumVolumeCheck`, `VacuumVolumeCompact`, `VacuumVolumeCommit`, `VacuumVolumeCleanup`. +- Profiles covered: P1. +- Gaps introduced/remaining: Copy/sync/tail, EC, tiering, query, scrub, and many HTTP matrix branches still pending. +- Commit: `0f7cc53dd` + +- Date: 2026-02-12 +- Change: Added gRPC data read/write error-path coverage for missing-volume and maintenance-mode branches. +- APIs covered: `ReadNeedleBlob`, `ReadNeedleMeta`, `WriteNeedleBlob`. +- Profiles covered: P1. +- Gaps introduced/remaining: Positive-path blob/meta and stream/copy/tail/EC/tiering/query/scrub families remain. +- Commit: `f83ad41b5` + +- Date: 2026-02-12 +- Change: Added HTTP JWT integration coverage for missing/invalid/valid token behavior across write and read paths. +- APIs covered: HTTP `POST /{fid}` and `GET /{fid}` auth paths with read/write signing keys. +- Profiles covered: P3. +- Gaps introduced/remaining: Remaining HTTP proxy/redirect/throttling branches still pending. +- Commit: `def509acb` + +- Date: 2026-02-12 +- Change: Added gRPC sync/copy family tests for success and missing-volume or maintenance-mode stream error paths. +- APIs covered: `VolumeSyncStatus`, `VolumeIncrementalCopy`, `ReadAllNeedles`, `ReadVolumeFileStatus`, `CopyFile`, `VolumeCopy`, `ReceiveFile`. +- Profiles covered: P1. +- Gaps introduced/remaining: Tail/EC/tiering/query/scrub and positive-path copy/tail flows still pending. +- Commit: `b13642838` + +- Date: 2026-02-12 +- Change: Added gRPC scrub and query integration coverage for supported/unsupported modes and invalid/missing fid paths. +- APIs covered: `ScrubVolume`, `ScrubEcVolume`, `Query`. +- Profiles covered: P1. +- Gaps introduced/remaining: Query success path and broader scrub mode matrix remain pending. +- Commit: `a2ab4cde8` + +- Date: 2026-02-12 +- Change: Added gRPC tail integration coverage for sender heartbeat/EOF behavior and sender/receiver missing-volume errors. +- APIs covered: `VolumeTailSender`, `VolumeTailReceiver`. +- Profiles covered: P1. +- Gaps introduced/remaining: Tail success replication path and large-needle chunking remain pending. +- Commit: `fd582ba58` + +- Date: 2026-02-12 +- Change: Expanded gRPC admin/lifecycle coverage with readonly/writable transitions, collection delete behavior, non-empty delete `only_empty` branch, and ping unknown/unreachable target variants. +- APIs covered: `VolumeMarkReadonly`, `VolumeMarkWritable`, `DeleteCollection`, `VolumeDelete` (`only_empty=true/false`), `Ping` (unknown and unreachable master target). +- Profiles covered: P1. +- Gaps introduced/remaining: Notify-master failure paths for readonly/writable and additional admin error branches still pending. +- Commit: `2e6d577f7`, `a3a2da791`, `9f887f25c`, `724bbe2d9` + +- Date: 2026-02-12 +- Change: Added gRPC positive-path data coverage for blob/meta read/write roundtrip and stream-all-needles payload validation. +- APIs covered: `ReadNeedleBlob`, `ReadNeedleMeta`, `WriteNeedleBlob`, `ReadAllNeedles`. +- Profiles covered: P1. +- Gaps introduced/remaining: Additional blob/meta offset/size corruption branches remain. +- Commit: `21e94d1d2` + +- Date: 2026-02-12 +- Change: Expanded gRPC scrub/query coverage for auto-select volume behavior, local/full scrub detail branches, JSON query success filtering, CSV no-output behavior, and EC auto-select empty result. +- APIs covered: `ScrubVolume` (`INDEX`, `LOCAL`, `FULL`, auto-select), `ScrubEcVolume` (missing-volume + auto-select empty), `Query` (JSON success + CSV no-output + invalid paths). +- Profiles covered: P1. +- Gaps introduced/remaining: EC scrub full/local positive paths need EC fixture setup. +- Commit: `8cdf3589a`, `12150a9a2` + +- Date: 2026-02-12 +- Change: Added gRPC tiering/remote early-branch error coverage. +- APIs covered: `FetchAndWriteNeedle` (maintenance + missing volume), `VolumeTierMoveDatToRemote` (missing volume, collection mismatch, maintenance), `VolumeTierMoveDatFromRemote` (missing volume, collection mismatch, already-local path). +- Profiles covered: P1. +- Gaps introduced/remaining: Tier upload/download success flows with real remote backend remain. +- Commit: `51e6fa749` + +- Date: 2026-02-12 +- Change: Expanded HTTP behavior coverage for split public port semantics, CORS on origin requests, unsupported-method parity, unchanged-write `204`, delete edge branches, and JWT fid-mismatch auth rejection. +- APIs covered: public/admin method divergence (`GET/HEAD/POST/DELETE/PATCH`), CORS headers, write unchanged response path, delete cookie-mismatch/missing-needle paths, JWT fid mismatch for write/read. +- Profiles covered: P1, P2, P3. +- Gaps introduced/remaining: Remaining HTTP proxy/redirect/throttling and transformation branches still pending. +- Commit: `2de39c548`, `9998d19dd`, `ea5d8b7b3` + +- Date: 2026-02-12 +- Change: Expanded tier/remote gRPC variation coverage with invalid remote config and missing destination backend branches. +- APIs covered: `FetchAndWriteNeedle` (invalid `RemoteConf`), `VolumeTierMoveDatToRemote` (destination backend not found). +- Profiles covered: P1. +- Gaps introduced/remaining: Tier upload/download success flows with an actual remote backend and replica fanout behavior remain. +- Commit: `855c84f31` + +- Date: 2026-02-12 +- Change: Expanded copy/receive stream coverage with incremental-copy data/no-data branches and receive-file protocol violation handling. +- APIs covered: `VolumeIncrementalCopy` (stream data + EOF no-data), `CopyFile` (ignore missing source + `stop_offset=0`), `ReceiveFile` (content-before-info and unknown message type response errors). +- Profiles covered: P1. +- Gaps introduced/remaining: Full `VolumeCopy` happy path with a real source volume node remains. +- Commit: `1e99407e1` + +- Date: 2026-02-12 +- Change: Added additional copy/receive branches for compaction mismatch and regular-volume receive-file success with byte-for-byte verification via `CopyFile`. +- APIs covered: `CopyFile` (compaction revision mismatch), `ReceiveFile` (successful regular volume write path). +- Profiles covered: P1. +- Gaps introduced/remaining: EC receive-file success path and cleanup failure branches remain. +- Commit: `4c710463e` + +- Date: 2026-02-12 +- Change: Added HTTP read-path variants and conditional request coverage. +- APIs covered: `GET /{vid}/{fid}`, `GET /{vid}/{fid}/{filename}`, malformed `/{vid}/{fid}` parse error path, `If-Modified-Since` (`304`) behavior. +- Profiles covered: P1. +- Gaps introduced/remaining: Proxy/redirect read mode matrix and image/chunk-manifest transformation branches remain. +- Commit: `1f64ebe1d` + +- Date: 2026-02-12 +- Change: Added HTTP passthrough header and static resource coverage. +- APIs covered: query-based `response-*` header passthrough, `dl=true` content-disposition attachment handling, `/favicon.ico`, `/seaweedfsstatic/seaweed50x50.png`. +- Profiles covered: P1. +- Gaps introduced/remaining: Additional static resource variants and multi-range response formatting checks remain. +- Commit: `f1ad1ec50` + +- Date: 2026-02-12 +- Change: Added gRPC ping branch coverage for unreachable filer target. +- APIs covered: `Ping` (`target_type=filer` unreachable target path). +- Profiles covered: P1. +- Gaps introduced/remaining: Successful ping path for filer/master targets in multi-service integration setup remains. +- Commit: `c6ace0331` + +- Date: 2026-02-12 +- Change: Added initial erasure-coding RPC integration coverage for maintenance-gate, missing-volume, invalid-disk, and no-op behaviors. +- APIs covered: `VolumeEcShardsGenerate`, `VolumeEcShardsRebuild`, `VolumeEcShardsCopy`, `VolumeEcShardsDelete`, `VolumeEcShardsMount`, `VolumeEcShardsUnmount`, `VolumeEcShardRead`, `VolumeEcBlobDelete`, `VolumeEcShardsToVolume`, `VolumeEcShardsInfo`. +- Profiles covered: P1. +- Gaps introduced/remaining: Positive EC data-path flows (generate/copy/mount/read/delete/to-volume/info with actual shard files) still require EC fixture setup. +- Commit: `c7592d118` + +- Date: 2026-02-12 +- Change: Added HTTP multi-range response coverage for multipart `206` behavior. +- APIs covered: `GET /{fid}` with multi-range header (`Range: bytes=0-1,4-5`) and multipart response validation. +- Profiles covered: P1. +- Gaps introduced/remaining: Oversized multi-range sum behavior and deeper range-edge normalization remain. +- Commit: `39c68c679` + +- Date: 2026-02-12 +- Change: Added query no-match parity coverage to lock current stream semantics. +- APIs covered: `Query` JSON filter no-match path (returns one empty stripe, then EOF). +- Profiles covered: P1. +- Gaps introduced/remaining: CSV parsing behavior beyond current no-output branch still pending. +- Commit: `39895cb84` + +- Date: 2026-02-12 +- Change: Added HTTP upload throttling integration coverage with deterministic timeout and replicate-bypass behavior. +- APIs covered: upload limit timeout path (`429`) and `type=replicate` bypass branch under concurrent upload pressure. +- Profiles covered: P8 (with short inflight timeout in test profile). +- Gaps introduced/remaining: download throttling wait/proxy branches remain. +- Commit: `464d0b2b6` + +- Date: 2026-02-12 +- Change: Added HTTP download throttling timeout coverage under concurrent large-read pressure. +- APIs covered: download limit timeout path (`429`) when another large response keeps in-flight download data above limit. +- Profiles covered: P8 (short inflight download timeout). +- Gaps introduced/remaining: download replica proxy fallback branch (`proxied=true`/replica redirect) remains. +- Commit: `a929e6ddc` + +- Date: 2026-02-12 +- Change: Expanded JWT auth mismatch variations for same-needle wrong-cookie tokens. +- APIs covered: write/read JWT rejection when token fid differs only by cookie from requested fid. +- Profiles covered: P3. +- Gaps introduced/remaining: token expiry boundary behavior remains untested. +- Commit: `61fe52398` + +- Date: 2026-02-12 +- Change: Added JWT expired-token rejection coverage for both write and read auth paths. +- APIs covered: write/read auth rejection when token signature is valid but `exp` is in the past. +- Profiles covered: P3. +- Gaps introduced/remaining: additional JWT transport variants (query/cookie token sources) remain. +- Commit: `6e808623f` + +- Date: 2026-02-12 +- Change: Added JWT token transport coverage via query parameter and HTTP-only cookie. +- APIs covered: write auth using `?jwt=` token and read auth using `AT` cookie token. +- Profiles covered: P3. +- Gaps introduced/remaining: JWT precedence rules when multiple token sources are present remain. +- Commit: `ccefdfe8d` + +- Date: 2026-02-12 +- Change: Added JWT token-source precedence coverage when both query and header tokens are present. +- APIs covered: query-token precedence over header-token for write/read auth checks. +- Profiles covered: P3. +- Gaps introduced/remaining: explicit query-vs-cookie precedence combination remains. +- Commit: `605054e5d` + +- Date: 2026-02-12 +- Change: Added JWT token-source precedence coverage when both header and cookie tokens are present. +- APIs covered: header-token precedence over cookie-token for write/read auth checks. +- Profiles covered: P3. +- Gaps introduced/remaining: JWT transport precedence matrix for query/header/cookie is now covered for tested combinations. +- Commit: `3fcaf845c` + +- Date: 2026-02-12 +- Change: Added JWT token-source precedence coverage when both query and cookie tokens are present. +- APIs covered: query-token precedence over cookie-token for write/read auth checks, including positive path when query is valid and cookie is invalid. +- Profiles covered: P3. +- Gaps introduced/remaining: none in current JWT token source precedence matrix. +- Commit: `4ea552973` + +- Date: 2026-02-12 +- Change: Added gRPC state update validation coverage for optimistic versioning and nil-state requests. +- APIs covered: `SetState` stale-version mismatch error path and nil-state no-op path. +- Profiles covered: P1. +- Gaps introduced/remaining: persistent state save failure branch remains environment-dependent. +- Commit: `34ff97996` + +- Date: 2026-02-12 +- Change: Added readonly lifecycle variation for persisted readonly flag path. +- APIs covered: `VolumeMarkReadonly` success path with `persist=true`. +- Profiles covered: P1. +- Gaps introduced/remaining: notify-master failure branches remain untested. +- Commit: `c37e6cd95` + +- Date: 2026-02-12 +- Change: Added CORS header validation on `OPTIONS` requests with `Origin` for admin and public ports. +- APIs covered: `OPTIONS /` CORS headers (`Access-Control-Allow-Origin`, `Access-Control-Allow-Credentials`) for split-port profile. +- Profiles covered: P2. +- Gaps introduced/remaining: unsupported-method parity for additional verbs beyond `PATCH`/`TRACE` remains. +- Commit: `ca08af7ba` + +- Date: 2026-02-12 +- Change: Added unsupported-method parity coverage for `TRACE` on admin/public split ports. +- APIs covered: admin `TRACE` error (`400`) vs public `TRACE` passthrough (`200`) behavior. +- Profiles covered: P2. +- Gaps introduced/remaining: broader unsupported verb matrix remains. +- Commit: `b03ddf855` + +- Date: 2026-02-12 +- Change: Added gRPC batch-delete cookie-check variation coverage. +- APIs covered: `BatchDelete` mismatch-cookie rejection path (`skip_cookie_check=false`) and skip-cookie-check acceptance/deletion path (`skip_cookie_check=true`). +- Profiles covered: P1. +- Gaps introduced/remaining: batch-delete malformed entry combinations are partially covered; mixed per-entry status permutations can be expanded. +- Commit: `87d75e786` + +- Date: 2026-02-12 +- Change: Expanded gRPC admin lifecycle variants for allocate/mount/unmount/delete edge cases. +- APIs covered: duplicate `AllocateVolume` rejection, missing-volume `VolumeMount` error, idempotent `VolumeUnmount` behavior for missing/already-unmounted volumes, and `VolumeDelete` maintenance-mode rejection. +- Profiles covered: P1. +- Gaps introduced/remaining: `VolumeConfigure` rollback/mount-failure branches still need dedicated fault-path coverage. +- Commit: `bc1faec8e` + +- Date: 2026-02-12 +- Change: Added mixed gRPC `BatchDelete` result-matrix coverage including early-stop behavior on cookie mismatch. +- APIs covered: per-entry status matrix in one request (`400` invalid fid, `202` accepted delete, `404` missing fid) and early break semantics when cookie mismatch occurs before later entries. +- Profiles covered: P1. +- Gaps introduced/remaining: chunk-manifest rejection (`406`) and EC batch-delete success path still require dedicated fixtures. +- Commit: `450f63ac4` + +- Date: 2026-02-12 +- Change: Added secured-UI HTTP behavior coverage under JWT-enabled profile. +- APIs covered: `/ui/index.html` route behavior when admin UI is not exposed due signing key; verified fallback auth-gated response path (`401`). +- Profiles covered: P3. +- Gaps introduced/remaining: explicit `access.ui=true` override scenario remains untested. +- Commit: `9c10ccb38` + +- Date: 2026-02-12 +- Change: Expanded split-port unsupported HTTP method matrix with non-standard verb coverage. +- APIs covered: admin/public parity for `PROPFIND` (`400` on admin, passthrough `200` on public) with post-call data-integrity verification. +- Profiles covered: P2. +- Gaps introduced/remaining: remaining unsupported-verb breadth now primarily around less common methods (e.g., `CONNECT`) and proxy-specific edge semantics. +- Commit: `1d7afd11e` + +- Date: 2026-02-12 +- Change: Expanded gRPC `VolumeConfigure` coverage for both success and configure-failure rollback reporting. +- APIs covered: valid replication success path and missing-volume configure failure path with remount-restore failure detail propagation. +- Profiles covered: P1. +- Gaps introduced/remaining: explicit unmount-failure and mount-failure branches via injected I/O faults are still pending. +- Commit: `287a60197` + +- Date: 2026-02-12 +- Change: Added `VolumeNeedleStatus` error-path coverage. +- APIs covered: missing-volume error path and missing-needle error path on existing normal volumes. +- Profiles covered: P1. +- Gaps introduced/remaining: EC-backed positive/error status permutations still require dedicated EC fixture state. +- Commit: `bf0c609a7` + +- Date: 2026-02-12 +- Change: Added HTTP deleted-needle read recovery coverage. +- APIs covered: `GET` with `readDeleted=true` returning deleted needle content, alongside normal post-delete `404` behavior. +- Profiles covered: P1. +- Gaps introduced/remaining: proxy/redirect interactions with `readDeleted` remain unverified. +- Commit: `2ed9434cf` + +- Date: 2026-02-12 +- Change: Added HTTP delete `ts` query parity coverage for deleted-read metadata behavior. +- APIs covered: `DELETE ?ts=` followed by `GET ?readDeleted=true`, asserting current Last-Modified parity with pre-delete reads. +- Profiles covered: P1. +- Gaps introduced/remaining: explicit externally visible timestamp override effects remain limited in current API responses. +- Commit: `225b8e800` + +- Date: 2026-02-12 +- Change: Added gRPC invalid-offset coverage for needle blob/meta reads. +- APIs covered: `ReadNeedleBlob` and `ReadNeedleMeta` failure paths on existing volumes with out-of-range offsets. +- Profiles covered: P1. +- Gaps introduced/remaining: low-level corrupted-size and backend I/O fault branches still require fault-injection hooks. +- Commit: `33ed77ad6` + +- Date: 2026-02-12 +- Change: Added mixed-volume `ReadAllNeedles` stream abort coverage. +- APIs covered: stream progression from an existing volume followed by missing-volume abort error in the same request. +- Profiles covered: P1. +- Gaps introduced/remaining: multi-volume happy-path ordering/volume-boundary assertions can be expanded further. +- Commit: `7799b28b1` + +- Date: 2026-02-12 +- Change: Tightened HTTP `HEAD` parity assertions for read path. +- APIs covered: `HEAD` behavior now verifies empty response body while retaining `Content-Length` parity expectations. +- Profiles covered: P1. +- Gaps introduced/remaining: additional conditional-header parity checks on `HEAD` can still be expanded. +- Commit: `9499e5400` + +- Date: 2026-02-12 +- Change: Expanded `VolumeServerStatus` payload assertions. +- APIs covered: `VolumeServerStatus` now validates presence of `State` and `MemoryStatus` (including non-zero goroutine count), in addition to version/disk payload checks. +- Profiles covered: P1. +- Gaps introduced/remaining: heartbeat-disabled and stopping-state transitions are still exercised indirectly rather than in a dedicated status-payload transition test. +- Commit: `374411418` + +- Date: 2026-02-12 +- Change: Added gRPC `BatchDelete` chunk-manifest rejection coverage. +- APIs covered: `BatchDelete` result status/error path for chunk-manifest needles (`406`, `ChunkManifest` message) and non-deletion parity after rejection. +- Profiles covered: P1. +- Gaps introduced/remaining: EC-backed `BatchDelete` positive path still pending dedicated EC fixture setup. +- Commit: `326be22a9` + +- Date: 2026-02-12 +- Change: Added gRPC `Query` cookie-mismatch branch parity coverage. +- APIs covered: `Query` behavior when fid id exists but cookie mismatches; verified current EOF/no-record stream outcome. +- Profiles covered: P1. +- Gaps introduced/remaining: CSV parsing behavior beyond current no-output path remains pending. +- Commit: `2aaf0a339` + +- Date: 2026-02-12 +- Change: Added positive gRPC `Ping` coverage for master targets. +- APIs covered: `Ping` success path for `target_type=master` with non-zero remote timestamp and valid timing envelope. +- Profiles covered: P1. +- Gaps introduced/remaining: positive filer-target ping path still requires a filer fixture in the integration harness. +- Commit: `fa5cad6dc` + +- Date: 2026-02-12 +- Change: Expanded HTTP conditional-header parity for `HEAD`. +- APIs covered: `HEAD` with `If-None-Match` now verifies `304` behavior and empty-body semantics. +- Profiles covered: P1. +- Gaps introduced/remaining: explicit `HEAD` + `If-Modified-Since` parity remains expandable. +- Commit: `9984f2ec4` + +- Date: 2026-02-12 +- Change: Added HTTP `HEAD` + `If-Modified-Since` conditional parity coverage. +- APIs covered: `HEAD` conditional path returning `304` with empty-body semantics when `If-Modified-Since` matches Last-Modified. +- Profiles covered: P1. +- Gaps introduced/remaining: deeper conditional-header combinations (`If-None-Match` + `If-Modified-Since` precedence) remain expandable. +- Commit: `e87563a3c` + +- Date: 2026-02-12 +- Change: Expanded split-port unsupported-method matrix with `CONNECT` parity coverage. +- APIs covered: admin/public behavior for `CONNECT` (`400` on admin, passthrough `200` on public) with post-call data-integrity verification. +- Profiles covered: P2. +- Gaps introduced/remaining: unsupported-method parity now covers `PATCH`, `TRACE`, `PROPFIND`, and `CONNECT`; additional uncommon verbs can still be sampled as needed. +- Commit: `2a893d10d` + +- Date: 2026-02-12 +- Change: Added explicit CORS `Access-Control-Allow-Headers` assertions for `OPTIONS`. +- APIs covered: admin/public `OPTIONS` now verify `Access-Control-Allow-Headers: *` in addition to allowed-method matrices. +- Profiles covered: P2. +- Gaps introduced/remaining: CORS method/header semantics are now covered for baseline split-port flows. +- Commit: `6fcb9fa9c` + +- Date: 2026-02-12 +- Change: Added dual-volume integration harness and read-mode matrix tests for missing-local volume behavior. +- APIs covered: HTTP read path when volume is missing locally across `readMode=proxy` (forward success), `readMode=redirect` (`301` + `proxied=true`), and `readMode=local` (`404`). +- Profiles covered: custom P1-derived profiles with `ReadMode` overrides. +- Gaps introduced/remaining: throttling-specific proxy fallback (`checkDownloadLimit` replica path) is still pending targeted pressure setup. +- Commits: `74b04a3f8`, `70ce0c8b8` + +- Date: 2026-02-12 +- Change: Added HTTP download-throttling replica fallback coverage under over-limit pressure. +- APIs covered: `checkDownloadLimit` replica-proxy branch (`download over limit + replica available -> proxy fallback`) with replicated dual-node setup. +- Profiles covered: P8-derived profile (`readMode=proxy`) with dual volume servers. +- Gaps introduced/remaining: cancellation branch (`499`) for download-limit waiting remains pending. +- Commit: `316cfb7a3` + +- Date: 2026-02-12 +- Change: Expanded missing-local deleted-read parity across proxy and redirect modes. +- APIs covered: `readDeleted=true` behavior from non-owning servers in `readMode=proxy` (forwarded success) and `readMode=redirect` (redirect query-drop parity leading to `404` on follow). +- Profiles covered: custom P1-derived profiles with `ReadMode` overrides. +- Gaps introduced/remaining: explicit proxied-loop edge behavior remains pending dedicated setup. +- Commit: `0164a383d` + +- Date: 2026-02-12 +- Change: Added filer-enabled harness and positive gRPC `Ping` coverage for filer targets. +- APIs covered: `Ping` success path for `target_type=filer` with non-zero remote timestamp and valid timing envelope. +- Profiles covered: P1 (single volume + filer auxiliary process). +- Gaps introduced/remaining: no additional ping target-type gaps remain in current harness scope. +- Commits: `5f09d86a8`, `2fc1dde3f` + +- Date: 2026-02-12 +- Change: Added download-limit proxied-loop guard coverage. +- APIs covered: over-limit download path with `proxied=true` now verifies replica fallback is skipped and timeout returns `429`. +- Profiles covered: P8-derived profile (`readMode=proxy`) with dual volume servers. +- Gaps introduced/remaining: explicit cancellation (`499`) branch for wait loops remains difficult to assert over HTTP transport semantics. +- Commit: `6d532eddc` + +- Date: 2026-02-12 +- Change: Added explicit no-limit throttling coverage for baseline profile. +- APIs covered: upload/download limit-disabled branches (`concurrent*Limit=0`) under concurrent pressure, verifying requests proceed (`200`/`201`) without throttling. +- Profiles covered: P1. +- Gaps introduced/remaining: cancellation (`499`) path remains pending due client-transport observability constraints. +- Commit: `2cd9a9c6f` + +- Date: 2026-02-12 +- Change: Added gRPC `VolumeServerLeave` idempotence coverage. +- APIs covered: repeated `VolumeServerLeave` calls (already-stopped heartbeat path) with persistent `healthz=503` verification. +- Profiles covered: P1. +- Gaps introduced/remaining: none for leave semantics in current harness. +- Commit: `0fd666916` + +- Date: 2026-02-12 +- Change: Expanded redirect read-mode query handling coverage for collection-aware redirects. +- APIs covered: non-owning redirect path now verifies `collection` query parameter preservation in `Location` alongside `proxied=true`. +- Profiles covered: P1-derived profile with `ReadMode=redirect` using dual volume servers. +- Gaps introduced/remaining: redirect branch currently preserves only `collection`; broader query propagation is intentionally untested for parity with current behavior. +- Commit: `ad287b392` + +- Date: 2026-02-12 +- Change: Tightened HTTP admin endpoint header parity checks. +- APIs covered: `/status` and `/healthz` now assert `Server` header format (`SeaweedFS Volume ...`) in addition to status and payload checks. +- Profiles covered: P1. +- Gaps introduced/remaining: none for baseline admin header checks. +- Commit: `cad34314b` + +- Date: 2026-02-12 +- Change: Expanded admin middleware parity checks for request-id propagation. +- APIs covered: `/healthz` now explicitly verifies request-id echo behavior via `x-amz-request-id` response header. +- Profiles covered: P1. +- Gaps introduced/remaining: none for request-id propagation on covered admin endpoints. +- Commit: `e0268a5b7` + +- Date: 2026-02-12 +- Change: Added over-limit invalid-vid branch coverage in download throttling proxy path. +- APIs covered: `checkDownloadLimit` -> `tryProxyToReplica` invalid volume-id parse path now explicitly verified as `400` under over-limit pressure. +- Profiles covered: P8. +- Gaps introduced/remaining: cancellation (`499`) branch remains pending due client-side transport observability limits. +- Commit: `b4984b335` + +- Date: 2026-02-12 +- Change: Expanded static-resource coverage to split public-port topology. +- APIs covered: public-port static endpoints (`/favicon.ico`, `/seaweedfsstatic/seaweed50x50.png`) under P2. +- Profiles covered: P2. +- Gaps introduced/remaining: static asset baseline coverage is now present for both admin and public ports. +- Commit: `e4c329811` + +- Date: 2026-02-12 +- Change: Added split public-port `HEAD` method parity coverage. +- APIs covered: public-port `HEAD` read behavior (`200`, content-length parity, empty-body semantics) for existing files. +- Profiles covered: P2. +- Gaps introduced/remaining: none for baseline public-port `GET/HEAD/OPTIONS` method coverage. +- Commit: `127c43b1a` + +- Date: 2026-02-12 +- Change: Added throttling wait-then-proceed branch coverage for both upload and download paths. +- APIs covered: over-limit `wait then proceed` behavior (`waitForUploadSlot` and `waitForDownloadSlot`) when in-flight pressure is released before timeout. +- Profiles covered: P8. +- Gaps introduced/remaining: explicit cancellation (`499`) path remains pending due client-side transport observability limits. +- Commit: `f7b362a2a` + +- Date: 2026-02-12 +- Change: Added HTTP read cookie-mismatch parity coverage. +- APIs covered: `GET` and `HEAD` wrong-cookie reads now explicitly verify `404` not-found semantics. +- Profiles covered: P1. +- Gaps introduced/remaining: internal read-failure (`500`) branch remains fault-injection dependent. +- Commit: `8fc192827` + +- Date: 2026-02-12 +- Change: Added throttling timeout-recovery coverage to validate in-flight counter release behavior. +- APIs covered: upload/download timeout (`429`) scenarios followed by successful recovery requests after pressure release. +- Profiles covered: P8. +- Gaps introduced/remaining: explicit cancellation (`499`) path remains pending due client-side transport observability limits. +- Commit: `3214972fa` + +- Date: 2026-02-12 +- Change: Added positive erasure-coding lifecycle coverage for generate/mount/info/unmount flows. +- APIs covered: `VolumeEcShardsGenerate` success, `VolumeEcShardsMount` success, `VolumeEcShardsInfo` success after mount, and expected not-found after unmount. +- Profiles covered: P1. +- Gaps introduced/remaining: multi-shard, rebuild/copy/blob-delete/to-volume positive permutations remain for broader EC matrix completeness. +- Commit: `80dce7c5b` + +- Date: 2026-02-12 +- Change: Expanded EC positive-path coverage for shard-read and blob-delete behavior. +- APIs covered: `VolumeEcShardRead` success, `VolumeEcBlobDelete` first-delete + idempotent second-delete, and `VolumeEcShardRead` deleted-marker (`IsDeleted`) path. +- Profiles covered: P1. +- Gaps introduced/remaining: EC copy/rebuild/to-volume multi-node success permutations remain for broader matrix completeness. +- Commit: `1f405f52d` + +- Date: 2026-02-13 +- Change: Added EC rebuild and `ShardsToVolume` branch coverage for missing-shard and no-live-entry conditions. +- APIs covered: `VolumeEcShardsRebuild` (missing-shard regenerate path), `VolumeEcShardsToVolume` (missing data shard error and no-live-entries `FailedPrecondition` path). +- Profiles covered: P1. +- Gaps introduced/remaining: `VolumeEcShardsToVolume` full success conversion path and EC copy/rebuild multi-node permutations remain pending. +- Commit: `e8c449c16` + +- Date: 2026-02-13 +- Change: Added positive `VolumeEcShardsToVolume` conversion coverage with post-conversion read verification. +- APIs covered: `VolumeEcShardsToVolume` success path (mounted data shards -> convert -> readable needle payload parity). +- Profiles covered: P1. +- Gaps introduced/remaining: EC multi-node shard-copy/rebuild permutations and last-shard cleanup edge cases remain pending. +- Commit: `6d223338a` + +- Date: 2026-02-13 +- Change: Added EC file-stream integration coverage for `ReceiveFile` and `CopyFile`. +- APIs covered: `ReceiveFile` success with `is_ec_volume=true`, and `CopyFile` EC missing-source behavior with `ignore_source_file_not_found=true/false`. +- Profiles covered: P1. +- Gaps introduced/remaining: multi-node EC shard copy/rebuild permutations and last-shard cleanup edge assertions remain pending. +- Commit: `a78290d56` + +- Date: 2026-02-13 +- Change: Added EC last-shard deletion cleanup coverage. +- APIs covered: `VolumeEcShardsDelete` branch where deleting the final shard set removes EC index files (`.ecx` cleanup), validated via EC `CopyFile` pre/post behavior. +- Profiles covered: P1. +- Gaps introduced/remaining: EC multi-node shard copy/rebuild permutations remain pending for broader distributed parity. +- Commit: `e8ef35346` + +- Date: 2026-02-13 +- Change: Added dual-node `VolumeCopy` integration success coverage. +- APIs covered: `VolumeCopy` full happy path (source status read, stream copy, destination mount) with post-copy HTTP read parity from destination node. +- Profiles covered: P4-derived dual-volume topology. +- Gaps introduced/remaining: additional distributed copy branches (e.g., existing-destination overwrite and remote-dat variations) remain pending. +- Commit: `7538653ad` + +- Date: 2026-02-13 +- Change: Added dual-node `VolumeCopy` overwrite coverage when destination volume already exists. +- APIs covered: existing-destination delete-before-copy branch, with pre-copy destination payload assertion and post-copy source parity validation. +- Profiles covered: P4-derived dual-volume topology. +- Gaps introduced/remaining: remote-dat-file and no-space source-copy branches remain pending. +- Commit: `2d49019e9` + +- Date: 2026-02-13 +- Change: Added HTTP write error-path integration coverage. +- APIs covered: `POST` invalid vid/fid parse rejections (`400`), malformed multipart form parse failure (`400`), and `Content-MD5` mismatch validation failure (`400`). +- Profiles covered: P1. +- Gaps introduced/remaining: file-size limit rejection and replicated-write failure branches remain pending for write-path breadth. +- Commit: `046390e54` + +- Date: 2026-02-13 +- Change: Added HTTP conditional-header precedence coverage. +- APIs covered: `GET` with combined `If-Modified-Since` + mismatched `If-None-Match` (`304` by current precedence), plus invalid `If-Modified-Since` fallback behavior (`200` body path). +- Profiles covered: P1. +- Gaps introduced/remaining: range-sum oversized-request behavior and image transform/chunk-manifest read branches remain pending. +- Commit: `31d59f0b8` + +- Date: 2026-02-13 +- Change: Added HTTP oversized-multi-range guard coverage. +- APIs covered: `GET` `Range` requests where combined ranges exceed total object size (`sumRangesSize > totalSize`) with current parity response (`200` + empty body). +- Profiles covered: P1. +- Gaps introduced/remaining: image resize/crop and chunk-manifest read branches remain pending. +- Commit: `612e5f61c` + +- Date: 2026-02-13 +- Change: Added HTTP image transformation integration coverage. +- APIs covered: read-path image resize (`width`,`height`) and crop (`crop_*`) branches with decoded output-dimension assertions. +- Profiles covered: P1. +- Gaps introduced/remaining: chunk-manifest read/`cm=false` parity and compressed-content encoding matrix remain pending. +- Commit: `d68803ecc` + +- Date: 2026-02-13 +- Change: Added HTTP chunk-manifest integration coverage for expansion and bypass. +- APIs covered: chunk-manifest auto-expansion read path (`X-File-Store: chunked`) and `cm=false` raw-manifest bypass parity. +- Profiles covered: P1. +- Gaps introduced/remaining: compressed-content encoding matrix and some write/delete failure-injection branches remain pending. +- Commit: `8ecf427c4` + +- Date: 2026-02-13 +- Change: Added HTTP compressed-read encoding matrix coverage. +- APIs covered: compressed needle read branch with `Accept-Encoding=gzip` passthrough (`Content-Encoding: gzip`) and `Accept-Encoding=identity` decompression parity. +- Profiles covered: P1. +- Gaps introduced/remaining: write/delete failure-injection and request-cancel (`499`) paths remain pending due transport/fault observability constraints. +- Commit: `e17604c57` + +- Date: 2026-02-13 +- Change: Added dual-node gRPC tail receiver success-path coverage. +- APIs covered: `VolumeTailReceiver` end-to-end replication from source server stream into destination volume, with post-tail HTTP payload parity verification. +- Profiles covered: P4-derived dual-volume topology. +- Gaps introduced/remaining: sender large-needle chunking specifics and transport-interruption branches remain pending. +- Commit: `3989bc6e5` + +- Date: 2026-02-13 +- Change: Added gRPC tail sender large-needle chunking coverage. +- APIs covered: `VolumeTailSender` stream chunk-splitting path for oversized needle bodies (multiple chunks with non-last and final `IsLastChunk=true` markers). +- Profiles covered: P1. +- Gaps introduced/remaining: sender/receiver transport interruption branches remain pending. +- Commit: `514c05131` + +- Date: 2026-02-13 +- Change: Added EC-backed `VolumeNeedleStatus` integration coverage. +- APIs covered: `VolumeNeedleStatus` EC execution path (normal volume unmounted + EC shards mounted), including missing-needle error behavior. +- Profiles covered: P1. +- Gaps introduced/remaining: transport-interruption and deep fault-injection branches remain pending. +- Commit: `4ef666791` + +- Date: 2026-02-13 +- Change: Added dual-node EC shard copy success coverage. +- APIs covered: `VolumeEcShardsCopy` positive path from source peer (including `copy_ecx_file`/`copy_vif_file`) with copied artifact verification via EC `CopyFile` on destination. +- Profiles covered: P4-derived dual-volume topology. +- Gaps introduced/remaining: EC rebuild/copy distributed failure injection permutations remain pending. +- Commit: `c9f6710c2` + +- Date: 2026-02-13 +- Change: Added HTTP chunk-manifest delete cleanup coverage. +- APIs covered: chunk-manifest delete success path with child-chunk deletion verification and expected delete response size parity. +- Profiles covered: P1. +- Gaps introduced/remaining: chunk-manifest delete failure-injection branch remains pending. +- Commit: `c6df98a02` + +- Date: 2026-02-13 +- Change: Added HTTP chunk-manifest delete failure-path coverage. +- APIs covered: chunk-manifest child-delete error branch (`500`) and non-deletion parity of manifest metadata after failed delete. +- Profiles covered: P1. +- Gaps introduced/remaining: request-cancel (`499`) and deep transport-interruption branches remain pending. +- Commit: `38a1f4f4f` + +- Date: 2026-02-13 +- Change: Added EC shard-copy source-unavailable error coverage. +- APIs covered: `VolumeEcShardsCopy` network/source-unreachable failure path with wrapped RPC error propagation. +- Profiles covered: P1. +- Gaps introduced/remaining: `499` cancellation and transport-interruption branches remain pending. +- Commit: `d1e5f390a` diff --git a/test/volume_server/Makefile b/test/volume_server/Makefile new file mode 100644 index 000000000..1801734a9 --- /dev/null +++ b/test/volume_server/Makefile @@ -0,0 +1,7 @@ +.PHONY: test-volume-server test-volume-server-short + +test-volume-server: + go test ./test/volume_server/... -v + +test-volume-server-short: + go test ./test/volume_server/... -short -v diff --git a/test/volume_server/README.md b/test/volume_server/README.md new file mode 100644 index 000000000..1fed73522 --- /dev/null +++ b/test/volume_server/README.md @@ -0,0 +1,27 @@ +# Volume Server Integration Tests + +This package contains integration tests for SeaweedFS volume server HTTP and gRPC APIs. + +## Run Tests + +Run tests from repo root: + +```bash +go test ./test/volume_server/... -v +``` + +If a `weed` binary is not found, the harness will build one automatically. + +## Optional environment variables + +- `WEED_BINARY`: explicit path to the `weed` executable (disables auto-build). +- `VOLUME_SERVER_IT_KEEP_LOGS=1`: keep temporary test directories and process logs. + +## Current scope (Phase 0) + +- Shared cluster/framework utilities +- Matrix profile definitions +- Initial HTTP admin endpoint checks +- Initial gRPC state/status checks + +More API coverage is tracked in `/Users/chris/dev/seaweedfs2/test/volume_server/DEV_PLAN.md`. diff --git a/test/volume_server/framework/cluster.go b/test/volume_server/framework/cluster.go new file mode 100644 index 000000000..4bb1b55d5 --- /dev/null +++ b/test/volume_server/framework/cluster.go @@ -0,0 +1,442 @@ +package framework + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +const ( + defaultWaitTimeout = 30 * time.Second + defaultWaitTick = 200 * time.Millisecond + testVolumeSizeLimitMB = 32 +) + +// Cluster is a lightweight SeaweedFS master + one volume server test harness. +type Cluster struct { + testingTB testing.TB + profile matrix.Profile + + weedBinary string + baseDir string + configDir string + logsDir string + keepLogs bool + + masterPort int + masterGrpcPort int + volumePort int + volumeGrpcPort int + volumePubPort int + + masterCmd *exec.Cmd + volumeCmd *exec.Cmd + + cleanupOnce sync.Once +} + +// StartSingleVolumeCluster boots one master and one volume server. +func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster { + t.Helper() + + weedBinary, err := FindOrBuildWeedBinary() + if err != nil { + t.Fatalf("resolve weed binary: %v", err) + } + + baseDir, keepLogs, err := newWorkDir() + if err != nil { + t.Fatalf("create temp test directory: %v", err) + } + + configDir := filepath.Join(baseDir, "config") + logsDir := filepath.Join(baseDir, "logs") + masterDataDir := filepath.Join(baseDir, "master") + volumeDataDir := filepath.Join(baseDir, "volume") + for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir} { + if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil { + t.Fatalf("create %s: %v", dir, mkErr) + } + } + + if err = writeSecurityConfig(configDir, profile); err != nil { + t.Fatalf("write security config: %v", err) + } + + masterPort, masterGrpcPort, err := allocateMasterPortPair() + if err != nil { + t.Fatalf("allocate master port pair: %v", err) + } + + ports, err := allocatePorts(3) + if err != nil { + t.Fatalf("allocate ports: %v", err) + } + + c := &Cluster{ + testingTB: t, + profile: profile, + weedBinary: weedBinary, + baseDir: baseDir, + configDir: configDir, + logsDir: logsDir, + keepLogs: keepLogs, + masterPort: masterPort, + masterGrpcPort: masterGrpcPort, + volumePort: ports[0], + volumeGrpcPort: ports[1], + volumePubPort: ports[0], + } + if profile.SplitPublicPort { + c.volumePubPort = ports[2] + } + + if err = c.startMaster(masterDataDir); err != nil { + c.Stop() + t.Fatalf("start master: %v", err) + } + if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil { + masterLog := c.tailLog("master.log") + c.Stop() + t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog) + } + + if err = c.startVolume(volumeDataDir); err != nil { + masterLog := c.tailLog("master.log") + c.Stop() + t.Fatalf("start volume: %v\nmaster log tail:\n%s", err, masterLog) + } + if err = c.waitForHTTP(c.VolumeAdminURL() + "/status"); err != nil { + volumeLog := c.tailLog("volume.log") + c.Stop() + t.Fatalf("wait for volume readiness: %v\nvolume log tail:\n%s", err, volumeLog) + } + if err = c.waitForTCP(c.VolumeGRPCAddress()); err != nil { + volumeLog := c.tailLog("volume.log") + c.Stop() + t.Fatalf("wait for volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog) + } + + t.Cleanup(func() { + c.Stop() + }) + + return c +} + +// Stop terminates all processes and cleans temporary files. +func (c *Cluster) Stop() { + if c == nil { + return + } + c.cleanupOnce.Do(func() { + stopProcess(c.volumeCmd) + stopProcess(c.masterCmd) + if !c.keepLogs && !c.testingTB.Failed() { + _ = os.RemoveAll(c.baseDir) + } else if c.baseDir != "" { + c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir) + } + }) +} + +func (c *Cluster) startMaster(dataDir string) error { + logFile, err := os.Create(filepath.Join(c.logsDir, "master.log")) + if err != nil { + return err + } + + args := []string{ + "-config_dir=" + c.configDir, + "master", + "-ip=127.0.0.1", + "-port=" + strconv.Itoa(c.masterPort), + "-port.grpc=" + strconv.Itoa(c.masterGrpcPort), + "-mdir=" + dataDir, + "-peers=none", + "-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB), + "-defaultReplication=000", + } + + c.masterCmd = exec.Command(c.weedBinary, args...) + c.masterCmd.Dir = c.baseDir + c.masterCmd.Stdout = logFile + c.masterCmd.Stderr = logFile + return c.masterCmd.Start() +} + +func (c *Cluster) startVolume(dataDir string) error { + logFile, err := os.Create(filepath.Join(c.logsDir, "volume.log")) + if err != nil { + return err + } + + args := []string{ + "-config_dir=" + c.configDir, + "volume", + "-ip=127.0.0.1", + "-port=" + strconv.Itoa(c.volumePort), + "-port.grpc=" + strconv.Itoa(c.volumeGrpcPort), + "-port.public=" + strconv.Itoa(c.volumePubPort), + "-dir=" + dataDir, + "-max=16", + "-master=127.0.0.1:" + strconv.Itoa(c.masterPort), + "-readMode=" + c.profile.ReadMode, + "-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB), + "-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB), + } + if c.profile.InflightUploadTimeout > 0 { + args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String()) + } + if c.profile.InflightDownloadTimeout > 0 { + args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String()) + } + + c.volumeCmd = exec.Command(c.weedBinary, args...) + c.volumeCmd.Dir = c.baseDir + c.volumeCmd.Stdout = logFile + c.volumeCmd.Stderr = logFile + return c.volumeCmd.Start() +} + +func (c *Cluster) waitForHTTP(url string) error { + client := &http.Client{Timeout: 1 * time.Second} + deadline := time.Now().Add(defaultWaitTimeout) + for time.Now().Before(deadline) { + resp, err := client.Get(url) + if err == nil { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.StatusCode < 500 { + return nil + } + } + time.Sleep(defaultWaitTick) + } + return fmt.Errorf("timed out waiting for %s", url) +} + +func (c *Cluster) waitForTCP(addr string) error { + deadline := time.Now().Add(defaultWaitTimeout) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", addr, time.Second) + if err == nil { + _ = conn.Close() + return nil + } + time.Sleep(defaultWaitTick) + } + return fmt.Errorf("timed out waiting for tcp %s", addr) +} + +func stopProcess(cmd *exec.Cmd) { + if cmd == nil || cmd.Process == nil { + return + } + + _ = cmd.Process.Signal(os.Interrupt) + done := make(chan error, 1) + go func() { + done <- cmd.Wait() + }() + + select { + case <-time.After(10 * time.Second): + _ = cmd.Process.Kill() + <-done + case <-done: + } +} + +func allocatePorts(count int) ([]int, error) { + listeners := make([]net.Listener, 0, count) + ports := make([]int, 0, count) + for i := 0; i < count; i++ { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + for _, ll := range listeners { + _ = ll.Close() + } + return nil, err + } + listeners = append(listeners, l) + ports = append(ports, l.Addr().(*net.TCPAddr).Port) + } + for _, l := range listeners { + _ = l.Close() + } + return ports, nil +} + +func allocateMasterPortPair() (int, int, error) { + for masterPort := 10000; masterPort <= 55535; masterPort++ { + masterGrpcPort := masterPort + 10000 + l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort))) + if err != nil { + continue + } + l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterGrpcPort))) + if err != nil { + _ = l1.Close() + continue + } + _ = l2.Close() + _ = l1.Close() + return masterPort, masterGrpcPort, nil + } + return 0, 0, errors.New("unable to find available master port pair") +} + +func newWorkDir() (dir string, keepLogs bool, err error) { + keepLogs = os.Getenv("VOLUME_SERVER_IT_KEEP_LOGS") == "1" + dir, err = os.MkdirTemp("", "seaweedfs_volume_server_it_") + return dir, keepLogs, err +} + +func writeSecurityConfig(configDir string, profile matrix.Profile) error { + var b strings.Builder + if profile.EnableJWT { + if profile.JWTSigningKey == "" || profile.JWTReadKey == "" { + return errors.New("jwt profile requires both write and read keys") + } + b.WriteString("[jwt.signing]\n") + b.WriteString("key = \"") + b.WriteString(profile.JWTSigningKey) + b.WriteString("\"\n") + b.WriteString("expires_after_seconds = 60\n\n") + + b.WriteString("[jwt.signing.read]\n") + b.WriteString("key = \"") + b.WriteString(profile.JWTReadKey) + b.WriteString("\"\n") + b.WriteString("expires_after_seconds = 60\n") + } + if b.Len() == 0 { + b.WriteString("# optional security config generated for integration tests\n") + } + return os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(b.String()), 0o644) +} + +// FindOrBuildWeedBinary returns an executable weed binary, building one when needed. +func FindOrBuildWeedBinary() (string, error) { + if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" { + if isExecutableFile(fromEnv) { + return fromEnv, nil + } + return "", fmt.Errorf("WEED_BINARY is set but not executable: %s", fromEnv) + } + + repoRoot := "" + if _, file, _, ok := runtime.Caller(0); ok { + repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..")) + candidate := filepath.Join(repoRoot, "weed", "weed") + if isExecutableFile(candidate) { + return candidate, nil + } + } + + if repoRoot == "" { + return "", errors.New("unable to detect repository root") + } + + binDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_bin") + if err := os.MkdirAll(binDir, 0o755); err != nil { + return "", fmt.Errorf("create binary directory %s: %w", binDir, err) + } + binPath := filepath.Join(binDir, "weed") + if isExecutableFile(binPath) { + return binPath, nil + } + + cmd := exec.Command("go", "build", "-o", binPath, ".") + cmd.Dir = filepath.Join(repoRoot, "weed") + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("build weed binary: %w\n%s", err, out.String()) + } + if !isExecutableFile(binPath) { + return "", fmt.Errorf("built weed binary is not executable: %s", binPath) + } + return binPath, nil +} + +func isExecutableFile(path string) bool { + info, err := os.Stat(path) + if err != nil || info.IsDir() { + return false + } + mode := info.Mode().Perm() + return mode&0o111 != 0 +} + +func (c *Cluster) tailLog(logName string) string { + f, err := os.Open(filepath.Join(c.logsDir, logName)) + if err != nil { + return "" + } + defer f.Close() + + scanner := bufio.NewScanner(f) + lines := make([]string, 0, 40) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + if len(lines) > 40 { + lines = lines[1:] + } + } + return strings.Join(lines, "\n") +} + +func (c *Cluster) MasterAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort)) +} + +func (c *Cluster) VolumeAdminAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort)) +} + +func (c *Cluster) VolumePublicAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort)) +} + +func (c *Cluster) VolumeGRPCAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort)) +} + +// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort +func (c *Cluster) VolumeServerAddress() string { + return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(), c.volumeGrpcPort) +} + +func (c *Cluster) MasterURL() string { + return "http://" + c.MasterAddress() +} + +func (c *Cluster) VolumeAdminURL() string { + return "http://" + c.VolumeAdminAddress() +} + +func (c *Cluster) VolumePublicURL() string { + return "http://" + c.VolumePublicAddress() +} + +func (c *Cluster) BaseDir() string { + return c.baseDir +} diff --git a/test/volume_server/framework/cluster_dual.go b/test/volume_server/framework/cluster_dual.go new file mode 100644 index 000000000..ffa28c75b --- /dev/null +++ b/test/volume_server/framework/cluster_dual.go @@ -0,0 +1,293 @@ +package framework + +import ( + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "sync" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +type DualVolumeCluster struct { + testingTB testing.TB + profile matrix.Profile + + weedBinary string + baseDir string + configDir string + logsDir string + keepLogs bool + + masterPort int + masterGrpcPort int + + volumePort0 int + volumeGrpcPort0 int + volumePubPort0 int + volumePort1 int + volumeGrpcPort1 int + volumePubPort1 int + + masterCmd *exec.Cmd + volumeCmd0 *exec.Cmd + volumeCmd1 *exec.Cmd + + cleanupOnce sync.Once +} + +func StartDualVolumeCluster(t testing.TB, profile matrix.Profile) *DualVolumeCluster { + t.Helper() + + weedBinary, err := FindOrBuildWeedBinary() + if err != nil { + t.Fatalf("resolve weed binary: %v", err) + } + + baseDir, keepLogs, err := newWorkDir() + if err != nil { + t.Fatalf("create temp test directory: %v", err) + } + + configDir := filepath.Join(baseDir, "config") + logsDir := filepath.Join(baseDir, "logs") + masterDataDir := filepath.Join(baseDir, "master") + volumeDataDir0 := filepath.Join(baseDir, "volume0") + volumeDataDir1 := filepath.Join(baseDir, "volume1") + for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir0, volumeDataDir1} { + if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil { + t.Fatalf("create %s: %v", dir, mkErr) + } + } + + if err = writeSecurityConfig(configDir, profile); err != nil { + t.Fatalf("write security config: %v", err) + } + + masterPort, masterGrpcPort, err := allocateMasterPortPair() + if err != nil { + t.Fatalf("allocate master port pair: %v", err) + } + + ports, err := allocatePorts(6) + if err != nil { + t.Fatalf("allocate volume ports: %v", err) + } + + c := &DualVolumeCluster{ + testingTB: t, + profile: profile, + weedBinary: weedBinary, + baseDir: baseDir, + configDir: configDir, + logsDir: logsDir, + keepLogs: keepLogs, + masterPort: masterPort, + masterGrpcPort: masterGrpcPort, + volumePort0: ports[0], + volumeGrpcPort0: ports[1], + volumePubPort0: ports[0], + volumePort1: ports[2], + volumeGrpcPort1: ports[3], + volumePubPort1: ports[2], + } + if profile.SplitPublicPort { + c.volumePubPort0 = ports[4] + c.volumePubPort1 = ports[5] + } + + if err = c.startMaster(masterDataDir); err != nil { + c.Stop() + t.Fatalf("start master: %v", err) + } + if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil { + masterLog := c.tailLog("master.log") + c.Stop() + t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog) + } + + if err = c.startVolume(0, volumeDataDir0); err != nil { + masterLog := c.tailLog("master.log") + c.Stop() + t.Fatalf("start first volume server: %v\nmaster log tail:\n%s", err, masterLog) + } + if err = c.waitForHTTP(c.VolumeAdminURL(0) + "/status"); err != nil { + volumeLog := c.tailLog("volume0.log") + c.Stop() + t.Fatalf("wait for first volume readiness: %v\nvolume log tail:\n%s", err, volumeLog) + } + if err = c.waitForTCP(c.VolumeGRPCAddress(0)); err != nil { + volumeLog := c.tailLog("volume0.log") + c.Stop() + t.Fatalf("wait for first volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog) + } + + if err = c.startVolume(1, volumeDataDir1); err != nil { + volumeLog := c.tailLog("volume0.log") + c.Stop() + t.Fatalf("start second volume server: %v\nfirst volume log tail:\n%s", err, volumeLog) + } + if err = c.waitForHTTP(c.VolumeAdminURL(1) + "/status"); err != nil { + volumeLog := c.tailLog("volume1.log") + c.Stop() + t.Fatalf("wait for second volume readiness: %v\nvolume log tail:\n%s", err, volumeLog) + } + if err = c.waitForTCP(c.VolumeGRPCAddress(1)); err != nil { + volumeLog := c.tailLog("volume1.log") + c.Stop() + t.Fatalf("wait for second volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog) + } + + t.Cleanup(func() { + c.Stop() + }) + + return c +} + +func (c *DualVolumeCluster) Stop() { + if c == nil { + return + } + c.cleanupOnce.Do(func() { + stopProcess(c.volumeCmd1) + stopProcess(c.volumeCmd0) + stopProcess(c.masterCmd) + if !c.keepLogs && !c.testingTB.Failed() { + _ = os.RemoveAll(c.baseDir) + } else if c.baseDir != "" { + c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir) + } + }) +} + +func (c *DualVolumeCluster) startMaster(dataDir string) error { + logFile, err := os.Create(filepath.Join(c.logsDir, "master.log")) + if err != nil { + return err + } + + args := []string{ + "-config_dir=" + c.configDir, + "master", + "-ip=127.0.0.1", + "-port=" + strconv.Itoa(c.masterPort), + "-port.grpc=" + strconv.Itoa(c.masterGrpcPort), + "-mdir=" + dataDir, + "-peers=none", + "-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB), + "-defaultReplication=000", + } + + c.masterCmd = exec.Command(c.weedBinary, args...) + c.masterCmd.Dir = c.baseDir + c.masterCmd.Stdout = logFile + c.masterCmd.Stderr = logFile + return c.masterCmd.Start() +} + +func (c *DualVolumeCluster) startVolume(index int, dataDir string) error { + logName := fmt.Sprintf("volume%d.log", index) + logFile, err := os.Create(filepath.Join(c.logsDir, logName)) + if err != nil { + return err + } + + volumePort := c.volumePort0 + volumeGrpcPort := c.volumeGrpcPort0 + volumePubPort := c.volumePubPort0 + if index == 1 { + volumePort = c.volumePort1 + volumeGrpcPort = c.volumeGrpcPort1 + volumePubPort = c.volumePubPort1 + } + + args := []string{ + "-config_dir=" + c.configDir, + "volume", + "-ip=127.0.0.1", + "-port=" + strconv.Itoa(volumePort), + "-port.grpc=" + strconv.Itoa(volumeGrpcPort), + "-port.public=" + strconv.Itoa(volumePubPort), + "-dir=" + dataDir, + "-max=16", + "-master=127.0.0.1:" + strconv.Itoa(c.masterPort), + "-readMode=" + c.profile.ReadMode, + "-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB), + "-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB), + } + if c.profile.InflightUploadTimeout > 0 { + args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String()) + } + if c.profile.InflightDownloadTimeout > 0 { + args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String()) + } + + cmd := exec.Command(c.weedBinary, args...) + cmd.Dir = c.baseDir + cmd.Stdout = logFile + cmd.Stderr = logFile + + if err = cmd.Start(); err != nil { + return err + } + if index == 1 { + c.volumeCmd1 = cmd + } else { + c.volumeCmd0 = cmd + } + return nil +} + +func (c *DualVolumeCluster) waitForHTTP(url string) error { + return (&Cluster{}).waitForHTTP(url) +} + +func (c *DualVolumeCluster) waitForTCP(addr string) error { + return (&Cluster{}).waitForTCP(addr) +} + +func (c *DualVolumeCluster) tailLog(logName string) string { + return (&Cluster{logsDir: c.logsDir}).tailLog(logName) +} + +func (c *DualVolumeCluster) MasterAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort)) +} + +func (c *DualVolumeCluster) MasterURL() string { + return "http://" + c.MasterAddress() +} + +func (c *DualVolumeCluster) VolumeAdminAddress(index int) string { + if index == 1 { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort1)) + } + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort0)) +} + +func (c *DualVolumeCluster) VolumePublicAddress(index int) string { + if index == 1 { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort1)) + } + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort0)) +} + +func (c *DualVolumeCluster) VolumeGRPCAddress(index int) string { + if index == 1 { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort1)) + } + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort0)) +} + +func (c *DualVolumeCluster) VolumeAdminURL(index int) string { + return "http://" + c.VolumeAdminAddress(index) +} + +func (c *DualVolumeCluster) VolumePublicURL(index int) string { + return "http://" + c.VolumePublicAddress(index) +} diff --git a/test/volume_server/framework/cluster_with_filer.go b/test/volume_server/framework/cluster_with_filer.go new file mode 100644 index 000000000..67f35fd26 --- /dev/null +++ b/test/volume_server/framework/cluster_with_filer.go @@ -0,0 +1,91 @@ +package framework + +import ( + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +type ClusterWithFiler struct { + *Cluster + + filerCmd *exec.Cmd + filerPort int + filerGrpcPort int +} + +func StartSingleVolumeClusterWithFiler(t testing.TB, profile matrix.Profile) *ClusterWithFiler { + t.Helper() + + baseCluster := StartSingleVolumeCluster(t, profile) + + ports, err := allocatePorts(2) + if err != nil { + t.Fatalf("allocate filer ports: %v", err) + } + + filerDataDir := filepath.Join(baseCluster.baseDir, "filer") + if mkErr := os.MkdirAll(filerDataDir, 0o755); mkErr != nil { + t.Fatalf("create filer data dir: %v", mkErr) + } + + logFile, err := os.Create(filepath.Join(baseCluster.logsDir, "filer.log")) + if err != nil { + t.Fatalf("create filer log file: %v", err) + } + + filerPort := ports[0] + filerGrpcPort := ports[1] + args := []string{ + "-config_dir=" + baseCluster.configDir, + "filer", + "-master=127.0.0.1:" + strconv.Itoa(baseCluster.masterPort), + "-ip=127.0.0.1", + "-port=" + strconv.Itoa(filerPort), + "-port.grpc=" + strconv.Itoa(filerGrpcPort), + "-defaultStoreDir=" + filerDataDir, + } + + filerCmd := exec.Command(baseCluster.weedBinary, args...) + filerCmd.Dir = baseCluster.baseDir + filerCmd.Stdout = logFile + filerCmd.Stderr = logFile + if err = filerCmd.Start(); err != nil { + t.Fatalf("start filer: %v", err) + } + + if err = baseCluster.waitForTCP(net.JoinHostPort("127.0.0.1", strconv.Itoa(filerGrpcPort))); err != nil { + filerLogTail := baseCluster.tailLog("filer.log") + stopProcess(filerCmd) + t.Fatalf("wait for filer grpc readiness: %v\nfiler log tail:\n%s", err, filerLogTail) + } + + t.Cleanup(func() { + stopProcess(filerCmd) + }) + + return &ClusterWithFiler{ + Cluster: baseCluster, + filerCmd: filerCmd, + filerPort: filerPort, + filerGrpcPort: filerGrpcPort, + } +} + +func (c *ClusterWithFiler) FilerAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerPort)) +} + +func (c *ClusterWithFiler) FilerGRPCAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerGrpcPort)) +} + +func (c *ClusterWithFiler) FilerServerAddress() string { + return fmt.Sprintf("%s.%d", c.FilerAddress(), c.filerGrpcPort) +} diff --git a/test/volume_server/framework/fault_injection.go b/test/volume_server/framework/fault_injection.go new file mode 100644 index 000000000..4a711a340 --- /dev/null +++ b/test/volume_server/framework/fault_injection.go @@ -0,0 +1,8 @@ +package framework + +// Phase 0 placeholder for future fault injection utilities. +// +// Planned extensions: +// - restart/kill selected processes +// - temporary network isolation hooks +// - master or peer outage helpers for proxy/replication branch coverage diff --git a/test/volume_server/framework/grpc_client.go b/test/volume_server/framework/grpc_client.go new file mode 100644 index 000000000..bf4d95182 --- /dev/null +++ b/test/volume_server/framework/grpc_client.go @@ -0,0 +1,28 @@ +package framework + +import ( + "context" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func DialVolumeServer(t testing.TB, address string) (*grpc.ClientConn, volume_server_pb.VolumeServerClient) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + conn, err := grpc.DialContext(ctx, address, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithBlock(), + ) + if err != nil { + t.Fatalf("dial volume grpc %s: %v", address, err) + } + + return conn, volume_server_pb.NewVolumeServerClient(conn) +} diff --git a/test/volume_server/framework/http_client.go b/test/volume_server/framework/http_client.go new file mode 100644 index 000000000..816b64720 --- /dev/null +++ b/test/volume_server/framework/http_client.go @@ -0,0 +1,34 @@ +package framework + +import ( + "io" + "net/http" + "testing" + "time" +) + +func NewHTTPClient() *http.Client { + return &http.Client{Timeout: 10 * time.Second} +} + +func DoRequest(t testing.TB, client *http.Client, req *http.Request) *http.Response { + t.Helper() + resp, err := client.Do(req) + if err != nil { + t.Fatalf("http request %s %s: %v", req.Method, req.URL.String(), err) + } + return resp +} + +func ReadAllAndClose(t testing.TB, resp *http.Response) []byte { + t.Helper() + if resp == nil { + return nil + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read response body: %v", err) + } + return body +} diff --git a/test/volume_server/framework/volume_fixture.go b/test/volume_server/framework/volume_fixture.go new file mode 100644 index 000000000..f6229f3f8 --- /dev/null +++ b/test/volume_server/framework/volume_fixture.go @@ -0,0 +1,56 @@ +package framework + +import ( + "bytes" + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" +) + +func AllocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{ + VolumeId: volumeID, + Collection: collection, + Replication: "000", + Version: uint32(needle.GetCurrentVersion()), + }) + if err != nil { + t.Fatalf("allocate volume %d: %v", volumeID, err) + } +} + +func NewFileID(volumeID uint32, key uint64, cookie uint32) string { + return needle.NewFileId(needle.VolumeId(volumeID), key, cookie).String() +} + +func UploadBytes(t testing.TB, client *http.Client, volumeURL, fid string, data []byte) *http.Response { + t.Helper() + + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/%s", volumeURL, fid), bytes.NewReader(data)) + if err != nil { + t.Fatalf("build upload request: %v", err) + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Length", fmt.Sprintf("%d", len(data))) + return DoRequest(t, client, req) +} + +func ReadBytes(t testing.TB, client *http.Client, volumeURL, fid string) *http.Response { + t.Helper() + + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", volumeURL, fid), nil) + if err != nil { + t.Fatalf("build read request: %v", err) + } + return DoRequest(t, client, req) +} diff --git a/test/volume_server/grpc/admin_extra_test.go b/test/volume_server/grpc/admin_extra_test.go new file mode 100644 index 000000000..de62fcdb8 --- /dev/null +++ b/test/volume_server/grpc/admin_extra_test.go @@ -0,0 +1,445 @@ +package volume_server_grpc_test + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/cluster" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestVolumeNeedleStatusForUploadedFile(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(21) + const needleID = uint64(778899) + const cookie = uint32(0xA1B2C3D4) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + client := framework.NewHTTPClient() + payload := []byte("needle-status-payload") + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: volumeID, + NeedleId: needleID, + }) + if err != nil { + t.Fatalf("VolumeNeedleStatus failed: %v", err) + } + if statusResp.GetNeedleId() != needleID { + t.Fatalf("needle id mismatch: got %d want %d", statusResp.GetNeedleId(), needleID) + } + if statusResp.GetCookie() != cookie { + t.Fatalf("cookie mismatch: got %d want %d", statusResp.GetCookie(), cookie) + } + if statusResp.GetSize() == 0 { + t.Fatalf("expected non-zero needle size") + } +} + +func TestVolumeNeedleStatusViaEcShardsWhenNormalVolumeUnmounted(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(26) + const needleID = uint64(778900) + const cookie = uint32(0xA1B2C3D5) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, cookie) + payload := []byte("needle-status-ec-path-payload") + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount data shards failed: %v", err) + } + + _, err = grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{ + VolumeId: volumeID, + }) + if err != nil { + t.Fatalf("VolumeUnmount failed: %v", err) + } + + statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: volumeID, + NeedleId: needleID, + }) + if err != nil { + t.Fatalf("VolumeNeedleStatus via EC shards failed: %v", err) + } + if statusResp.GetNeedleId() != needleID { + t.Fatalf("needle id mismatch: got %d want %d", statusResp.GetNeedleId(), needleID) + } + if statusResp.GetCookie() != cookie { + t.Fatalf("cookie mismatch: got %d want %d", statusResp.GetCookie(), cookie) + } + if statusResp.GetSize() == 0 { + t.Fatalf("expected non-zero needle size from EC-backed needle status") + } + + _, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: volumeID, + NeedleId: needleID + 999999, + }) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "not found") { + t.Fatalf("VolumeNeedleStatus via EC shards missing-needle error mismatch: %v", err) + } +} + +func TestVolumeNeedleStatusMissingVolumeAndNeedle(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(25) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: 99925, + NeedleId: 1, + }) + if err == nil { + t.Fatalf("VolumeNeedleStatus should fail for missing volume") + } + if !strings.Contains(strings.ToLower(err.Error()), "volume not found") { + t.Fatalf("VolumeNeedleStatus missing-volume error mismatch: %v", err) + } + + _, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: volumeID, + NeedleId: 123456789, + }) + if err == nil { + t.Fatalf("VolumeNeedleStatus should fail for missing needle") + } + if !strings.Contains(strings.ToLower(err.Error()), "not found") { + t.Fatalf("VolumeNeedleStatus missing-needle error mismatch: %v", err) + } +} + +func mustNewRequest(t testing.TB, method, url string) *http.Request { + t.Helper() + req, err := http.NewRequest(method, url, nil) + if err != nil { + t.Fatalf("create request %s %s: %v", method, url, err) + } + return req +} + +func TestVolumeConfigureInvalidReplication(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(22) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{ + VolumeId: volumeID, + Replication: "bad-replication", + }) + if err != nil { + t.Fatalf("VolumeConfigure returned grpc error: %v", err) + } + if resp.GetError() == "" { + t.Fatalf("VolumeConfigure expected response error for invalid replication") + } + if !strings.Contains(strings.ToLower(resp.GetError()), "replication") { + t.Fatalf("VolumeConfigure error should mention replication, got: %q", resp.GetError()) + } +} + +func TestVolumeConfigureSuccessAndMissingRollbackPath(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(24) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + successResp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{ + VolumeId: volumeID, + Replication: "000", + }) + if err != nil { + t.Fatalf("VolumeConfigure success path returned grpc error: %v", err) + } + if successResp.GetError() != "" { + t.Fatalf("VolumeConfigure success path expected empty response error, got: %q", successResp.GetError()) + } + + statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeStatus after successful configure failed: %v", err) + } + if statusResp.GetIsReadOnly() { + t.Fatalf("VolumeStatus after configure expected writable volume") + } + + missingResp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{ + VolumeId: 99024, + Replication: "000", + }) + if err != nil { + t.Fatalf("VolumeConfigure missing-volume branch should return response error, got grpc error: %v", err) + } + if missingResp.GetError() == "" { + t.Fatalf("VolumeConfigure missing-volume expected non-empty response error") + } + lower := strings.ToLower(missingResp.GetError()) + if !strings.Contains(lower, "not found on disk") { + t.Fatalf("VolumeConfigure missing-volume error should mention not found on disk, got: %q", missingResp.GetError()) + } + if !strings.Contains(lower, "failed to restore mount") { + t.Fatalf("VolumeConfigure missing-volume error should include remount rollback failure, got: %q", missingResp.GetError()) + } +} + +func TestPingVolumeTargetAndLeaveAffectsHealthz(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + pingResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ + TargetType: cluster.VolumeServerType, + Target: clusterHarness.VolumeServerAddress(), + }) + if err != nil { + t.Fatalf("Ping target volume server failed: %v", err) + } + if pingResp.GetRemoteTimeNs() == 0 { + t.Fatalf("expected remote timestamp from ping target volume server") + } + + if _, err = grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil { + t.Fatalf("VolumeServerLeave failed: %v", err) + } + + client := framework.NewHTTPClient() + healthURL := clusterHarness.VolumeAdminURL() + "/healthz" + deadline := time.Now().Add(5 * time.Second) + for { + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, healthURL)) + _ = framework.ReadAllAndClose(t, resp) + if resp.StatusCode == http.StatusServiceUnavailable { + return + } + if time.Now().After(deadline) { + t.Fatalf("expected healthz to return 503 after leave, got %d", resp.StatusCode) + } + time.Sleep(100 * time.Millisecond) + } +} + +func TestVolumeServerLeaveIsIdempotent(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if _, err := grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil { + t.Fatalf("first VolumeServerLeave failed: %v", err) + } + if _, err := grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil { + t.Fatalf("second VolumeServerLeave should be idempotent success, got: %v", err) + } + + client := framework.NewHTTPClient() + healthURL := clusterHarness.VolumeAdminURL() + "/healthz" + deadline := time.Now().Add(5 * time.Second) + for { + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, healthURL)) + _ = framework.ReadAllAndClose(t, resp) + if resp.StatusCode == http.StatusServiceUnavailable { + return + } + if time.Now().After(deadline) { + t.Fatalf("expected healthz to stay 503 after repeated leave, got %d", resp.StatusCode) + } + time.Sleep(100 * time.Millisecond) + } +} + +func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + unknownResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ + TargetType: "unknown-type", + Target: "127.0.0.1:12345", + }) + if err != nil { + t.Fatalf("Ping unknown target type should not return grpc error, got: %v", err) + } + if unknownResp.GetRemoteTimeNs() != 0 { + t.Fatalf("Ping unknown target type expected remote_time_ns=0, got %d", unknownResp.GetRemoteTimeNs()) + } + if unknownResp.GetStopTimeNs() < unknownResp.GetStartTimeNs() { + t.Fatalf("Ping unknown target type expected stop_time_ns >= start_time_ns") + } + + _, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ + TargetType: cluster.MasterType, + Target: "127.0.0.1:1", + }) + if err == nil { + t.Fatalf("Ping master target should fail when target is unreachable") + } + if !strings.Contains(err.Error(), "ping master") { + t.Fatalf("Ping master unreachable error mismatch: %v", err) + } + + _, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ + TargetType: cluster.FilerType, + Target: "127.0.0.1:1", + }) + if err == nil { + t.Fatalf("Ping filer target should fail when target is unreachable") + } + if !strings.Contains(err.Error(), "ping filer") { + t.Fatalf("Ping filer unreachable error mismatch: %v", err) + } +} + +func TestPingMasterTargetSuccess(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ + TargetType: cluster.MasterType, + Target: clusterHarness.MasterAddress(), + }) + if err != nil { + t.Fatalf("Ping master target success path failed: %v", err) + } + if resp.GetRemoteTimeNs() == 0 { + t.Fatalf("Ping master target expected non-zero remote time") + } + if resp.GetStopTimeNs() < resp.GetStartTimeNs() { + t.Fatalf("Ping master target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs()) + } +} + +func TestPingFilerTargetSuccess(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeClusterWithFiler(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ + TargetType: cluster.FilerType, + Target: clusterHarness.FilerServerAddress(), + }) + if err != nil { + t.Fatalf("Ping filer target success path failed: %v", err) + } + if resp.GetRemoteTimeNs() == 0 { + t.Fatalf("Ping filer target expected non-zero remote time") + } + if resp.GetStopTimeNs() < resp.GetStartTimeNs() { + t.Fatalf("Ping filer target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs()) + } +} diff --git a/test/volume_server/grpc/admin_lifecycle_test.go b/test/volume_server/grpc/admin_lifecycle_test.go new file mode 100644 index 000000000..bdc4e5a45 --- /dev/null +++ b/test/volume_server/grpc/admin_lifecycle_test.go @@ -0,0 +1,215 @@ +package volume_server_grpc_test + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestVolumeAdminLifecycleRPCs(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + const volumeID = uint32(11) + framework.AllocateVolume(t, client, volumeID, "") + + statusResp, err := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeStatus failed: %v", err) + } + if statusResp.GetFileCount() != 0 { + t.Fatalf("new volume should be empty, got file_count=%d", statusResp.GetFileCount()) + } + + if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil { + t.Fatalf("VolumeUnmount failed: %v", err) + } + if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil { + t.Fatalf("VolumeMount failed: %v", err) + } + + if _, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}); err != nil { + t.Fatalf("VolumeDelete failed: %v", err) + } + + _, err = client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err == nil { + t.Fatalf("VolumeStatus should fail after delete") + } + if st, ok := status.FromError(err); !ok || st.Code() == codes.OK { + t.Fatalf("VolumeStatus error should be a non-OK grpc status, got: %v", err) + } +} + +func TestVolumeDeleteOnlyEmptyVariants(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(13) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 66001, 0x11223344) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("volume-delete-only-empty")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}) + if err == nil || !strings.Contains(err.Error(), "volume not empty") { + t.Fatalf("VolumeDelete only_empty=true expected volume-not-empty error, got: %v", err) + } + + _, err = grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: false}) + if err != nil { + t.Fatalf("VolumeDelete only_empty=false failed: %v", err) + } + + _, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err == nil { + t.Fatalf("VolumeStatus should fail after non-empty delete with only_empty=false") + } +} + +func TestMaintenanceModeRejectsAllocateVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{VolumeId: 12, Replication: "000"}) + if err == nil { + t.Fatalf("AllocateVolume should fail when maintenance mode is enabled") + } + if !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("expected maintenance mode error, got: %v", err) + } +} + +func TestAllocateDuplicateAndMountUnmountMissingVariants(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + const missingVolumeID = uint32(99331) + const volumeID = uint32(14) + + if _, err := client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: missingVolumeID}); err != nil { + t.Fatalf("VolumeUnmount missing volume should be idempotent success, got: %v", err) + } + + _, err := client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: missingVolumeID}) + if err == nil { + t.Fatalf("VolumeMount missing volume should fail") + } + if !strings.Contains(err.Error(), "not found on disk") { + t.Fatalf("VolumeMount missing volume error mismatch: %v", err) + } + + framework.AllocateVolume(t, client, volumeID, "") + + _, err = client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{ + VolumeId: volumeID, + Replication: "000", + }) + if err == nil { + t.Fatalf("AllocateVolume duplicate should fail") + } + if !strings.Contains(strings.ToLower(err.Error()), "already exists") { + t.Fatalf("AllocateVolume duplicate error mismatch: %v", err) + } + + if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil { + t.Fatalf("VolumeUnmount existing volume failed: %v", err) + } + if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil { + t.Fatalf("VolumeUnmount already-unmounted volume should be idempotent success, got: %v", err) + } + if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil { + t.Fatalf("VolumeMount remount failed: %v", err) + } +} + +func TestMaintenanceModeRejectsVolumeDelete(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(15) + framework.AllocateVolume(t, client, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}) + if err == nil { + t.Fatalf("VolumeDelete should fail when maintenance mode is enabled") + } + if !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("expected maintenance mode error, got: %v", err) + } +} diff --git a/test/volume_server/grpc/admin_readonly_collection_test.go b/test/volume_server/grpc/admin_readonly_collection_test.go new file mode 100644 index 000000000..621309729 --- /dev/null +++ b/test/volume_server/grpc/admin_readonly_collection_test.go @@ -0,0 +1,177 @@ +package volume_server_grpc_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestVolumeMarkReadonlyAndWritableLifecycle(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(72) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{ + VolumeId: volumeID, + Persist: false, + }) + if err != nil { + t.Fatalf("VolumeMarkReadonly failed: %v", err) + } + + readOnlyStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeStatus after readonly failed: %v", err) + } + if !readOnlyStatus.GetIsReadOnly() { + t.Fatalf("VolumeStatus expected readonly=true after VolumeMarkReadonly") + } + + _, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeMarkWritable failed: %v", err) + } + + writableStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeStatus after writable failed: %v", err) + } + if writableStatus.GetIsReadOnly() { + t.Fatalf("VolumeStatus expected readonly=false after VolumeMarkWritable") + } +} + +func TestVolumeMarkReadonlyPersistTrue(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(74) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{ + VolumeId: volumeID, + Persist: true, + }) + if err != nil { + t.Fatalf("VolumeMarkReadonly persist=true failed: %v", err) + } + + statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeStatus after persist readonly failed: %v", err) + } + if !statusResp.GetIsReadOnly() { + t.Fatalf("VolumeStatus expected readonly=true after persist readonly") + } +} + +func TestVolumeMarkReadonlyWritableErrorPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 98771, Persist: true}) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeMarkReadonly missing-volume error mismatch: %v", err) + } + + _, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 98772}) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeMarkWritable missing-volume error mismatch: %v", err) + } + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: true, + Version: stateResp.GetState().GetVersion(), + }, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 1, Persist: true}) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeMarkReadonly maintenance error mismatch: %v", err) + } + + _, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 1}) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeMarkWritable maintenance error mismatch: %v", err) + } +} + +func TestDeleteCollectionRemovesVolumeAndIsIdempotent(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(73) + const collection = "it-delete-collection" + + framework.AllocateVolume(t, grpcClient, volumeID, collection) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeStatus before DeleteCollection failed: %v", err) + } + + _, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection}) + if err != nil { + t.Fatalf("DeleteCollection existing collection failed: %v", err) + } + + _, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err == nil { + t.Fatalf("VolumeStatus should fail after collection delete") + } + if !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("VolumeStatus after DeleteCollection error mismatch: %v", err) + } + + _, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection}) + if err != nil { + t.Fatalf("DeleteCollection idempotent retry failed: %v", err) + } +} diff --git a/test/volume_server/grpc/batch_delete_test.go b/test/volume_server/grpc/batch_delete_test.go new file mode 100644 index 000000000..b02d4ea27 --- /dev/null +++ b/test/volume_server/grpc/batch_delete_test.go @@ -0,0 +1,264 @@ +package volume_server_grpc_test + +import ( + "bytes" + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestBatchDeleteInvalidFidAndMaintenanceMode(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"bad-fid"}}) + if err != nil { + t.Fatalf("BatchDelete invalid fid should return response, got error: %v", err) + } + if len(resp.GetResults()) != 1 { + t.Fatalf("expected one batch delete result, got %d", len(resp.GetResults())) + } + if got := resp.GetResults()[0].GetStatus(); got != 400 { + t.Fatalf("invalid fid expected status 400, got %d", got) + } + + stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"1,1234567890ab"}}) + if err == nil { + t.Fatalf("BatchDelete should fail when maintenance mode is enabled") + } + if !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("expected maintenance mode error, got: %v", err) + } +} + +func TestBatchDeleteCookieMismatchAndSkipCheck(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(31) + const needleID = uint64(900001) + const correctCookie = uint32(0x1122AABB) + const wrongCookie = uint32(0x1122AABC) + framework.AllocateVolume(t, client, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, correctCookie) + uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, []byte("batch-delete-cookie-check")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + wrongCookieFid := framework.NewFileID(volumeID, needleID, wrongCookie) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + mismatchResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{ + FileIds: []string{wrongCookieFid}, + SkipCookieCheck: false, + }) + if err != nil { + t.Fatalf("BatchDelete with cookie check failed: %v", err) + } + if len(mismatchResp.GetResults()) != 1 { + t.Fatalf("BatchDelete cookie mismatch expected 1 result, got %d", len(mismatchResp.GetResults())) + } + if mismatchResp.GetResults()[0].GetStatus() != http.StatusBadRequest { + t.Fatalf("BatchDelete cookie mismatch expected status 400, got %d", mismatchResp.GetResults()[0].GetStatus()) + } + + skipCheckResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{ + FileIds: []string{wrongCookieFid}, + SkipCookieCheck: true, + }) + if err != nil { + t.Fatalf("BatchDelete skip cookie check failed: %v", err) + } + if len(skipCheckResp.GetResults()) != 1 { + t.Fatalf("BatchDelete skip check expected 1 result, got %d", len(skipCheckResp.GetResults())) + } + if skipCheckResp.GetResults()[0].GetStatus() != http.StatusAccepted { + t.Fatalf("BatchDelete skip check expected status 202, got %d", skipCheckResp.GetResults()[0].GetStatus()) + } + + readAfterDelete := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, readAfterDelete) + if readAfterDelete.StatusCode != http.StatusNotFound { + t.Fatalf("read after skip-check batch delete expected 404, got %d", readAfterDelete.StatusCode) + } +} + +func TestBatchDeleteMixedStatusesAndMismatchStopsProcessing(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(32) + framework.AllocateVolume(t, client, volumeID, "") + + const needleA = uint64(910001) + const needleB = uint64(910002) + const needleC = uint64(910003) + const cookieA = uint32(0x11111111) + const cookieB = uint32(0x22222222) + const cookieC = uint32(0x33333333) + + httpClient := framework.NewHTTPClient() + fidA := framework.NewFileID(volumeID, needleA, cookieA) + fidB := framework.NewFileID(volumeID, needleB, cookieB) + fidC := framework.NewFileID(volumeID, needleC, cookieC) + + for _, tc := range []struct { + fid string + body string + }{ + {fid: fidA, body: "batch-delete-mixed-a"}, + {fid: fidB, body: "batch-delete-mixed-b"}, + {fid: fidC, body: "batch-delete-mixed-c"}, + } { + uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, []byte(tc.body)) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload %s expected 201, got %d", tc.fid, uploadResp.StatusCode) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + missingFid := framework.NewFileID(volumeID, 919999, 0x44444444) + mixedResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{ + FileIds: []string{"bad-fid", fidA, missingFid}, + }) + if err != nil { + t.Fatalf("BatchDelete mixed status request failed: %v", err) + } + if len(mixedResp.GetResults()) != 3 { + t.Fatalf("BatchDelete mixed status expected 3 results, got %d", len(mixedResp.GetResults())) + } + if mixedResp.GetResults()[0].GetStatus() != http.StatusBadRequest { + t.Fatalf("BatchDelete mixed result[0] expected 400, got %d", mixedResp.GetResults()[0].GetStatus()) + } + if mixedResp.GetResults()[1].GetStatus() != http.StatusAccepted { + t.Fatalf("BatchDelete mixed result[1] expected 202, got %d", mixedResp.GetResults()[1].GetStatus()) + } + if mixedResp.GetResults()[2].GetStatus() != http.StatusNotFound { + t.Fatalf("BatchDelete mixed result[2] expected 404, got %d", mixedResp.GetResults()[2].GetStatus()) + } + + readDeletedA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA) + _ = framework.ReadAllAndClose(t, readDeletedA) + if readDeletedA.StatusCode != http.StatusNotFound { + t.Fatalf("fidA should be deleted after batch delete, got status %d", readDeletedA.StatusCode) + } + + wrongCookieB := framework.NewFileID(volumeID, needleB, cookieB+1) + stopResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{ + FileIds: []string{wrongCookieB, fidC}, + }) + if err != nil { + t.Fatalf("BatchDelete mismatch-stop request failed: %v", err) + } + if len(stopResp.GetResults()) != 1 { + t.Fatalf("BatchDelete mismatch-stop expected 1 result due early break, got %d", len(stopResp.GetResults())) + } + if stopResp.GetResults()[0].GetStatus() != http.StatusBadRequest { + t.Fatalf("BatchDelete mismatch-stop expected 400, got %d", stopResp.GetResults()[0].GetStatus()) + } + + readB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB) + _ = framework.ReadAllAndClose(t, readB) + if readB.StatusCode != http.StatusOK { + t.Fatalf("fidB should remain after cookie mismatch path, got %d", readB.StatusCode) + } + + readC := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidC) + _ = framework.ReadAllAndClose(t, readC) + if readC.StatusCode != http.StatusOK { + t.Fatalf("fidC should remain when batch processing stops on mismatch, got %d", readC.StatusCode) + } +} + +func TestBatchDeleteRejectsChunkManifestNeedles(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(33) + framework.AllocateVolume(t, client, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 920001, 0x5555AAAA) + req, err := http.NewRequest(http.MethodPost, cluster.VolumeAdminURL()+"/"+fid+"?cm=true", bytes.NewReader([]byte("manifest-placeholder-payload"))) + if err != nil { + t.Fatalf("create chunk manifest upload request: %v", err) + } + req.Header.Set("Content-Type", "application/octet-stream") + uploadResp := framework.DoRequest(t, httpClient, req) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("chunk manifest upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{fid}}) + if err != nil { + t.Fatalf("BatchDelete chunk manifest should return response, got grpc error: %v", err) + } + if len(resp.GetResults()) != 1 { + t.Fatalf("BatchDelete chunk manifest expected one result, got %d", len(resp.GetResults())) + } + if resp.GetResults()[0].GetStatus() != http.StatusNotAcceptable { + t.Fatalf("BatchDelete chunk manifest expected status 406, got %d", resp.GetResults()[0].GetStatus()) + } + if !strings.Contains(resp.GetResults()[0].GetError(), "ChunkManifest") { + t.Fatalf("BatchDelete chunk manifest expected error mentioning ChunkManifest, got %q", resp.GetResults()[0].GetError()) + } + + readResp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusOK { + t.Fatalf("chunk manifest should not be deleted by BatchDelete reject path, got %d", readResp.StatusCode) + } +} diff --git a/test/volume_server/grpc/copy_receive_variants_test.go b/test/volume_server/grpc/copy_receive_variants_test.go new file mode 100644 index 000000000..14d9cee72 --- /dev/null +++ b/test/volume_server/grpc/copy_receive_variants_test.go @@ -0,0 +1,431 @@ +package volume_server_grpc_test + +import ( + "context" + "io" + "math" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestVolumeIncrementalCopyDataAndNoDataPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(91) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 770001, 0x1122AABB) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("incremental-copy-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + dataStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{ + VolumeId: volumeID, + SinceNs: 0, + }) + if err != nil { + t.Fatalf("VolumeIncrementalCopy start failed: %v", err) + } + + totalBytes := 0 + for { + msg, recvErr := dataStream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("VolumeIncrementalCopy recv failed: %v", recvErr) + } + totalBytes += len(msg.GetFileContent()) + } + if totalBytes == 0 { + t.Fatalf("VolumeIncrementalCopy expected streamed bytes for since_ns=0") + } + + noDataStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{ + VolumeId: volumeID, + SinceNs: math.MaxUint64, + }) + if err != nil { + t.Fatalf("VolumeIncrementalCopy no-data start failed: %v", err) + } + _, err = noDataStream.Recv() + if err != io.EOF { + t.Fatalf("VolumeIncrementalCopy no-data expected EOF, got: %v", err) + } +} + +func TestCopyFileIgnoreNotFoundAndStopOffsetZeroPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(92) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + missingNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Ext: ".definitely-missing", + CompactionRevision: math.MaxUint32, + StopOffset: 1, + IgnoreSourceFileNotFound: false, + }) + if err == nil { + _, err = missingNoIgnore.Recv() + } + if err == nil { + t.Fatalf("CopyFile should fail for missing source file when ignore_source_file_not_found=false") + } + + missingIgnored, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Ext: ".definitely-missing", + CompactionRevision: math.MaxUint32, + StopOffset: 1, + IgnoreSourceFileNotFound: true, + }) + if err != nil { + t.Fatalf("CopyFile ignore-not-found start failed: %v", err) + } + _, err = missingIgnored.Recv() + if err != io.EOF { + t.Fatalf("CopyFile ignore-not-found expected EOF, got: %v", err) + } + + stopZeroStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Ext: ".definitely-missing", + CompactionRevision: math.MaxUint32, + StopOffset: 0, + IgnoreSourceFileNotFound: false, + }) + if err != nil { + t.Fatalf("CopyFile stop_offset=0 start failed: %v", err) + } + _, err = stopZeroStream.Recv() + if err != io.EOF { + t.Fatalf("CopyFile stop_offset=0 expected EOF, got: %v", err) + } +} + +func TestCopyFileCompactionRevisionMismatch(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(94) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Ext: ".idx", + CompactionRevision: 1, // fresh volume starts at revision 0 + StopOffset: 1, + }) + if err == nil { + _, err = stream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "is compacted") { + t.Fatalf("CopyFile compaction mismatch error mismatch: %v", err) + } +} + +func TestReceiveFileProtocolViolationResponses(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + contentFirstStream, err := grpcClient.ReceiveFile(ctx) + if err != nil { + t.Fatalf("ReceiveFile stream create failed: %v", err) + } + if err = contentFirstStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_FileContent{ + FileContent: []byte("content-before-info"), + }, + }); err != nil { + t.Fatalf("ReceiveFile send content-first failed: %v", err) + } + contentFirstResp, err := contentFirstStream.CloseAndRecv() + if err != nil { + t.Fatalf("ReceiveFile content-first close failed: %v", err) + } + if !strings.Contains(contentFirstResp.GetError(), "file info must be sent first") { + t.Fatalf("ReceiveFile content-first response mismatch: %+v", contentFirstResp) + } + + unknownTypeStream, err := grpcClient.ReceiveFile(ctx) + if err != nil { + t.Fatalf("ReceiveFile stream create for unknown-type failed: %v", err) + } + if err = unknownTypeStream.Send(&volume_server_pb.ReceiveFileRequest{}); err != nil { + t.Fatalf("ReceiveFile send unknown-type request failed: %v", err) + } + unknownTypeResp, err := unknownTypeStream.CloseAndRecv() + if err != nil { + t.Fatalf("ReceiveFile unknown-type close failed: %v", err) + } + if !strings.Contains(unknownTypeResp.GetError(), "unknown message type") { + t.Fatalf("ReceiveFile unknown-type response mismatch: %+v", unknownTypeResp) + } +} + +func TestReceiveFileSuccessForRegularVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(95) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + payloadA := []byte("receive-file-chunk-a:") + payloadB := []byte("receive-file-chunk-b") + expected := append(append([]byte{}, payloadA...), payloadB...) + + receiveStream, err := grpcClient.ReceiveFile(ctx) + if err != nil { + t.Fatalf("ReceiveFile stream create failed: %v", err) + } + + if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_Info{ + Info: &volume_server_pb.ReceiveFileInfo{ + VolumeId: volumeID, + Ext: ".tmprecv", + Collection: "", + IsEcVolume: false, + FileSize: uint64(len(expected)), + }, + }, + }); err != nil { + t.Fatalf("ReceiveFile send info failed: %v", err) + } + if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadA}, + }); err != nil { + t.Fatalf("ReceiveFile send payloadA failed: %v", err) + } + if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadB}, + }); err != nil { + t.Fatalf("ReceiveFile send payloadB failed: %v", err) + } + + resp, err := receiveStream.CloseAndRecv() + if err != nil { + t.Fatalf("ReceiveFile close failed: %v", err) + } + if resp.GetError() != "" { + t.Fatalf("ReceiveFile unexpected error response: %+v", resp) + } + if resp.GetBytesWritten() != uint64(len(expected)) { + t.Fatalf("ReceiveFile bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(expected)) + } + + copyStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Ext: ".tmprecv", + CompactionRevision: math.MaxUint32, + StopOffset: uint64(len(expected)), + }) + if err != nil { + t.Fatalf("CopyFile for received data start failed: %v", err) + } + + var copied []byte + for { + msg, recvErr := copyStream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("CopyFile for received data recv failed: %v", recvErr) + } + copied = append(copied, msg.GetFileContent()...) + } + + if string(copied) != string(expected) { + t.Fatalf("received file data mismatch: got %q want %q", string(copied), string(expected)) + } +} + +func TestReceiveFileSuccessForEcVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + const volumeID = uint32(96) + const collection = "ec-receive-success" + const ext = ".ec00" + + payloadA := []byte("receive-ec-file-chunk-a:") + payloadB := []byte("receive-ec-file-chunk-b") + expected := append(append([]byte{}, payloadA...), payloadB...) + + receiveStream, err := grpcClient.ReceiveFile(ctx) + if err != nil { + t.Fatalf("ReceiveFile stream create failed: %v", err) + } + + if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_Info{ + Info: &volume_server_pb.ReceiveFileInfo{ + VolumeId: volumeID, + Ext: ext, + Collection: collection, + IsEcVolume: true, + ShardId: 0, + FileSize: uint64(len(expected)), + }, + }, + }); err != nil { + t.Fatalf("ReceiveFile send EC info failed: %v", err) + } + if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadA}, + }); err != nil { + t.Fatalf("ReceiveFile send EC payloadA failed: %v", err) + } + if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadB}, + }); err != nil { + t.Fatalf("ReceiveFile send EC payloadB failed: %v", err) + } + + resp, err := receiveStream.CloseAndRecv() + if err != nil { + t.Fatalf("ReceiveFile EC close failed: %v", err) + } + if resp.GetError() != "" { + t.Fatalf("ReceiveFile EC unexpected error response: %+v", resp) + } + if resp.GetBytesWritten() != uint64(len(expected)) { + t.Fatalf("ReceiveFile EC bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(expected)) + } + + copyStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Collection: collection, + IsEcVolume: true, + Ext: ext, + CompactionRevision: math.MaxUint32, + StopOffset: uint64(len(expected)), + }) + if err != nil { + t.Fatalf("CopyFile for received EC data start failed: %v", err) + } + + var copied []byte + for { + msg, recvErr := copyStream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("CopyFile for received EC data recv failed: %v", recvErr) + } + copied = append(copied, msg.GetFileContent()...) + } + + if string(copied) != string(expected) { + t.Fatalf("received EC file data mismatch: got %q want %q", string(copied), string(expected)) + } +} + +func TestCopyFileEcVolumeIgnoreMissingSourcePaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + streamNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: 99601, + Collection: "ec-copy-missing", + IsEcVolume: true, + Ext: ".ec00", + CompactionRevision: math.MaxUint32, + StopOffset: 1, + IgnoreSourceFileNotFound: false, + }) + if err == nil { + _, err = streamNoIgnore.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found ec volume id") { + t.Fatalf("CopyFile EC missing source error mismatch: %v", err) + } + + streamIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: 99602, + Collection: "ec-copy-missing", + IsEcVolume: true, + Ext: ".ec00", + CompactionRevision: math.MaxUint32, + StopOffset: 1, + IgnoreSourceFileNotFound: true, + }) + if err != nil { + t.Fatalf("CopyFile EC ignore-missing start failed: %v", err) + } + _, err = streamIgnore.Recv() + if err != io.EOF { + t.Fatalf("CopyFile EC ignore-missing expected EOF, got: %v", err) + } +} diff --git a/test/volume_server/grpc/copy_sync_test.go b/test/volume_server/grpc/copy_sync_test.go new file mode 100644 index 000000000..3c2916fd0 --- /dev/null +++ b/test/volume_server/grpc/copy_sync_test.go @@ -0,0 +1,284 @@ +package volume_server_grpc_test + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestVolumeSyncStatusAndReadVolumeFileStatus(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(41) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + syncResp, err := grpcClient.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeSyncStatus failed: %v", err) + } + if syncResp.GetVolumeId() != volumeID { + t.Fatalf("VolumeSyncStatus volume id mismatch: got %d want %d", syncResp.GetVolumeId(), volumeID) + } + + statusResp, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("ReadVolumeFileStatus failed: %v", err) + } + if statusResp.GetVolumeId() != volumeID { + t.Fatalf("ReadVolumeFileStatus volume id mismatch: got %d want %d", statusResp.GetVolumeId(), volumeID) + } + if statusResp.GetVersion() == 0 { + t.Fatalf("ReadVolumeFileStatus expected non-zero version") + } +} + +func TestCopyAndStreamMethodsMissingVolumePaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{VolumeId: 98761}) + if err == nil { + t.Fatalf("VolumeSyncStatus should fail for missing volume") + } + + incrementalStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{VolumeId: 98762, SinceNs: 0}) + if err == nil { + _, err = incrementalStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("VolumeIncrementalCopy missing-volume error mismatch: %v", err) + } + + readAllStream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{98763}}) + if err == nil { + _, err = readAllStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", err) + } + + copyFileStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{VolumeId: 98764, Ext: ".dat", StopOffset: 1}) + if err == nil { + _, err = copyFileStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("CopyFile missing-volume error mismatch: %v", err) + } + + _, err = grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: 98765}) + if err == nil || !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("ReadVolumeFileStatus missing-volume error mismatch: %v", err) + } +} + +func TestVolumeCopyAndReceiveFileMaintenanceRejection(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + copyStream, err := grpcClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{VolumeId: 1, SourceDataNode: "127.0.0.1:1234"}) + if err == nil { + _, err = copyStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeCopy maintenance error mismatch: %v", err) + } + + receiveClient, err := grpcClient.ReceiveFile(ctx) + if err != nil { + t.Fatalf("ReceiveFile client creation failed: %v", err) + } + _ = receiveClient.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_Info{ + Info: &volume_server_pb.ReceiveFileInfo{VolumeId: 1, Ext: ".dat"}, + }, + }) + _, err = receiveClient.CloseAndRecv() + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("ReceiveFile maintenance error mismatch: %v", err) + } +} + +func TestVolumeCopySuccessFromPeerAndMountsDestination(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1()) + sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer sourceConn.Close() + destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) + defer destConn.Close() + + const volumeID = uint32(42) + framework.AllocateVolume(t, sourceClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 880001, 0x12345678) + payload := []byte("volume-copy-success-payload") + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload to source expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{ + VolumeId: volumeID, + Collection: "", + SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1], + }) + if err != nil { + t.Fatalf("VolumeCopy start failed: %v", err) + } + + sawFinalAppendTimestamp := false + for { + msg, recvErr := copyStream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("VolumeCopy recv failed: %v", recvErr) + } + if msg.GetLastAppendAtNs() > 0 { + sawFinalAppendTimestamp = true + } + } + if !sawFinalAppendTimestamp { + t.Fatalf("VolumeCopy expected final response with last_append_at_ns") + } + + destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid) + destReadBody := framework.ReadAllAndClose(t, destReadResp) + if destReadResp.StatusCode != http.StatusOK { + t.Fatalf("read from copied destination expected 200, got %d", destReadResp.StatusCode) + } + if string(destReadBody) != string(payload) { + t.Fatalf("destination copied payload mismatch: got %q want %q", string(destReadBody), string(payload)) + } +} + +func TestVolumeCopyOverwritesExistingDestinationVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1()) + sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer sourceConn.Close() + destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) + defer destConn.Close() + + const volumeID = uint32(43) + framework.AllocateVolume(t, sourceClient, volumeID, "") + framework.AllocateVolume(t, destClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 880002, 0x23456789) + sourcePayload := []byte("volume-copy-overwrite-source") + destPayload := []byte("volume-copy-overwrite-destination-old") + + sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, sourcePayload) + _ = framework.ReadAllAndClose(t, sourceUploadResp) + if sourceUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload to source expected 201, got %d", sourceUploadResp.StatusCode) + } + + destUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid, destPayload) + _ = framework.ReadAllAndClose(t, destUploadResp) + if destUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload to destination expected 201, got %d", destUploadResp.StatusCode) + } + + destReadBeforeResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid) + destReadBeforeBody := framework.ReadAllAndClose(t, destReadBeforeResp) + if destReadBeforeResp.StatusCode != http.StatusOK { + t.Fatalf("destination pre-copy read expected 200, got %d", destReadBeforeResp.StatusCode) + } + if string(destReadBeforeBody) != string(destPayload) { + t.Fatalf("destination pre-copy payload mismatch: got %q want %q", string(destReadBeforeBody), string(destPayload)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{ + VolumeId: volumeID, + Collection: "", + SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1], + }) + if err != nil { + t.Fatalf("VolumeCopy overwrite start failed: %v", err) + } + + sawFinalAppendTimestamp := false + for { + msg, recvErr := copyStream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("VolumeCopy overwrite recv failed: %v", recvErr) + } + if msg.GetLastAppendAtNs() > 0 { + sawFinalAppendTimestamp = true + } + } + if !sawFinalAppendTimestamp { + t.Fatalf("VolumeCopy overwrite expected final response with last_append_at_ns") + } + + destReadAfterResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid) + destReadAfterBody := framework.ReadAllAndClose(t, destReadAfterResp) + if destReadAfterResp.StatusCode != http.StatusOK { + t.Fatalf("destination post-copy read expected 200, got %d", destReadAfterResp.StatusCode) + } + if string(destReadAfterBody) != string(sourcePayload) { + t.Fatalf("destination post-copy payload mismatch: got %q want %q", string(destReadAfterBody), string(sourcePayload)) + } +} diff --git a/test/volume_server/grpc/data_rw_test.go b/test/volume_server/grpc/data_rw_test.go new file mode 100644 index 000000000..43969532d --- /dev/null +++ b/test/volume_server/grpc/data_rw_test.go @@ -0,0 +1,146 @@ +package volume_server_grpc_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestReadNeedleBlobAndMetaMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{ + VolumeId: 99111, + Offset: 0, + Size: 16, + }) + if err == nil { + t.Fatalf("ReadNeedleBlob should fail for missing volume") + } + if !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("ReadNeedleBlob missing volume error mismatch: %v", err) + } + + _, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{ + VolumeId: 99112, + NeedleId: 1, + Offset: 0, + Size: 16, + }) + if err == nil { + t.Fatalf("ReadNeedleMeta should fail for missing volume") + } + if !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("ReadNeedleMeta missing volume error mismatch: %v", err) + } +} + +func TestWriteNeedleBlobMaintenanceAndMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{ + VolumeId: 99113, + NeedleId: 1, + NeedleBlob: []byte("abc"), + Size: 3, + }) + if err == nil { + t.Fatalf("WriteNeedleBlob should fail for missing volume") + } + if !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("WriteNeedleBlob missing volume error mismatch: %v", err) + } + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{ + VolumeId: 1, + NeedleId: 2, + NeedleBlob: []byte("def"), + Size: 3, + }) + if err == nil { + t.Fatalf("WriteNeedleBlob should fail in maintenance mode") + } + if !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("WriteNeedleBlob maintenance mode error mismatch: %v", err) + } +} + +func TestReadNeedleBlobAndMetaInvalidOffsets(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(92) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 880001, 0xCCDD1122) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("invalid-offset-check")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{ + VolumeId: volumeID, + Offset: 1 << 40, + Size: 64, + }) + if err == nil { + t.Fatalf("ReadNeedleBlob should fail for invalid offset") + } + if !strings.Contains(strings.ToLower(err.Error()), "read needle blob") { + t.Fatalf("ReadNeedleBlob invalid offset error mismatch: %v", err) + } + + _, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{ + VolumeId: volumeID, + NeedleId: 880001, + Offset: 1 << 40, + Size: 64, + }) + if err == nil { + t.Fatalf("ReadNeedleMeta should fail for invalid offset") + } +} diff --git a/test/volume_server/grpc/data_stream_success_test.go b/test/volume_server/grpc/data_stream_success_test.go new file mode 100644 index 000000000..90f2a8248 --- /dev/null +++ b/test/volume_server/grpc/data_stream_success_test.go @@ -0,0 +1,273 @@ +package volume_server_grpc_test + +import ( + "context" + "io" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/idx" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(83) + const sourceNeedleID = uint64(333333) + const sourceCookie = uint32(0xABCD0102) + const clonedNeedleID = uint64(333334) + + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + payload := []byte("blob-roundtrip-content") + fid := framework.NewFileID(volumeID, sourceNeedleID, sourceCookie) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + fileStatus, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("ReadVolumeFileStatus failed: %v", err) + } + if fileStatus.GetIdxFileSize() == 0 { + t.Fatalf("expected non-zero idx file size after upload") + } + + idxBytes := copyFileBytes(t, grpcClient, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Ext: ".idx", + CompactionRevision: fileStatus.GetCompactionRevision(), + StopOffset: fileStatus.GetIdxFileSize(), + }) + offset, size := findNeedleOffsetAndSize(t, idxBytes, sourceNeedleID) + + blobResp, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{ + VolumeId: volumeID, + Offset: offset, + Size: size, + }) + if err != nil { + t.Fatalf("ReadNeedleBlob failed: %v", err) + } + if len(blobResp.GetNeedleBlob()) == 0 { + t.Fatalf("ReadNeedleBlob returned empty blob") + } + + metaResp, err := grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{ + VolumeId: volumeID, + NeedleId: sourceNeedleID, + Offset: offset, + Size: size, + }) + if err != nil { + t.Fatalf("ReadNeedleMeta failed: %v", err) + } + if metaResp.GetCookie() != sourceCookie { + t.Fatalf("ReadNeedleMeta cookie mismatch: got %d want %d", metaResp.GetCookie(), sourceCookie) + } + + _, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{ + VolumeId: volumeID, + NeedleId: clonedNeedleID, + Size: size, + NeedleBlob: blobResp.GetNeedleBlob(), + }) + if err != nil { + t.Fatalf("WriteNeedleBlob failed: %v", err) + } + + clonedStatus, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: volumeID, + NeedleId: clonedNeedleID, + }) + if err != nil { + t.Fatalf("VolumeNeedleStatus for cloned needle failed: %v", err) + } + if clonedStatus.GetNeedleId() != sourceNeedleID { + t.Fatalf("cloned needle status id mismatch: got %d want %d", clonedStatus.GetNeedleId(), sourceNeedleID) + } + if clonedStatus.GetCookie() != sourceCookie { + t.Fatalf("cloned needle cookie mismatch: got %d want %d", clonedStatus.GetCookie(), sourceCookie) + } + + clonedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, clonedNeedleID, sourceCookie)) + clonedReadBody := framework.ReadAllAndClose(t, clonedReadResp) + if clonedReadResp.StatusCode != 200 { + t.Fatalf("cloned needle GET expected 200, got %d", clonedReadResp.StatusCode) + } + if string(clonedReadBody) != string(payload) { + t.Fatalf("cloned needle body mismatch: got %q want %q", string(clonedReadBody), string(payload)) + } +} + +func TestReadAllNeedlesStreamsUploadedRecords(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(84) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + expected := map[uint64]string{ + 444441: "read-all-needle-one", + 444442: "read-all-needle-two", + } + for key, body := range expected { + resp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, key, 0xA0B0C0D0), []byte(body)) + _ = framework.ReadAllAndClose(t, resp) + if resp.StatusCode != 201 { + t.Fatalf("upload for key %d expected 201, got %d", key, resp.StatusCode) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{volumeID}}) + if err != nil { + t.Fatalf("ReadAllNeedles start failed: %v", err) + } + + seen := map[uint64]string{} + for { + msg, recvErr := stream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("ReadAllNeedles recv failed: %v", recvErr) + } + if _, wanted := expected[msg.GetNeedleId()]; wanted { + seen[msg.GetNeedleId()] = string(msg.GetNeedleBlob()) + } + } + + for key, body := range expected { + got, found := seen[key] + if !found { + t.Fatalf("ReadAllNeedles missing key %d in stream", key) + } + if got != body { + t.Fatalf("ReadAllNeedles body mismatch for key %d: got %q want %q", key, got, body) + } + } +} + +func TestReadAllNeedlesExistingThenMissingVolumeAbortsStream(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const existingVolumeID = uint32(85) + const missingVolumeID = uint32(98585) + const needleID = uint64(445551) + framework.AllocateVolume(t, grpcClient, existingVolumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(existingVolumeID, needleID, 0xAA11BB22) + payload := "read-all-existing-then-missing" + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte(payload)) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{ + VolumeIds: []uint32{existingVolumeID, missingVolumeID}, + }) + if err != nil { + t.Fatalf("ReadAllNeedles start failed: %v", err) + } + + seenUploadedNeedle := false + for { + msg, recvErr := stream.Recv() + if recvErr == io.EOF { + t.Fatalf("ReadAllNeedles expected stream error for missing volume, got EOF") + } + if recvErr != nil { + if !strings.Contains(recvErr.Error(), "not found volume id") { + t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", recvErr) + } + break + } + if msg.GetNeedleId() == needleID && string(msg.GetNeedleBlob()) == payload { + seenUploadedNeedle = true + } + } + + if !seenUploadedNeedle { + t.Fatalf("ReadAllNeedles should stream entries from existing volume before missing-volume abort") + } +} + +func copyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.CopyFile(ctx, req) + if err != nil { + t.Fatalf("CopyFile start failed: %v", err) + } + + var out []byte + for { + msg, recvErr := stream.Recv() + if recvErr == io.EOF { + return out + } + if recvErr != nil { + t.Fatalf("CopyFile recv failed: %v", recvErr) + } + out = append(out, msg.GetFileContent()...) + } +} + +func findNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) { + t.Helper() + + for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize { + key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize]) + if uint64(key) != needleID { + continue + } + if entryOffset.IsZero() || entrySize <= 0 { + continue + } + return entryOffset.ToActualOffset(), int32(entrySize) + } + + t.Fatalf("needle id %d not found in idx entries", needleID) + return 0, 0 +} diff --git a/test/volume_server/grpc/erasure_coding_test.go b/test/volume_server/grpc/erasure_coding_test.go new file mode 100644 index 000000000..8a0d8f75f --- /dev/null +++ b/test/volume_server/grpc/erasure_coding_test.go @@ -0,0 +1,777 @@ +package volume_server_grpc_test + +import ( + "context" + "io" + "math" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestEcMaintenanceModeRejections(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: true, + Version: stateResp.GetState().GetVersion(), + }, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: 1, Collection: ""}) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeEcShardsGenerate maintenance error mismatch: %v", err) + } + + _, err = grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: 1, + Collection: "", + SourceDataNode: "127.0.0.1:1", + ShardIds: []uint32{0}, + }) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeEcShardsCopy maintenance error mismatch: %v", err) + } + + _, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: 1, + Collection: "", + ShardIds: []uint32{0}, + }) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeEcShardsDelete maintenance error mismatch: %v", err) + } + + _, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{ + VolumeId: 1, + Collection: "", + FileKey: 1, + Version: 3, + }) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeEcBlobDelete maintenance error mismatch: %v", err) + } + + _, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ + VolumeId: 1, + Collection: "", + }) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeEcShardsToVolume maintenance error mismatch: %v", err) + } +} + +func TestEcMissingInvalidAndNoopPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: 98791, + Collection: "", + }) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeEcShardsGenerate missing-volume error mismatch: %v", err) + } + + rebuildResp, err := grpcClient.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{ + VolumeId: 98792, + Collection: "ec-rebuild", + }) + if err != nil { + t.Fatalf("VolumeEcShardsRebuild missing-volume should return empty success, got: %v", err) + } + if len(rebuildResp.GetRebuiltShardIds()) != 0 { + t.Fatalf("VolumeEcShardsRebuild expected no rebuilt shards for missing volume, got %v", rebuildResp.GetRebuiltShardIds()) + } + + _, err = grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: 98793, + Collection: "ec-copy", + SourceDataNode: "127.0.0.1:1", + ShardIds: []uint32{0}, + DiskId: 99, + }) + if err == nil || !strings.Contains(err.Error(), "invalid disk_id") { + t.Fatalf("VolumeEcShardsCopy invalid-disk error mismatch: %v", err) + } + + _, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: 98794, + Collection: "ec-delete", + ShardIds: []uint32{0, 1}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsDelete missing-volume should be no-op success, got: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: 98795, + Collection: "ec-mount", + ShardIds: []uint32{0}, + }) + if err == nil { + t.Fatalf("VolumeEcShardsMount should fail for missing EC shards") + } + + _, err = grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{ + VolumeId: 98796, + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsUnmount missing shards should be no-op success, got: %v", err) + } + + readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{ + VolumeId: 98797, + ShardId: 0, + Offset: 0, + Size: 1, + }) + if err == nil { + _, err = readStream.Recv() + } + if err == nil || err == io.EOF { + t.Fatalf("VolumeEcShardRead should fail for missing EC volume") + } + + _, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{ + VolumeId: 98798, + Collection: "ec-blob", + FileKey: 1, + Version: 3, + }) + if err != nil { + t.Fatalf("VolumeEcBlobDelete missing local EC volume should be no-op success, got: %v", err) + } + + _, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ + VolumeId: 98799, + Collection: "ec-to-volume", + }) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeEcShardsToVolume missing-volume error mismatch: %v", err) + } + + _, err = grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{ + VolumeId: 98800, + }) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeEcShardsInfo missing-volume error mismatch: %v", err) + } +} + +func TestEcGenerateMountInfoUnmountLifecycle(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(115) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 990001, 0x1234ABCD) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-generate-lifecycle-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate success path failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount success path failed: %v", err) + } + + infoResp, err := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{ + VolumeId: volumeID, + }) + if err != nil { + t.Fatalf("VolumeEcShardsInfo after mount failed: %v", err) + } + if len(infoResp.GetEcShardInfos()) == 0 { + t.Fatalf("VolumeEcShardsInfo expected non-empty shard infos after mount") + } + if infoResp.GetVolumeSize() == 0 { + t.Fatalf("VolumeEcShardsInfo expected non-zero volume size after mount") + } + + _, err = grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{ + VolumeId: volumeID, + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsUnmount success path failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{ + VolumeId: volumeID, + }) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeEcShardsInfo after unmount expected not-found error, got: %v", err) + } +} + +func TestEcShardReadAndBlobDeleteLifecycle(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(116) + const fileKey = uint64(990002) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, fileKey, 0x2233CCDD) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-shard-read-delete-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount failed: %v", err) + } + + readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{ + VolumeId: volumeID, + ShardId: 0, + Offset: 0, + Size: 1, + }) + if err != nil { + t.Fatalf("VolumeEcShardRead start failed: %v", err) + } + firstChunk, err := readStream.Recv() + if err != nil { + t.Fatalf("VolumeEcShardRead recv failed: %v", err) + } + if len(firstChunk.GetData()) == 0 { + t.Fatalf("VolumeEcShardRead expected non-empty data chunk before deletion") + } + + _, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{ + VolumeId: volumeID, + Collection: "", + FileKey: fileKey, + Version: uint32(needle.GetCurrentVersion()), + }) + if err != nil { + t.Fatalf("VolumeEcBlobDelete first delete failed: %v", err) + } + + _, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{ + VolumeId: volumeID, + Collection: "", + FileKey: fileKey, + Version: uint32(needle.GetCurrentVersion()), + }) + if err != nil { + t.Fatalf("VolumeEcBlobDelete second delete should be idempotent success, got: %v", err) + } + + deletedStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{ + VolumeId: volumeID, + ShardId: 0, + FileKey: fileKey, + Offset: 0, + Size: 1, + }) + if err != nil { + t.Fatalf("VolumeEcShardRead deleted-check start failed: %v", err) + } + deletedMsg, err := deletedStream.Recv() + if err != nil { + t.Fatalf("VolumeEcShardRead deleted-check recv failed: %v", err) + } + if !deletedMsg.GetIsDeleted() { + t.Fatalf("VolumeEcShardRead expected IsDeleted=true after blob delete") + } + _, err = deletedStream.Recv() + if err != io.EOF { + t.Fatalf("VolumeEcShardRead deleted-check expected EOF after deleted marker, got: %v", err) + } +} + +func TestEcRebuildMissingShardLifecycle(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(117) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 990003, 0x3344DDEE) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-rebuild-shard-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsDelete shard 0 failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0}, + }) + if err == nil { + t.Fatalf("VolumeEcShardsMount should fail when shard 0 has been deleted") + } + + rebuildResp, err := grpcClient.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsRebuild failed: %v", err) + } + if len(rebuildResp.GetRebuiltShardIds()) == 0 { + t.Fatalf("VolumeEcShardsRebuild expected rebuilt shard ids") + } + foundShard0 := false + for _, shardID := range rebuildResp.GetRebuiltShardIds() { + if shardID == 0 { + foundShard0 = true + break + } + } + if !foundShard0 { + t.Fatalf("VolumeEcShardsRebuild expected shard 0 to be rebuilt, got %v", rebuildResp.GetRebuiltShardIds()) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount shard 0 after rebuild failed: %v", err) + } +} + +func TestEcShardsToVolumeMissingShardAndNoLiveEntries(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + httpClient := framework.NewHTTPClient() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + t.Run("missing shard returns error", func(t *testing.T) { + const volumeID = uint32(118) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 990004, 0x4455EEFF) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-to-volume-missing-shard-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsDelete shard 0 failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{1}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount shard 1 failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err == nil || !strings.Contains(err.Error(), "missing shard 0") { + t.Fatalf("VolumeEcShardsToVolume missing-shard error mismatch: %v", err) + } + }) + + t.Run("no live entries returns failed precondition", func(t *testing.T) { + const volumeID = uint32(119) + const needleID = uint64(990005) + const cookie = uint32(0x5566FF11) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-no-live-entries-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + deleteResp := framework.DoRequest(t, httpClient, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid)) + _ = framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode) + } + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount data shards failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err == nil { + t.Fatalf("VolumeEcShardsToVolume expected failed-precondition error when no live entries") + } + if status.Code(err) != codes.FailedPrecondition { + t.Fatalf("VolumeEcShardsToVolume no-live-entries expected FailedPrecondition, got %v (%v)", status.Code(err), err) + } + if !strings.Contains(err.Error(), erasure_coding.EcNoLiveEntriesSubstring) { + t.Fatalf("VolumeEcShardsToVolume no-live-entries error should mention %q, got %v", erasure_coding.EcNoLiveEntriesSubstring, err) + } + }) +} + +func TestEcShardsToVolumeSuccessRoundTrip(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(120) + const needleID = uint64(990006) + const cookie = uint32(0x66771122) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, cookie) + payload := []byte("ec-shards-to-volume-success-roundtrip-content") + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount data shards failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsToVolume success path failed: %v", err) + } + + readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid) + readBody := framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusOK { + t.Fatalf("post-conversion read expected 200, got %d", readResp.StatusCode) + } + if string(readBody) != string(payload) { + t.Fatalf("post-conversion payload mismatch: got %q want %q", string(readBody), string(payload)) + } +} + +func TestEcShardsDeleteLastShardRemovesEcx(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(121) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 990007, 0x77882233) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-delete-all-shards-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + // Verify .ecx is present before deleting all shards. + ecxBeforeDelete, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Collection: "", + IsEcVolume: true, + Ext: ".ecx", + CompactionRevision: math.MaxUint32, + StopOffset: 1, + }) + if err != nil { + t.Fatalf("CopyFile .ecx before shard deletion start failed: %v", err) + } + if _, err = ecxBeforeDelete.Recv(); err != nil { + t.Fatalf("CopyFile .ecx before shard deletion recv failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsDelete all shards failed: %v", err) + } + + ecxAfterDelete, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Collection: "", + IsEcVolume: true, + Ext: ".ecx", + CompactionRevision: math.MaxUint32, + StopOffset: 1, + }) + if err == nil { + _, err = ecxAfterDelete.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found ec volume id") { + t.Fatalf("CopyFile .ecx after deleting all shards should fail not-found, got: %v", err) + } +} + +func TestEcShardsCopyFromPeerSuccess(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1()) + sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer sourceConn.Close() + destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) + defer destConn.Close() + + const volumeID = uint32(122) + framework.AllocateVolume(t, sourceClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 990008, 0x88993344) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, []byte("ec-copy-from-peer-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("source upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := sourceClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("source VolumeEcShardsGenerate failed: %v", err) + } + + sourceDataNode := clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1] + _, err = destClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: volumeID, + Collection: "", + SourceDataNode: sourceDataNode, + ShardIds: []uint32{0}, + CopyEcxFile: true, + CopyVifFile: true, + }) + if err != nil { + t.Fatalf("destination VolumeEcShardsCopy success path failed: %v", err) + } + + for _, ext := range []string{".ec00", ".ecx", ".vif"} { + copyStream, copyErr := destClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Collection: "", + IsEcVolume: true, + Ext: ext, + CompactionRevision: math.MaxUint32, + StopOffset: 1, + }) + if copyErr != nil { + t.Fatalf("destination CopyFile %s start failed: %v", ext, copyErr) + } + if _, copyErr = copyStream.Recv(); copyErr != nil { + t.Fatalf("destination CopyFile %s recv failed: %v", ext, copyErr) + } + } +} + +func TestEcShardsCopyFailsWhenSourceUnavailable(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: 12345, + Collection: "", + SourceDataNode: "127.0.0.1:1.1", + ShardIds: []uint32{0}, + CopyEcxFile: true, + }) + if err == nil || !strings.Contains(err.Error(), "VolumeEcShardsCopy volume") { + t.Fatalf("VolumeEcShardsCopy source-unavailable error mismatch: %v", err) + } +} diff --git a/test/volume_server/grpc/health_state_test.go b/test/volume_server/grpc/health_state_test.go new file mode 100644 index 000000000..cac40731b --- /dev/null +++ b/test/volume_server/grpc/health_state_test.go @@ -0,0 +1,139 @@ +package volume_server_grpc_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestStateAndStatusRPCs(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + initialState, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + if initialState.GetState() == nil { + t.Fatalf("GetState returned nil state") + } + + setResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: true, + Version: initialState.GetState().GetVersion(), + }, + }) + if err != nil { + t.Fatalf("SetState(maintenance=true) failed: %v", err) + } + if !setResp.GetState().GetMaintenance() { + t.Fatalf("expected maintenance=true after SetState") + } + + setResp, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: false, + Version: setResp.GetState().GetVersion(), + }, + }) + if err != nil { + t.Fatalf("SetState(maintenance=false) failed: %v", err) + } + if setResp.GetState().GetMaintenance() { + t.Fatalf("expected maintenance=false after SetState") + } + + statusResp, err := client.VolumeServerStatus(ctx, &volume_server_pb.VolumeServerStatusRequest{}) + if err != nil { + t.Fatalf("VolumeServerStatus failed: %v", err) + } + if statusResp.GetVersion() == "" { + t.Fatalf("VolumeServerStatus returned empty version") + } + if len(statusResp.GetDiskStatuses()) == 0 { + t.Fatalf("VolumeServerStatus returned no disk statuses") + } + if statusResp.GetState() == nil { + t.Fatalf("VolumeServerStatus returned nil state") + } + if statusResp.GetMemoryStatus() == nil { + t.Fatalf("VolumeServerStatus returned nil memory status") + } + if statusResp.GetMemoryStatus().GetGoroutines() <= 0 { + t.Fatalf("VolumeServerStatus memory status should report goroutines, got %d", statusResp.GetMemoryStatus().GetGoroutines()) + } + + pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{}) + if err != nil { + t.Fatalf("Ping failed: %v", err) + } + if pingResp.GetStartTimeNs() == 0 || pingResp.GetStopTimeNs() == 0 { + t.Fatalf("Ping timestamps should be non-zero: %+v", pingResp) + } + if pingResp.GetStopTimeNs() < pingResp.GetStartTimeNs() { + t.Fatalf("Ping stop time should be >= start time: %+v", pingResp) + } +} + +func TestSetStateVersionMismatchAndNilStateNoop(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + initialState, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + initialVersion := initialState.GetState().GetVersion() + + staleResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: true, + Version: initialVersion + 1, + }, + }) + if err == nil { + t.Fatalf("SetState with stale version should fail") + } + if !strings.Contains(err.Error(), "version mismatch") { + t.Fatalf("SetState stale version error mismatch: %v", err) + } + if staleResp.GetState().GetVersion() != initialVersion { + t.Fatalf("SetState stale version should not mutate server version: got %d want %d", staleResp.GetState().GetVersion(), initialVersion) + } + if staleResp.GetState().GetMaintenance() != initialState.GetState().GetMaintenance() { + t.Fatalf("SetState stale version should not mutate maintenance flag") + } + + nilResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{}) + if err != nil { + t.Fatalf("SetState nil-state request should be no-op success: %v", err) + } + if nilResp.GetState().GetVersion() != initialVersion { + t.Fatalf("SetState nil-state should keep version unchanged: got %d want %d", nilResp.GetState().GetVersion(), initialVersion) + } + if nilResp.GetState().GetMaintenance() != initialState.GetState().GetMaintenance() { + t.Fatalf("SetState nil-state should keep maintenance unchanged") + } +} diff --git a/test/volume_server/grpc/scrub_query_test.go b/test/volume_server/grpc/scrub_query_test.go new file mode 100644 index 000000000..66766f79d --- /dev/null +++ b/test/volume_server/grpc/scrub_query_test.go @@ -0,0 +1,385 @@ +package volume_server_grpc_test + +import ( + "context" + "io" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestScrubVolumeIndexAndUnsupportedMode(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(61) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + indexResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ + VolumeIds: []uint32{volumeID}, + Mode: volume_server_pb.VolumeScrubMode_INDEX, + }) + if err != nil { + t.Fatalf("ScrubVolume index mode failed: %v", err) + } + if indexResp.GetTotalVolumes() != 1 { + t.Fatalf("ScrubVolume expected total_volumes=1, got %d", indexResp.GetTotalVolumes()) + } + + _, err = grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ + VolumeIds: []uint32{volumeID}, + Mode: volume_server_pb.VolumeScrubMode(99), + }) + if err == nil { + t.Fatalf("ScrubVolume should fail for unsupported mode") + } + if !strings.Contains(err.Error(), "unsupported volume scrub mode") { + t.Fatalf("ScrubVolume unsupported mode error mismatch: %v", err) + } +} + +func TestScrubEcVolumeMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{ + VolumeIds: []uint32{98765}, + Mode: volume_server_pb.VolumeScrubMode_INDEX, + }) + if err == nil { + t.Fatalf("ScrubEcVolume should fail for missing EC volume") + } + if !strings.Contains(err.Error(), "EC volume id") { + t.Fatalf("ScrubEcVolume missing-volume error mismatch: %v", err) + } +} + +func TestScrubEcVolumeAutoSelectNoEcVolumes(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{ + Mode: volume_server_pb.VolumeScrubMode_INDEX, + }) + if err != nil { + t.Fatalf("ScrubEcVolume auto-select failed: %v", err) + } + if resp.GetTotalVolumes() != 0 { + t.Fatalf("ScrubEcVolume auto-select expected total_volumes=0 without EC data, got %d", resp.GetTotalVolumes()) + } + if len(resp.GetBrokenVolumeIds()) != 0 { + t.Fatalf("ScrubEcVolume auto-select expected no broken volumes, got %v", resp.GetBrokenVolumeIds()) + } +} + +func TestQueryInvalidAndMissingFileIDPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + invalidStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ + FromFileIds: []string{"bad-fid"}, + Selections: []string{"name"}, + Filter: &volume_server_pb.QueryRequest_Filter{}, + InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ + JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{}, + }, + }) + if err == nil { + _, err = invalidStream.Recv() + } + if err == nil { + t.Fatalf("Query should fail for invalid file id") + } + + missingFid := framework.NewFileID(98766, 1, 1) + missingStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ + FromFileIds: []string{missingFid}, + Selections: []string{"name"}, + Filter: &volume_server_pb.QueryRequest_Filter{}, + InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ + JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{}, + }, + }) + if err == nil { + _, err = missingStream.Recv() + } + if err == nil { + t.Fatalf("Query should fail for missing file id volume") + } +} + +func TestScrubVolumeAutoSelectAndNotImplementedModes(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeIDA = uint32(62) + const volumeIDB = uint32(63) + framework.AllocateVolume(t, grpcClient, volumeIDA, "") + framework.AllocateVolume(t, grpcClient, volumeIDB, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + autoResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ + Mode: volume_server_pb.VolumeScrubMode_INDEX, + }) + if err != nil { + t.Fatalf("ScrubVolume auto-select failed: %v", err) + } + if autoResp.GetTotalVolumes() < 2 { + t.Fatalf("ScrubVolume auto-select expected at least 2 volumes, got %d", autoResp.GetTotalVolumes()) + } + + localResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ + VolumeIds: []uint32{volumeIDA}, + Mode: volume_server_pb.VolumeScrubMode_LOCAL, + }) + if err != nil { + t.Fatalf("ScrubVolume local mode failed: %v", err) + } + if localResp.GetTotalVolumes() != 1 { + t.Fatalf("ScrubVolume local mode expected total_volumes=1, got %d", localResp.GetTotalVolumes()) + } + if len(localResp.GetBrokenVolumeIds()) != 1 || localResp.GetBrokenVolumeIds()[0] != volumeIDA { + t.Fatalf("ScrubVolume local mode expected broken volume %d, got %v", volumeIDA, localResp.GetBrokenVolumeIds()) + } + if len(localResp.GetDetails()) == 0 || !strings.Contains(strings.Join(localResp.GetDetails(), " "), "not implemented") { + t.Fatalf("ScrubVolume local mode expected not-implemented details, got %v", localResp.GetDetails()) + } + + fullResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ + VolumeIds: []uint32{volumeIDA}, + Mode: volume_server_pb.VolumeScrubMode_FULL, + }) + if err != nil { + t.Fatalf("ScrubVolume full mode failed: %v", err) + } + if fullResp.GetTotalVolumes() != 1 { + t.Fatalf("ScrubVolume full mode expected total_volumes=1, got %d", fullResp.GetTotalVolumes()) + } + if len(fullResp.GetDetails()) == 0 || !strings.Contains(strings.Join(fullResp.GetDetails(), " "), "not implemented") { + t.Fatalf("ScrubVolume full mode expected not-implemented details, got %v", fullResp.GetDetails()) + } +} + +func TestQueryJsonSuccessAndCsvNoOutput(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(64) + const needleID = uint64(777001) + const cookie = uint32(0xAABBCCDD) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + jsonLines := []byte("{\"score\":3}\n{\"score\":12}\n{\"score\":18}\n") + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, cookie) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + queryStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ + FromFileIds: []string{fid}, + Selections: []string{"score"}, + Filter: &volume_server_pb.QueryRequest_Filter{ + Field: "score", + Operand: ">", + Value: "10", + }, + InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ + JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"}, + }, + }) + if err != nil { + t.Fatalf("Query json start failed: %v", err) + } + + firstStripe, err := queryStream.Recv() + if err != nil { + t.Fatalf("Query json recv failed: %v", err) + } + records := string(firstStripe.GetRecords()) + if !strings.Contains(records, "score:12") || !strings.Contains(records, "score:18") { + t.Fatalf("Query json records missing expected filtered scores: %q", records) + } + if strings.Contains(records, "score:3") { + t.Fatalf("Query json records should not include filtered-out score: %q", records) + } + _, err = queryStream.Recv() + if err != io.EOF { + t.Fatalf("Query json expected EOF after first stripe, got: %v", err) + } + + csvStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ + FromFileIds: []string{fid}, + Selections: []string{"score"}, + Filter: &volume_server_pb.QueryRequest_Filter{}, + InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ + CsvInput: &volume_server_pb.QueryRequest_InputSerialization_CSVInput{}, + }, + }) + if err != nil { + t.Fatalf("Query csv start failed: %v", err) + } + _, err = csvStream.Recv() + if err != io.EOF { + t.Fatalf("Query csv expected EOF with no rows, got: %v", err) + } +} + +func TestQueryJsonNoMatchReturnsEmptyStripe(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(65) + const needleID = uint64(777002) + const cookie = uint32(0xABABCDCD) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + jsonLines := []byte("{\"score\":1}\n{\"score\":2}\n") + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, cookie) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + queryStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ + FromFileIds: []string{fid}, + Selections: []string{"score"}, + Filter: &volume_server_pb.QueryRequest_Filter{ + Field: "score", + Operand: ">", + Value: "100", + }, + InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ + JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"}, + }, + }) + if err != nil { + t.Fatalf("Query json no-match start failed: %v", err) + } + + firstStripe, err := queryStream.Recv() + if err != nil { + t.Fatalf("Query json no-match recv failed: %v", err) + } + if len(firstStripe.GetRecords()) != 0 { + t.Fatalf("Query json no-match expected empty records stripe, got: %q", string(firstStripe.GetRecords())) + } + + _, err = queryStream.Recv() + if err != io.EOF { + t.Fatalf("Query json no-match expected EOF after first empty stripe, got: %v", err) + } +} + +func TestQueryCookieMismatchReturnsEOFNoResults(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(66) + const needleID = uint64(777003) + const cookie = uint32(0xCDCDABAB) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + jsonLines := []byte("{\"score\":7}\n{\"score\":8}\n") + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, cookie) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + wrongCookieFid := framework.NewFileID(volumeID, needleID, cookie+1) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ + FromFileIds: []string{wrongCookieFid}, + Selections: []string{"score"}, + Filter: &volume_server_pb.QueryRequest_Filter{ + Field: "score", + Operand: ">", + Value: "0", + }, + InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ + JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"}, + }, + }) + if err != nil { + t.Fatalf("Query start for cookie mismatch should not fail immediately, got: %v", err) + } + + _, err = stream.Recv() + if err != io.EOF { + t.Fatalf("Query cookie mismatch expected EOF with no streamed records, got: %v", err) + } +} diff --git a/test/volume_server/grpc/tail_test.go b/test/volume_server/grpc/tail_test.go new file mode 100644 index 000000000..09657edb5 --- /dev/null +++ b/test/volume_server/grpc/tail_test.go @@ -0,0 +1,206 @@ +package volume_server_grpc_test + +import ( + "bytes" + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestVolumeTailSenderMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{VolumeId: 77777, SinceNs: 0, IdleTimeoutSeconds: 1}) + if err == nil { + _, err = stream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("VolumeTailSender missing-volume error mismatch: %v", err) + } +} + +func TestVolumeTailSenderHeartbeatThenEOF(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(71) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{ + VolumeId: volumeID, + SinceNs: 0, + IdleTimeoutSeconds: 1, + }) + if err != nil { + t.Fatalf("VolumeTailSender start failed: %v", err) + } + + msg, err := stream.Recv() + if err != nil { + t.Fatalf("VolumeTailSender first recv failed: %v", err) + } + if !msg.GetIsLastChunk() { + t.Fatalf("expected first tail message to be heartbeat IsLastChunk=true") + } + + _, err = stream.Recv() + if err != io.EOF { + t.Fatalf("expected EOF after idle timeout drain, got: %v", err) + } +} + +func TestVolumeTailReceiverMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{VolumeId: 88888, SourceVolumeServer: clusterHarness.VolumeServerAddress(), SinceNs: 0, IdleTimeoutSeconds: 1}) + if err == nil || !strings.Contains(err.Error(), "receiver not found volume") { + t.Fatalf("VolumeTailReceiver missing-volume error mismatch: %v", err) + } +} + +func TestVolumeTailReceiverReplicatesSourceUpdates(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1()) + sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer sourceConn.Close() + destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) + defer destConn.Close() + + const volumeID = uint32(72) + framework.AllocateVolume(t, sourceClient, volumeID, "") + framework.AllocateVolume(t, destClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 880003, 0x3456789A) + payload := []byte("tail-receiver-replicates-source-updates") + + sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, sourceUploadResp) + if sourceUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("source upload expected 201, got %d", sourceUploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := destClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{ + VolumeId: volumeID, + SourceVolumeServer: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1], + SinceNs: 0, + IdleTimeoutSeconds: 1, + }) + if err != nil { + t.Fatalf("VolumeTailReceiver success path failed: %v", err) + } + + destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid) + destReadBody := framework.ReadAllAndClose(t, destReadResp) + if destReadResp.StatusCode != http.StatusOK { + t.Fatalf("destination read after tail receive expected 200, got %d", destReadResp.StatusCode) + } + if string(destReadBody) != string(payload) { + t.Fatalf("destination tail-received payload mismatch: got %q want %q", string(destReadBody), string(payload)) + } +} + +func TestVolumeTailSenderLargeNeedleChunking(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(73) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 880004, 0x456789AB) + largePayload := bytes.Repeat([]byte("L"), 2*1024*1024+128*1024) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{ + VolumeId: volumeID, + SinceNs: 0, + IdleTimeoutSeconds: 1, + }) + if err != nil { + t.Fatalf("VolumeTailSender start failed: %v", err) + } + + dataChunkCount := 0 + sawNonLastDataChunk := false + sawLastDataChunk := false + for { + msg, recvErr := stream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("VolumeTailSender recv failed: %v", recvErr) + } + if len(msg.GetNeedleBody()) == 0 { + continue + } + dataChunkCount++ + if msg.GetIsLastChunk() { + sawLastDataChunk = true + } else { + sawNonLastDataChunk = true + } + } + + if dataChunkCount < 2 { + t.Fatalf("VolumeTailSender expected multiple chunks for large needle, got %d", dataChunkCount) + } + if !sawNonLastDataChunk { + t.Fatalf("VolumeTailSender expected at least one non-last data chunk") + } + if !sawLastDataChunk { + t.Fatalf("VolumeTailSender expected a final data chunk marked IsLastChunk=true") + } +} diff --git a/test/volume_server/grpc/tiering_remote_test.go b/test/volume_server/grpc/tiering_remote_test.go new file mode 100644 index 000000000..db36e7cfd --- /dev/null +++ b/test/volume_server/grpc/tiering_remote_test.go @@ -0,0 +1,236 @@ +package volume_server_grpc_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestFetchAndWriteNeedleMaintenanceAndMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{ + VolumeId: 98781, + NeedleId: 1, + }) + if err == nil || !strings.Contains(err.Error(), "not found volume id") { + t.Fatalf("FetchAndWriteNeedle missing-volume error mismatch: %v", err) + } + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: true, + Version: stateResp.GetState().GetVersion(), + }, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{ + VolumeId: 1, + NeedleId: 1, + }) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("FetchAndWriteNeedle maintenance error mismatch: %v", err) + } +} + +func TestFetchAndWriteNeedleInvalidRemoteConfig(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(88) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{ + VolumeId: volumeID, + NeedleId: 1, + Cookie: 1, + Size: 1, + RemoteConf: &remote_pb.RemoteConf{ + Name: "it-invalid-remote", + Type: "does-not-exist", + }, + RemoteLocation: &remote_pb.RemoteStorageLocation{ + Name: "it-invalid-remote", + Path: "/test", + }, + }) + if err == nil || !strings.Contains(err.Error(), "get remote client") { + t.Fatalf("FetchAndWriteNeedle invalid-remote error mismatch: %v", err) + } +} + +func TestVolumeTierMoveDatToRemoteErrorPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(85) + const collection = "tier-collection" + framework.AllocateVolume(t, grpcClient, volumeID, collection) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + missingStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ + VolumeId: 98782, + Collection: collection, + DestinationBackendName: "dummy", + }) + if err == nil { + _, err = missingStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeTierMoveDatToRemote missing-volume error mismatch: %v", err) + } + + mismatchStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ + VolumeId: volumeID, + Collection: "wrong-collection", + DestinationBackendName: "dummy", + }) + if err == nil { + _, err = mismatchStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "unexpected input") { + t.Fatalf("VolumeTierMoveDatToRemote collection mismatch error mismatch: %v", err) + } + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: true, + Version: stateResp.GetState().GetVersion(), + }, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + maintenanceStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ + VolumeId: volumeID, + Collection: collection, + DestinationBackendName: "dummy", + }) + if err == nil { + _, err = maintenanceStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeTierMoveDatToRemote maintenance error mismatch: %v", err) + } +} + +func TestVolumeTierMoveDatToRemoteMissingBackend(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(89) + const collection = "tier-missing-backend" + framework.AllocateVolume(t, grpcClient, volumeID, collection) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ + VolumeId: volumeID, + Collection: collection, + DestinationBackendName: "definitely-missing-backend", + }) + if err == nil { + _, err = stream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "destination definitely-missing-backend not found") { + t.Fatalf("VolumeTierMoveDatToRemote missing-backend error mismatch: %v", err) + } +} + +func TestVolumeTierMoveDatFromRemoteErrorPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(86) + const collection = "tier-download-collection" + framework.AllocateVolume(t, grpcClient, volumeID, collection) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + missingStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{ + VolumeId: 98783, + Collection: collection, + }) + if err == nil { + _, err = missingStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeTierMoveDatFromRemote missing-volume error mismatch: %v", err) + } + + mismatchStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{ + VolumeId: volumeID, + Collection: "wrong-collection", + }) + if err == nil { + _, err = mismatchStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "unexpected input") { + t.Fatalf("VolumeTierMoveDatFromRemote collection mismatch error mismatch: %v", err) + } + + localDiskStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{ + VolumeId: volumeID, + Collection: collection, + }) + if err == nil { + _, err = localDiskStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "already on local disk") { + t.Fatalf("VolumeTierMoveDatFromRemote local-disk error mismatch: %v", err) + } +} diff --git a/test/volume_server/grpc/vacuum_test.go b/test/volume_server/grpc/vacuum_test.go new file mode 100644 index 000000000..ea986fed2 --- /dev/null +++ b/test/volume_server/grpc/vacuum_test.go @@ -0,0 +1,87 @@ +package volume_server_grpc_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestVacuumVolumeCheckSuccessAndMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(31) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := grpcClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VacuumVolumeCheck existing volume failed: %v", err) + } + if resp.GetGarbageRatio() < 0 || resp.GetGarbageRatio() > 1 { + t.Fatalf("unexpected garbage ratio: %f", resp.GetGarbageRatio()) + } + + _, err = grpcClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{VolumeId: 99999}) + if err == nil { + t.Fatalf("VacuumVolumeCheck should fail for missing volume") + } +} + +func TestVacuumMaintenanceModeRejections(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + assertMaintenanceErr := func(name string, err error) { + t.Helper() + if err == nil { + t.Fatalf("%s should fail in maintenance mode", name) + } + if !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("%s expected maintenance mode error, got: %v", name, err) + } + } + + compactStream, err := grpcClient.VacuumVolumeCompact(ctx, &volume_server_pb.VacuumVolumeCompactRequest{VolumeId: 31}) + if err == nil { + _, err = compactStream.Recv() + } + assertMaintenanceErr("VacuumVolumeCompact", err) + + _, err = grpcClient.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: 31}) + assertMaintenanceErr("VacuumVolumeCommit", err) + + _, err = grpcClient.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{VolumeId: 31}) + assertMaintenanceErr("VacuumVolumeCleanup", err) +} diff --git a/test/volume_server/http/admin_test.go b/test/volume_server/http/admin_test.go new file mode 100644 index 000000000..be4445ebc --- /dev/null +++ b/test/volume_server/http/admin_test.go @@ -0,0 +1,174 @@ +package volume_server_http_test + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/util/request_id" +) + +func TestAdminStatusAndHealthz(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + client := framework.NewHTTPClient() + + statusReq, err := http.NewRequest(http.MethodGet, cluster.VolumeAdminURL()+"/status", nil) + if err != nil { + t.Fatalf("create status request: %v", err) + } + statusReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-1") + + statusResp := framework.DoRequest(t, client, statusReq) + statusBody := framework.ReadAllAndClose(t, statusResp) + + if statusResp.StatusCode != http.StatusOK { + t.Fatalf("expected /status code 200, got %d, body: %s", statusResp.StatusCode, string(statusBody)) + } + if got := statusResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") { + t.Fatalf("expected /status Server header to contain SeaweedFS Volume, got %q", got) + } + if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-1" { + t.Fatalf("expected echoed request id, got %q", got) + } + + var payload map[string]interface{} + if err := json.Unmarshal(statusBody, &payload); err != nil { + t.Fatalf("decode status response: %v", err) + } + for _, field := range []string{"Version", "DiskStatuses", "Volumes"} { + if _, found := payload[field]; !found { + t.Fatalf("status payload missing field %q", field) + } + } + + healthReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz") + healthReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-2") + healthResp := framework.DoRequest(t, client, healthReq) + _ = framework.ReadAllAndClose(t, healthResp) + if healthResp.StatusCode != http.StatusOK { + t.Fatalf("expected /healthz code 200, got %d", healthResp.StatusCode) + } + if got := healthResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") { + t.Fatalf("expected /healthz Server header to contain SeaweedFS Volume, got %q", got) + } + if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-2" { + t.Fatalf("expected /healthz echoed request id, got %q", got) + } + + uiResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html")) + uiBody := framework.ReadAllAndClose(t, uiResp) + if uiResp.StatusCode != http.StatusOK { + t.Fatalf("expected /ui/index.html code 200, got %d, body: %s", uiResp.StatusCode, string(uiBody)) + } + if !strings.Contains(strings.ToLower(string(uiBody)), "volume") { + t.Fatalf("ui page does not look like volume status page") + } +} + +func TestOptionsMethodsByPort(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P2()) + client := framework.NewHTTPClient() + + adminResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/")) + _ = framework.ReadAllAndClose(t, adminResp) + if adminResp.StatusCode != http.StatusOK { + t.Fatalf("admin OPTIONS expected 200, got %d", adminResp.StatusCode) + } + adminAllowed := adminResp.Header.Get("Access-Control-Allow-Methods") + for _, expected := range []string{"PUT", "POST", "GET", "DELETE", "OPTIONS"} { + if !strings.Contains(adminAllowed, expected) { + t.Fatalf("admin allow methods missing %q, got %q", expected, adminAllowed) + } + } + if adminResp.Header.Get("Access-Control-Allow-Headers") != "*" { + t.Fatalf("admin allow headers expected '*', got %q", adminResp.Header.Get("Access-Control-Allow-Headers")) + } + + publicResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodOptions, cluster.VolumePublicURL()+"/")) + _ = framework.ReadAllAndClose(t, publicResp) + if publicResp.StatusCode != http.StatusOK { + t.Fatalf("public OPTIONS expected 200, got %d", publicResp.StatusCode) + } + publicAllowed := publicResp.Header.Get("Access-Control-Allow-Methods") + if !strings.Contains(publicAllowed, "GET") || !strings.Contains(publicAllowed, "OPTIONS") { + t.Fatalf("public allow methods expected GET and OPTIONS, got %q", publicAllowed) + } + if strings.Contains(publicAllowed, "POST") { + t.Fatalf("public allow methods should not include POST, got %q", publicAllowed) + } + if publicResp.Header.Get("Access-Control-Allow-Headers") != "*" { + t.Fatalf("public allow headers expected '*', got %q", publicResp.Header.Get("Access-Control-Allow-Headers")) + } +} + +func TestOptionsWithOriginIncludesCorsHeaders(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P2()) + client := framework.NewHTTPClient() + + adminReq := mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/") + adminReq.Header.Set("Origin", "https://example.com") + adminResp := framework.DoRequest(t, client, adminReq) + _ = framework.ReadAllAndClose(t, adminResp) + if adminResp.StatusCode != http.StatusOK { + t.Fatalf("admin OPTIONS expected 200, got %d", adminResp.StatusCode) + } + if adminResp.Header.Get("Access-Control-Allow-Origin") != "*" { + t.Fatalf("admin OPTIONS expected Access-Control-Allow-Origin=*, got %q", adminResp.Header.Get("Access-Control-Allow-Origin")) + } + if adminResp.Header.Get("Access-Control-Allow-Credentials") != "true" { + t.Fatalf("admin OPTIONS expected Access-Control-Allow-Credentials=true, got %q", adminResp.Header.Get("Access-Control-Allow-Credentials")) + } + + publicReq := mustNewRequest(t, http.MethodOptions, cluster.VolumePublicURL()+"/") + publicReq.Header.Set("Origin", "https://example.com") + publicResp := framework.DoRequest(t, client, publicReq) + _ = framework.ReadAllAndClose(t, publicResp) + if publicResp.StatusCode != http.StatusOK { + t.Fatalf("public OPTIONS expected 200, got %d", publicResp.StatusCode) + } + if publicResp.Header.Get("Access-Control-Allow-Origin") != "*" { + t.Fatalf("public OPTIONS expected Access-Control-Allow-Origin=*, got %q", publicResp.Header.Get("Access-Control-Allow-Origin")) + } + if publicResp.Header.Get("Access-Control-Allow-Credentials") != "true" { + t.Fatalf("public OPTIONS expected Access-Control-Allow-Credentials=true, got %q", publicResp.Header.Get("Access-Control-Allow-Credentials")) + } +} + +func TestUiIndexNotExposedWhenJwtSigningEnabled(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P3()) + client := framework.NewHTTPClient() + + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html")) + body := framework.ReadAllAndClose(t, resp) + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected /ui/index.html to be gated by auth under JWT profile (401), got %d body=%s", resp.StatusCode, string(body)) + } +} + +func mustNewRequest(t testing.TB, method, url string) *http.Request { + t.Helper() + req, err := http.NewRequest(method, url, nil) + if err != nil { + t.Fatalf("create request %s %s: %v", method, url, err) + } + return req +} diff --git a/test/volume_server/http/auth_test.go b/test/volume_server/http/auth_test.go new file mode 100644 index 000000000..5b093bba1 --- /dev/null +++ b/test/volume_server/http/auth_test.go @@ -0,0 +1,419 @@ +package volume_server_http_test + +import ( + "bytes" + "net/http" + "testing" + "time" + + jwt "github.com/golang-jwt/jwt/v5" + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/security" +) + +func TestJWTAuthForWriteAndRead(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(51) + const needleID = uint64(123456) + const cookie = uint32(0xABCDEF12) + + framework.AllocateVolume(t, grpcClient, volumeID, "") + fid := framework.NewFileID(volumeID, needleID, cookie) + payload := []byte("jwt-protected-content") + client := framework.NewHTTPClient() + + unauthWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + unauthWriteResp := framework.DoRequest(t, client, unauthWrite) + _ = framework.ReadAllAndClose(t, unauthWriteResp) + if unauthWriteResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("unauthorized write expected 401, got %d", unauthWriteResp.StatusCode) + } + + invalidWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + invalidWrite.Header.Set("Authorization", "Bearer invalid") + invalidWriteResp := framework.DoRequest(t, client, invalidWrite) + _ = framework.ReadAllAndClose(t, invalidWriteResp) + if invalidWriteResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("invalid write token expected 401, got %d", invalidWriteResp.StatusCode) + } + + writeToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + authWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + authWrite.Header.Set("Authorization", "Bearer "+string(writeToken)) + authWriteResp := framework.DoRequest(t, client, authWrite) + _ = framework.ReadAllAndClose(t, authWriteResp) + if authWriteResp.StatusCode != http.StatusCreated { + t.Fatalf("authorized write expected 201, got %d", authWriteResp.StatusCode) + } + + unauthReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + unauthReadResp := framework.DoRequest(t, client, unauthReadReq) + _ = framework.ReadAllAndClose(t, unauthReadResp) + if unauthReadResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("unauthorized read expected 401, got %d", unauthReadResp.StatusCode) + } + + readToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) + authReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + authReadReq.Header.Set("Authorization", "Bearer "+string(readToken)) + authReadResp := framework.DoRequest(t, client, authReadReq) + authReadBody := framework.ReadAllAndClose(t, authReadResp) + if authReadResp.StatusCode != http.StatusOK { + t.Fatalf("authorized read expected 200, got %d", authReadResp.StatusCode) + } + if string(authReadBody) != string(payload) { + t.Fatalf("authorized read content mismatch: got %q want %q", string(authReadBody), string(payload)) + } +} + +func TestJWTAuthRejectsFidMismatch(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(52) + const needleID = uint64(223344) + const cookie = uint32(0x10203040) + const otherNeedleID = uint64(223345) + const otherCookie = uint32(0x50607080) + const wrongCookie = uint32(0x10203041) + + framework.AllocateVolume(t, grpcClient, volumeID, "") + fid := framework.NewFileID(volumeID, needleID, cookie) + otherFid := framework.NewFileID(volumeID, otherNeedleID, otherCookie) + payload := []byte("jwt-fid-mismatch-content") + client := framework.NewHTTPClient() + + writeTokenForOtherFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFid) + mismatchedWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + mismatchedWrite.Header.Set("Authorization", "Bearer "+string(writeTokenForOtherFid)) + mismatchedWriteResp := framework.DoRequest(t, client, mismatchedWrite) + _ = framework.ReadAllAndClose(t, mismatchedWriteResp) + if mismatchedWriteResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("write with mismatched fid token expected 401, got %d", mismatchedWriteResp.StatusCode) + } + + wrongCookieFid := framework.NewFileID(volumeID, needleID, wrongCookie) + writeTokenWrongCookie := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, wrongCookieFid) + wrongCookieWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + wrongCookieWrite.Header.Set("Authorization", "Bearer "+string(writeTokenWrongCookie)) + wrongCookieWriteResp := framework.DoRequest(t, client, wrongCookieWrite) + _ = framework.ReadAllAndClose(t, wrongCookieWriteResp) + if wrongCookieWriteResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("write with wrong-cookie fid token expected 401, got %d", wrongCookieWriteResp.StatusCode) + } + + writeTokenForFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + validWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + validWrite.Header.Set("Authorization", "Bearer "+string(writeTokenForFid)) + validWriteResp := framework.DoRequest(t, client, validWrite) + _ = framework.ReadAllAndClose(t, validWriteResp) + if validWriteResp.StatusCode != http.StatusCreated { + t.Fatalf("authorized write expected 201, got %d", validWriteResp.StatusCode) + } + + readTokenForOtherFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFid) + mismatchedReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + mismatchedReadReq.Header.Set("Authorization", "Bearer "+string(readTokenForOtherFid)) + mismatchedReadResp := framework.DoRequest(t, client, mismatchedReadReq) + _ = framework.ReadAllAndClose(t, mismatchedReadResp) + if mismatchedReadResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("read with mismatched fid token expected 401, got %d", mismatchedReadResp.StatusCode) + } + + readTokenWrongCookie := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, wrongCookieFid) + wrongCookieReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + wrongCookieReadReq.Header.Set("Authorization", "Bearer "+string(readTokenWrongCookie)) + wrongCookieReadResp := framework.DoRequest(t, client, wrongCookieReadReq) + _ = framework.ReadAllAndClose(t, wrongCookieReadResp) + if wrongCookieReadResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("read with wrong-cookie fid token expected 401, got %d", wrongCookieReadResp.StatusCode) + } +} + +func newUploadRequest(t testing.TB, url string, payload []byte) *http.Request { + t.Helper() + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(payload)) + if err != nil { + t.Fatalf("create upload request %s: %v", url, err) + } + req.Header.Set("Content-Type", "application/octet-stream") + return req +} + +func TestJWTAuthRejectsExpiredTokens(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(53) + const needleID = uint64(334455) + const cookie = uint32(0x22334455) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + payload := []byte("expired-token-content") + client := framework.NewHTTPClient() + + expiredWriteToken := mustGenExpiredToken(t, []byte(profile.JWTSigningKey), fid) + writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + writeReq.Header.Set("Authorization", "Bearer "+expiredWriteToken) + writeResp := framework.DoRequest(t, client, writeReq) + _ = framework.ReadAllAndClose(t, writeResp) + if writeResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expired write token expected 401, got %d", writeResp.StatusCode) + } + + // Seed data with a valid token so read auth path can be exercised against existing content. + validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + validWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + validWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) + validWriteResp := framework.DoRequest(t, client, validWriteReq) + _ = framework.ReadAllAndClose(t, validWriteResp) + if validWriteResp.StatusCode != http.StatusCreated { + t.Fatalf("valid write expected 201, got %d", validWriteResp.StatusCode) + } + + expiredReadToken := mustGenExpiredToken(t, []byte(profile.JWTReadKey), fid) + readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + readReq.Header.Set("Authorization", "Bearer "+expiredReadToken) + readResp := framework.DoRequest(t, client, readReq) + _ = framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expired read token expected 401, got %d", readResp.StatusCode) + } +} + +func TestJWTAuthViaQueryParamAndCookie(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(54) + const needleID = uint64(445566) + const cookie = uint32(0x31415926) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + payload := []byte("jwt-query-cookie-content") + client := framework.NewHTTPClient() + + writeToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(writeToken), payload) + writeResp := framework.DoRequest(t, client, writeReq) + _ = framework.ReadAllAndClose(t, writeResp) + if writeResp.StatusCode != http.StatusCreated { + t.Fatalf("query-jwt write expected 201, got %d", writeResp.StatusCode) + } + + readToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) + readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(readToken)}) + readResp := framework.DoRequest(t, client, readReq) + readBody := framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusOK { + t.Fatalf("cookie-jwt read expected 200, got %d", readResp.StatusCode) + } + if string(readBody) != string(payload) { + t.Fatalf("cookie-jwt read body mismatch: got %q want %q", string(readBody), string(payload)) + } +} + +func TestJWTTokenSourcePrecedenceQueryOverHeader(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(55) + const needleID = uint64(556677) + const cookie = uint32(0x99887766) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1) + payload := []byte("jwt-precedence-content") + client := framework.NewHTTPClient() + + validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + invalidWriteQueryToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID) + writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidWriteQueryToken), payload) + writeReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) + writeResp := framework.DoRequest(t, client, writeReq) + _ = framework.ReadAllAndClose(t, writeResp) + if writeResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("query token should take precedence over header token for write, expected 401 got %d", writeResp.StatusCode) + } + + // Seed data with valid write token, then exercise read precedence. + seedWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + seedWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) + seedWriteResp := framework.DoRequest(t, client, seedWriteReq) + _ = framework.ReadAllAndClose(t, seedWriteResp) + if seedWriteResp.StatusCode != http.StatusCreated { + t.Fatalf("seed write expected 201, got %d", seedWriteResp.StatusCode) + } + + validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) + invalidReadQueryToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID) + readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidReadQueryToken)) + readReq.Header.Set("Authorization", "Bearer "+string(validReadToken)) + readResp := framework.DoRequest(t, client, readReq) + _ = framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("query token should take precedence over header token for read, expected 401 got %d", readResp.StatusCode) + } +} + +func TestJWTTokenSourcePrecedenceHeaderOverCookie(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(56) + const needleID = uint64(667788) + const cookie = uint32(0x11229988) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1) + payload := []byte("jwt-precedence-header-cookie") + client := framework.NewHTTPClient() + + validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + invalidCookieWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID) + writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + writeReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) + writeReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieWriteToken)}) + writeResp := framework.DoRequest(t, client, writeReq) + _ = framework.ReadAllAndClose(t, writeResp) + if writeResp.StatusCode != http.StatusCreated { + t.Fatalf("header token should take precedence over cookie token for write, expected 201 got %d", writeResp.StatusCode) + } + + validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) + invalidCookieReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID) + readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + readReq.Header.Set("Authorization", "Bearer "+string(validReadToken)) + readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieReadToken)}) + readResp := framework.DoRequest(t, client, readReq) + readBody := framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusOK { + t.Fatalf("header token should take precedence over cookie token for read, expected 200 got %d", readResp.StatusCode) + } + if string(readBody) != string(payload) { + t.Fatalf("header-over-cookie read body mismatch: got %q want %q", string(readBody), string(payload)) + } +} + +func TestJWTTokenSourcePrecedenceQueryOverCookie(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(57) + const needleID = uint64(778899) + const cookie = uint32(0x88776655) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1) + payload := []byte("jwt-precedence-query-cookie") + client := framework.NewHTTPClient() + + validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + invalidQueryWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID) + writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidQueryWriteToken), payload) + writeReq.AddCookie(&http.Cookie{Name: "AT", Value: string(validWriteToken)}) + writeResp := framework.DoRequest(t, client, writeReq) + _ = framework.ReadAllAndClose(t, writeResp) + if writeResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("query token should take precedence over cookie token for write, expected 401 got %d", writeResp.StatusCode) + } + + // Seed data with valid write token so read precedence can be exercised. + seedWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + seedWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) + seedWriteResp := framework.DoRequest(t, client, seedWriteReq) + _ = framework.ReadAllAndClose(t, seedWriteResp) + if seedWriteResp.StatusCode != http.StatusCreated { + t.Fatalf("seed write expected 201, got %d", seedWriteResp.StatusCode) + } + + validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) + invalidQueryReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID) + readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidQueryReadToken)) + readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(validReadToken)}) + readResp := framework.DoRequest(t, client, readReq) + _ = framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("query token should take precedence over cookie token for read, expected 401 got %d", readResp.StatusCode) + } + + // Validate positive path: valid query token should succeed even if cookie token is invalid. + validQueryReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(validReadToken)) + invalidCookieReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID) + validQueryReadReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieReadToken)}) + validQueryReadResp := framework.DoRequest(t, client, validQueryReadReq) + validQueryReadBody := framework.ReadAllAndClose(t, validQueryReadResp) + if validQueryReadResp.StatusCode != http.StatusOK { + t.Fatalf("valid query token should succeed over invalid cookie token, expected 200 got %d", validQueryReadResp.StatusCode) + } + if string(validQueryReadBody) != string(payload) { + t.Fatalf("query-over-cookie read body mismatch: got %q want %q", string(validQueryReadBody), string(payload)) + } +} + +func mustGenExpiredToken(t testing.TB, key []byte, fid string) string { + t.Helper() + claims := security.SeaweedFileIdClaims{ + Fid: fid, + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(-1 * time.Minute)), + }, + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + signed, err := token.SignedString(key) + if err != nil { + t.Fatalf("sign expired token: %v", err) + } + return signed +} diff --git a/test/volume_server/http/chunk_manifest_test.go b/test/volume_server/http/chunk_manifest_test.go new file mode 100644 index 000000000..d3806d7f4 --- /dev/null +++ b/test/volume_server/http/chunk_manifest_test.go @@ -0,0 +1,232 @@ +package volume_server_http_test + +import ( + "bytes" + "encoding/json" + "net/http" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/operation" +) + +func TestChunkManifestExpansionAndBypass(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(102) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + + chunkFID := framework.NewFileID(volumeID, 772005, 0x5E6F7081) + chunkPayload := []byte("chunk-manifest-expanded-content") + chunkUploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID, chunkPayload) + _ = framework.ReadAllAndClose(t, chunkUploadResp) + if chunkUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("chunk upload expected 201, got %d", chunkUploadResp.StatusCode) + } + + manifest := &operation.ChunkManifest{ + Name: "manifest.bin", + Mime: "application/octet-stream", + Size: int64(len(chunkPayload)), + Chunks: []*operation.ChunkInfo{ + { + Fid: chunkFID, + Offset: 0, + Size: int64(len(chunkPayload)), + }, + }, + } + manifestBytes, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal chunk manifest: %v", err) + } + + manifestFID := framework.NewFileID(volumeID, 772006, 0x6F708192) + manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes)) + if err != nil { + t.Fatalf("create manifest upload request: %v", err) + } + manifestUploadReq.Header.Set("Content-Type", "application/json") + manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq) + _ = framework.ReadAllAndClose(t, manifestUploadResp) + if manifestUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode) + } + + expandedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), manifestFID) + expandedReadBody := framework.ReadAllAndClose(t, expandedReadResp) + if expandedReadResp.StatusCode != http.StatusOK { + t.Fatalf("manifest expanded read expected 200, got %d", expandedReadResp.StatusCode) + } + if string(expandedReadBody) != string(chunkPayload) { + t.Fatalf("manifest expanded read mismatch: got %q want %q", string(expandedReadBody), string(chunkPayload)) + } + if expandedReadResp.Header.Get("X-File-Store") != "chunked" { + t.Fatalf("manifest expanded read expected X-File-Store=chunked, got %q", expandedReadResp.Header.Get("X-File-Store")) + } + + bypassReadResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=false")) + bypassReadBody := framework.ReadAllAndClose(t, bypassReadResp) + if bypassReadResp.StatusCode != http.StatusOK { + t.Fatalf("manifest bypass read expected 200, got %d", bypassReadResp.StatusCode) + } + if bypassReadResp.Header.Get("X-File-Store") != "" { + t.Fatalf("manifest bypass read expected empty X-File-Store header, got %q", bypassReadResp.Header.Get("X-File-Store")) + } + + var gotManifest operation.ChunkManifest + if err = json.Unmarshal(bypassReadBody, &gotManifest); err != nil { + t.Fatalf("manifest bypass read expected JSON payload, got decode error: %v body=%q", err, string(bypassReadBody)) + } + if len(gotManifest.Chunks) != 1 || gotManifest.Chunks[0].Fid != chunkFID { + t.Fatalf("manifest bypass read payload mismatch: %+v", gotManifest) + } +} + +func TestChunkManifestDeleteRemovesChildChunks(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(104) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + + chunkFID := framework.NewFileID(volumeID, 772008, 0x8192A3B4) + chunkPayload := []byte("chunk-manifest-delete-content") + chunkUploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID, chunkPayload) + _ = framework.ReadAllAndClose(t, chunkUploadResp) + if chunkUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("chunk upload expected 201, got %d", chunkUploadResp.StatusCode) + } + + manifest := &operation.ChunkManifest{ + Name: "manifest-delete.bin", + Mime: "application/octet-stream", + Size: int64(len(chunkPayload)), + Chunks: []*operation.ChunkInfo{ + { + Fid: chunkFID, + Offset: 0, + Size: int64(len(chunkPayload)), + }, + }, + } + manifestBytes, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal chunk manifest: %v", err) + } + + manifestFID := framework.NewFileID(volumeID, 772009, 0x92A3B4C5) + manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes)) + if err != nil { + t.Fatalf("create manifest upload request: %v", err) + } + manifestUploadReq.Header.Set("Content-Type", "application/json") + manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq) + _ = framework.ReadAllAndClose(t, manifestUploadResp) + if manifestUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode) + } + + deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+manifestFID)) + deleteBody := framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("manifest delete expected 202, got %d", deleteResp.StatusCode) + } + var deleteResult map[string]int64 + if err = json.Unmarshal(deleteBody, &deleteResult); err != nil { + t.Fatalf("decode manifest delete response: %v body=%q", err, string(deleteBody)) + } + if deleteResult["size"] != int64(len(chunkPayload)) { + t.Fatalf("manifest delete expected size=%d, got %d", len(chunkPayload), deleteResult["size"]) + } + + manifestReadAfterDelete := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), manifestFID) + _ = framework.ReadAllAndClose(t, manifestReadAfterDelete) + if manifestReadAfterDelete.StatusCode != http.StatusNotFound { + t.Fatalf("manifest read after delete expected 404, got %d", manifestReadAfterDelete.StatusCode) + } + + chunkReadAfterDelete := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID) + _ = framework.ReadAllAndClose(t, chunkReadAfterDelete) + if chunkReadAfterDelete.StatusCode != http.StatusNotFound { + t.Fatalf("chunk read after manifest delete expected 404, got %d", chunkReadAfterDelete.StatusCode) + } +} + +func TestChunkManifestDeleteFailsWhenChildDeletionFails(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(105) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + manifest := &operation.ChunkManifest{ + Name: "manifest-delete-failure.bin", + Mime: "application/octet-stream", + Size: 1, + Chunks: []*operation.ChunkInfo{ + { + Fid: "not-a-valid-fid", + Offset: 0, + Size: 1, + }, + }, + } + manifestBytes, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal chunk manifest: %v", err) + } + + manifestFID := framework.NewFileID(volumeID, 772010, 0xA3B4C5D6) + manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes)) + if err != nil { + t.Fatalf("create manifest upload request: %v", err) + } + manifestUploadReq.Header.Set("Content-Type", "application/json") + manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq) + _ = framework.ReadAllAndClose(t, manifestUploadResp) + if manifestUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode) + } + + deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+manifestFID)) + deleteBody := framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusInternalServerError { + t.Fatalf("manifest delete with invalid child fid expected 500, got %d body=%q", deleteResp.StatusCode, string(deleteBody)) + } + + manifestBypassRead := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=false")) + manifestBypassBody := framework.ReadAllAndClose(t, manifestBypassRead) + if manifestBypassRead.StatusCode != http.StatusOK { + t.Fatalf("manifest bypass read after failed delete expected 200, got %d", manifestBypassRead.StatusCode) + } + var gotManifest operation.ChunkManifest + if err = json.Unmarshal(manifestBypassBody, &gotManifest); err != nil { + t.Fatalf("manifest bypass read expected JSON payload, got decode error: %v body=%q", err, string(manifestBypassBody)) + } + if len(gotManifest.Chunks) != 1 || gotManifest.Chunks[0].Fid != "not-a-valid-fid" { + t.Fatalf("manifest payload mismatch after failed delete: %+v", gotManifest) + } +} diff --git a/test/volume_server/http/compressed_read_test.go b/test/volume_server/http/compressed_read_test.go new file mode 100644 index 000000000..8a9ac5c41 --- /dev/null +++ b/test/volume_server/http/compressed_read_test.go @@ -0,0 +1,97 @@ +package volume_server_http_test + +import ( + "bytes" + "compress/gzip" + "io" + "net/http" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func gzipData(t testing.TB, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write(data); err != nil { + t.Fatalf("gzip write: %v", err) + } + if err := zw.Close(); err != nil { + t.Fatalf("gzip close: %v", err) + } + return buf.Bytes() +} + +func gunzipData(t testing.TB, data []byte) []byte { + t.Helper() + zr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("gunzip new reader: %v", err) + } + defer zr.Close() + out, err := io.ReadAll(zr) + if err != nil { + t.Fatalf("gunzip read: %v", err) + } + return out +} + +func TestCompressedReadAcceptEncodingMatrix(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(103) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 772007, 0x708192A3) + plainPayload := []byte("compressed-read-accept-encoding-matrix-content-compressed-read-accept-encoding-matrix-content") + compressedPayload := gzipData(t, plainPayload) + + uploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+fid, bytes.NewReader(compressedPayload)) + if err != nil { + t.Fatalf("create compressed upload request: %v", err) + } + uploadReq.Header.Set("Content-Type", "text/plain") + uploadReq.Header.Set("Content-Encoding", "gzip") + uploadResp := framework.DoRequest(t, client, uploadReq) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("compressed upload expected 201, got %d", uploadResp.StatusCode) + } + + gzipReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + gzipReadReq.Header.Set("Accept-Encoding", "gzip") + gzipReadResp := framework.DoRequest(t, client, gzipReadReq) + gzipReadBody := framework.ReadAllAndClose(t, gzipReadResp) + if gzipReadResp.StatusCode != http.StatusOK { + t.Fatalf("gzip-accepted read expected 200, got %d", gzipReadResp.StatusCode) + } + if gzipReadResp.Header.Get("Content-Encoding") != "gzip" { + t.Fatalf("gzip-accepted read expected Content-Encoding=gzip, got %q", gzipReadResp.Header.Get("Content-Encoding")) + } + if string(gunzipData(t, gzipReadBody)) != string(plainPayload) { + t.Fatalf("gzip-accepted read body mismatch after gunzip") + } + + identityReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + identityReadReq.Header.Set("Accept-Encoding", "identity") + identityReadResp := framework.DoRequest(t, client, identityReadReq) + identityReadBody := framework.ReadAllAndClose(t, identityReadResp) + if identityReadResp.StatusCode != http.StatusOK { + t.Fatalf("identity read expected 200, got %d", identityReadResp.StatusCode) + } + if identityReadResp.Header.Get("Content-Encoding") != "" { + t.Fatalf("identity read expected no Content-Encoding header, got %q", identityReadResp.Header.Get("Content-Encoding")) + } + if string(identityReadBody) != string(plainPayload) { + t.Fatalf("identity read body mismatch: got %q want %q", string(identityReadBody), string(plainPayload)) + } +} diff --git a/test/volume_server/http/headers_static_test.go b/test/volume_server/http/headers_static_test.go new file mode 100644 index 000000000..5b4a2fd93 --- /dev/null +++ b/test/volume_server/http/headers_static_test.go @@ -0,0 +1,102 @@ +package volume_server_http_test + +import ( + "fmt" + "net/http" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestReadPassthroughHeadersAndDownloadDisposition(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(96) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fullFileID := framework.NewFileID(volumeID, 661122, 0x55667788) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fullFileID, []byte("passthrough-header-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + parts := strings.SplitN(fullFileID, ",", 2) + if len(parts) != 2 { + t.Fatalf("unexpected file id format: %q", fullFileID) + } + fidOnly := parts[1] + + url := fmt.Sprintf("%s/%d/%s/%s?response-content-type=text/plain&response-cache-control=no-store&dl=true", + clusterHarness.VolumeAdminURL(), + volumeID, + fidOnly, + "report.txt", + ) + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, url)) + _ = framework.ReadAllAndClose(t, resp) + if resp.StatusCode != http.StatusOK { + t.Fatalf("passthrough read expected 200, got %d", resp.StatusCode) + } + if resp.Header.Get("Content-Type") != "text/plain" { + t.Fatalf("response-content-type override mismatch: %q", resp.Header.Get("Content-Type")) + } + if resp.Header.Get("Cache-Control") != "no-store" { + t.Fatalf("response-cache-control override mismatch: %q", resp.Header.Get("Cache-Control")) + } + contentDisposition := resp.Header.Get("Content-Disposition") + if !strings.Contains(contentDisposition, "attachment") || !strings.Contains(contentDisposition, "report.txt") { + t.Fatalf("download disposition header mismatch: %q", contentDisposition) + } +} + +func TestStaticAssetEndpoints(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + client := framework.NewHTTPClient() + + faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/favicon.ico")) + _ = framework.ReadAllAndClose(t, faviconResp) + if faviconResp.StatusCode != http.StatusOK { + t.Fatalf("/favicon.ico expected 200, got %d", faviconResp.StatusCode) + } + + staticResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/seaweedfsstatic/seaweed50x50.png")) + _ = framework.ReadAllAndClose(t, staticResp) + if staticResp.StatusCode != http.StatusOK { + t.Fatalf("/seaweedfsstatic/seaweed50x50.png expected 200, got %d", staticResp.StatusCode) + } +} + +func TestStaticAssetEndpointsOnPublicPort(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + client := framework.NewHTTPClient() + + faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/favicon.ico")) + _ = framework.ReadAllAndClose(t, faviconResp) + if faviconResp.StatusCode != http.StatusOK { + t.Fatalf("public /favicon.ico expected 200, got %d", faviconResp.StatusCode) + } + + staticResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/seaweedfsstatic/seaweed50x50.png")) + _ = framework.ReadAllAndClose(t, staticResp) + if staticResp.StatusCode != http.StatusOK { + t.Fatalf("public /seaweedfsstatic/seaweed50x50.png expected 200, got %d", staticResp.StatusCode) + } +} diff --git a/test/volume_server/http/image_transform_test.go b/test/volume_server/http/image_transform_test.go new file mode 100644 index 000000000..222fc951f --- /dev/null +++ b/test/volume_server/http/image_transform_test.go @@ -0,0 +1,92 @@ +package volume_server_http_test + +import ( + "bytes" + "fmt" + "image" + "image/color" + "image/png" + "net/http" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func makePNGFixture(t testing.TB, width, height int) []byte { + t.Helper() + + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + img.Set(x, y, color.RGBA{R: uint8(x * 20), G: uint8(y * 20), B: 200, A: 255}) + } + } + + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatalf("encode png fixture: %v", err) + } + return buf.Bytes() +} + +func decodeImageConfig(t testing.TB, data []byte) image.Config { + t.Helper() + cfg, _, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil { + t.Fatalf("decode image config: %v", err) + } + return cfg +} + +func TestImageResizeAndCropReadVariants(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(101) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fullFileID := framework.NewFileID(volumeID, 772004, 0x4D5E6F70) + uploadReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fullFileID, makePNGFixture(t, 6, 4)) + uploadReq.Header.Set("Content-Type", "image/png") + uploadResp := framework.DoRequest(t, client, uploadReq) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("image upload expected 201, got %d", uploadResp.StatusCode) + } + + parts := strings.SplitN(fullFileID, ",", 2) + if len(parts) != 2 { + t.Fatalf("unexpected file id format: %q", fullFileID) + } + fidOnly := parts[1] + + resizeURL := fmt.Sprintf("%s/%d/%s/%s?width=2&height=1", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "fixture.png") + resizeResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, resizeURL)) + resizeBody := framework.ReadAllAndClose(t, resizeResp) + if resizeResp.StatusCode != http.StatusOK { + t.Fatalf("image resize read expected 200, got %d", resizeResp.StatusCode) + } + resizeCfg := decodeImageConfig(t, resizeBody) + if resizeCfg.Width > 2 || resizeCfg.Height > 1 { + t.Fatalf("image resize expected dimensions <= 2x1, got %dx%d", resizeCfg.Width, resizeCfg.Height) + } + + cropURL := fmt.Sprintf("%s/%d/%s/%s?crop_x1=1&crop_y1=1&crop_x2=4&crop_y2=3", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "fixture.png") + cropResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cropURL)) + cropBody := framework.ReadAllAndClose(t, cropResp) + if cropResp.StatusCode != http.StatusOK { + t.Fatalf("image crop read expected 200, got %d", cropResp.StatusCode) + } + cropCfg := decodeImageConfig(t, cropBody) + if cropCfg.Width != 3 || cropCfg.Height != 2 { + t.Fatalf("image crop expected 3x2, got %dx%d", cropCfg.Width, cropCfg.Height) + } +} diff --git a/test/volume_server/http/public_cors_methods_test.go b/test/volume_server/http/public_cors_methods_test.go new file mode 100644 index 000000000..5328b9a8b --- /dev/null +++ b/test/volume_server/http/public_cors_methods_test.go @@ -0,0 +1,287 @@ +package volume_server_http_test + +import ( + "bytes" + "net/http" + "strconv" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestPublicPortReadOnlyMethodBehavior(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(81) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 123321, 0x01020304) + originalData := []byte("public-port-original") + replacementData := []byte("public-port-replacement") + client := framework.NewHTTPClient() + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, originalData) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("admin upload expected 201, got %d", uploadResp.StatusCode) + } + + publicReadResp := framework.ReadBytes(t, client, clusterHarness.VolumePublicURL(), fid) + publicReadBody := framework.ReadAllAndClose(t, publicReadResp) + if publicReadResp.StatusCode != http.StatusOK { + t.Fatalf("public GET expected 200, got %d", publicReadResp.StatusCode) + } + if string(publicReadBody) != string(originalData) { + t.Fatalf("public GET body mismatch: got %q want %q", string(publicReadBody), string(originalData)) + } + + publicPostReq := newUploadRequest(t, clusterHarness.VolumePublicURL()+"/"+fid, replacementData) + publicPostResp := framework.DoRequest(t, client, publicPostReq) + _ = framework.ReadAllAndClose(t, publicPostResp) + if publicPostResp.StatusCode != http.StatusOK { + t.Fatalf("public POST expected passthrough 200, got %d", publicPostResp.StatusCode) + } + + publicDeleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumePublicURL()+"/"+fid)) + _ = framework.ReadAllAndClose(t, publicDeleteResp) + if publicDeleteResp.StatusCode != http.StatusOK { + t.Fatalf("public DELETE expected passthrough 200, got %d", publicDeleteResp.StatusCode) + } + + adminReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) + adminReadBody := framework.ReadAllAndClose(t, adminReadResp) + if adminReadResp.StatusCode != http.StatusOK { + t.Fatalf("admin GET after public POST/DELETE expected 200, got %d", adminReadResp.StatusCode) + } + if string(adminReadBody) != string(originalData) { + t.Fatalf("public port should not mutate data: got %q want %q", string(adminReadBody), string(originalData)) + } +} + +func TestCorsAndUnsupportedMethodBehavior(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(82) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 789789, 0x0A0B0C0D) + client := framework.NewHTTPClient() + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("cors-check")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("admin upload expected 201, got %d", uploadResp.StatusCode) + } + + adminOriginReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + adminOriginReq.Header.Set("Origin", "https://example.com") + adminOriginResp := framework.DoRequest(t, client, adminOriginReq) + _ = framework.ReadAllAndClose(t, adminOriginResp) + if adminOriginResp.Header.Get("Access-Control-Allow-Origin") != "*" { + t.Fatalf("admin GET origin header mismatch: %q", adminOriginResp.Header.Get("Access-Control-Allow-Origin")) + } + if adminOriginResp.Header.Get("Access-Control-Allow-Credentials") != "true" { + t.Fatalf("admin GET credentials header mismatch: %q", adminOriginResp.Header.Get("Access-Control-Allow-Credentials")) + } + + publicOriginReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/"+fid) + publicOriginReq.Header.Set("Origin", "https://example.com") + publicOriginResp := framework.DoRequest(t, client, publicOriginReq) + _ = framework.ReadAllAndClose(t, publicOriginResp) + if publicOriginResp.Header.Get("Access-Control-Allow-Origin") != "*" { + t.Fatalf("public GET origin header mismatch: %q", publicOriginResp.Header.Get("Access-Control-Allow-Origin")) + } + if publicOriginResp.Header.Get("Access-Control-Allow-Credentials") != "true" { + t.Fatalf("public GET credentials header mismatch: %q", publicOriginResp.Header.Get("Access-Control-Allow-Credentials")) + } + + adminPatchReq, err := http.NewRequest(http.MethodPatch, clusterHarness.VolumeAdminURL()+"/"+fid, bytes.NewReader([]byte("patch"))) + if err != nil { + t.Fatalf("create admin PATCH request: %v", err) + } + adminPatchResp := framework.DoRequest(t, client, adminPatchReq) + _ = framework.ReadAllAndClose(t, adminPatchResp) + if adminPatchResp.StatusCode != http.StatusBadRequest { + t.Fatalf("admin PATCH expected 400, got %d", adminPatchResp.StatusCode) + } + + publicPatchReq, err := http.NewRequest(http.MethodPatch, clusterHarness.VolumePublicURL()+"/"+fid, bytes.NewReader([]byte("patch"))) + if err != nil { + t.Fatalf("create public PATCH request: %v", err) + } + publicPatchResp := framework.DoRequest(t, client, publicPatchReq) + _ = framework.ReadAllAndClose(t, publicPatchResp) + if publicPatchResp.StatusCode != http.StatusOK { + t.Fatalf("public PATCH expected passthrough 200, got %d", publicPatchResp.StatusCode) + } +} + +func TestUnsupportedMethodTraceParity(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(83) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 123999, 0x01010101) + client := framework.NewHTTPClient() + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("trace-method-check")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + adminTraceReq := mustNewRequest(t, http.MethodTrace, clusterHarness.VolumeAdminURL()+"/"+fid) + adminTraceResp := framework.DoRequest(t, client, adminTraceReq) + _ = framework.ReadAllAndClose(t, adminTraceResp) + if adminTraceResp.StatusCode != http.StatusBadRequest { + t.Fatalf("admin TRACE expected 400, got %d", adminTraceResp.StatusCode) + } + + publicTraceReq := mustNewRequest(t, http.MethodTrace, clusterHarness.VolumePublicURL()+"/"+fid) + publicTraceResp := framework.DoRequest(t, client, publicTraceReq) + _ = framework.ReadAllAndClose(t, publicTraceResp) + if publicTraceResp.StatusCode != http.StatusOK { + t.Fatalf("public TRACE expected passthrough 200, got %d", publicTraceResp.StatusCode) + } +} + +func TestUnsupportedMethodPropfindParity(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(84) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 124000, 0x02020202) + client := framework.NewHTTPClient() + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("propfind-method-check")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + adminReq := mustNewRequest(t, "PROPFIND", clusterHarness.VolumeAdminURL()+"/"+fid) + adminResp := framework.DoRequest(t, client, adminReq) + _ = framework.ReadAllAndClose(t, adminResp) + if adminResp.StatusCode != http.StatusBadRequest { + t.Fatalf("admin PROPFIND expected 400, got %d", adminResp.StatusCode) + } + + publicReq := mustNewRequest(t, "PROPFIND", clusterHarness.VolumePublicURL()+"/"+fid) + publicResp := framework.DoRequest(t, client, publicReq) + _ = framework.ReadAllAndClose(t, publicResp) + if publicResp.StatusCode != http.StatusOK { + t.Fatalf("public PROPFIND expected passthrough 200, got %d", publicResp.StatusCode) + } + + verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) + verifyBody := framework.ReadAllAndClose(t, verifyResp) + if verifyResp.StatusCode != http.StatusOK { + t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode) + } + if string(verifyBody) != "propfind-method-check" { + t.Fatalf("PROPFIND should not mutate data, got %q", string(verifyBody)) + } +} + +func TestUnsupportedMethodConnectParity(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(85) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 124001, 0x03030303) + client := framework.NewHTTPClient() + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("connect-method-check")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + adminReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumeAdminURL()+"/"+fid) + adminResp := framework.DoRequest(t, client, adminReq) + _ = framework.ReadAllAndClose(t, adminResp) + if adminResp.StatusCode != http.StatusBadRequest { + t.Fatalf("admin CONNECT expected 400, got %d", adminResp.StatusCode) + } + + publicReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumePublicURL()+"/"+fid) + publicResp := framework.DoRequest(t, client, publicReq) + _ = framework.ReadAllAndClose(t, publicResp) + if publicResp.StatusCode != http.StatusOK { + t.Fatalf("public CONNECT expected passthrough 200, got %d", publicResp.StatusCode) + } + + verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) + verifyBody := framework.ReadAllAndClose(t, verifyResp) + if verifyResp.StatusCode != http.StatusOK { + t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode) + } + if string(verifyBody) != "connect-method-check" { + t.Fatalf("CONNECT should not mutate data, got %q", string(verifyBody)) + } +} + +func TestPublicPortHeadReadParity(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(86) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 124002, 0x04040404) + payload := []byte("public-head-parity-content") + client := framework.NewHTTPClient() + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumePublicURL()+"/"+fid)) + headBody := framework.ReadAllAndClose(t, headResp) + if headResp.StatusCode != http.StatusOK { + t.Fatalf("public HEAD expected 200, got %d", headResp.StatusCode) + } + if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) { + t.Fatalf("public HEAD content-length mismatch: got %q want %d", got, len(payload)) + } + if len(headBody) != 0 { + t.Fatalf("public HEAD body should be empty, got %d bytes", len(headBody)) + } +} diff --git a/test/volume_server/http/range_variants_test.go b/test/volume_server/http/range_variants_test.go new file mode 100644 index 000000000..2e1f5e286 --- /dev/null +++ b/test/volume_server/http/range_variants_test.go @@ -0,0 +1,82 @@ +package volume_server_http_test + +import ( + "net/http" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestMultiRangeReadReturnsMultipartPayload(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(97) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 771999, 0x0A1B2C3D) + payload := []byte("0123456789abcdef") + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + multiRangeReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + multiRangeReq.Header.Set("Range", "bytes=0-1,4-5") + multiRangeResp := framework.DoRequest(t, client, multiRangeReq) + multiRangeBody := framework.ReadAllAndClose(t, multiRangeResp) + if multiRangeResp.StatusCode != http.StatusPartialContent { + t.Fatalf("multi-range expected 206, got %d", multiRangeResp.StatusCode) + } + if !strings.Contains(multiRangeResp.Header.Get("Content-Type"), "multipart/byteranges") { + t.Fatalf("multi-range content-type mismatch: %q", multiRangeResp.Header.Get("Content-Type")) + } + + bodyText := string(multiRangeBody) + if !strings.Contains(bodyText, "01") || !strings.Contains(bodyText, "45") { + t.Fatalf("multi-range body missing expected segments: %q", bodyText) + } +} + +func TestOversizedCombinedRangesAreIgnored(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(100) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 772003, 0x3C4D5E6F) + payload := []byte("0123456789abcdef") + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + // Range bytes sum is 22 (> payload size 16), which exercises the oversized-range guard path. + oversizedRangeReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + oversizedRangeReq.Header.Set("Range", "bytes=0-10,5-15") + oversizedRangeResp := framework.DoRequest(t, client, oversizedRangeReq) + oversizedRangeBody := framework.ReadAllAndClose(t, oversizedRangeResp) + if oversizedRangeResp.StatusCode != http.StatusOK { + t.Fatalf("oversized combined range expected 200, got %d", oversizedRangeResp.StatusCode) + } + if len(oversizedRangeBody) != 0 { + t.Fatalf("oversized combined range expected empty body, got %d bytes", len(oversizedRangeBody)) + } +} diff --git a/test/volume_server/http/read_deleted_test.go b/test/volume_server/http/read_deleted_test.go new file mode 100644 index 000000000..23d400e23 --- /dev/null +++ b/test/volume_server/http/read_deleted_test.go @@ -0,0 +1,54 @@ +package volume_server_http_test + +import ( + "net/http" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestReadDeletedQueryReturnsDeletedNeedleData(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(94) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 551234, 0xCAFE1234) + payload := []byte("read-deleted-needle-payload") + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid)) + _ = framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode) + } + + normalRead := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, normalRead) + if normalRead.StatusCode != http.StatusNotFound { + t.Fatalf("normal read after delete expected 404, got %d", normalRead.StatusCode) + } + + readDeletedReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true") + readDeletedResp := framework.DoRequest(t, client, readDeletedReq) + readDeletedBody := framework.ReadAllAndClose(t, readDeletedResp) + if readDeletedResp.StatusCode != http.StatusOK { + t.Fatalf("read with readDeleted=true expected 200, got %d", readDeletedResp.StatusCode) + } + if string(readDeletedBody) != string(payload) { + t.Fatalf("readDeleted body mismatch: got %q want %q", string(readDeletedBody), string(payload)) + } +} diff --git a/test/volume_server/http/read_mode_proxy_redirect_test.go b/test/volume_server/http/read_mode_proxy_redirect_test.go new file mode 100644 index 000000000..f82438043 --- /dev/null +++ b/test/volume_server/http/read_mode_proxy_redirect_test.go @@ -0,0 +1,319 @@ +package volume_server_http_test + +import ( + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestReadModeProxyMissingLocalVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P1() + profile.ReadMode = "proxy" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + + const volumeID = uint32(101) + framework.AllocateVolume(t, grpc0, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 120001, 0x0102ABCD) + payload := []byte("proxy-read-mode-forwarded-content") + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + var finalBody []byte + if !waitForHTTPStatus(t, client, readURL, http.StatusOK, 10*time.Second, func(resp *http.Response) { + finalBody = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("proxy read mode did not return 200 from non-owning volume server within deadline") + } + if string(finalBody) != string(payload) { + t.Fatalf("proxy read mode body mismatch: got %q want %q", string(finalBody), string(payload)) + } +} + +func TestReadModeRedirectMissingLocalVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P1() + profile.ReadMode = "redirect" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + + const volumeID = uint32(102) + framework.AllocateVolume(t, grpc0, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 120002, 0x0102DCBA) + payload := []byte("redirect-read-mode-content") + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + noRedirectClient := &http.Client{ + Timeout: 10 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + var redirectLocation string + if !waitForHTTPStatus(t, noRedirectClient, readURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) { + redirectLocation = resp.Header.Get("Location") + _ = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("redirect read mode did not return 301 from non-owning volume server within deadline") + } + if redirectLocation == "" { + t.Fatalf("redirect response missing Location header") + } + if !strings.Contains(redirectLocation, "proxied=true") { + t.Fatalf("redirect Location should include proxied=true, got %q", redirectLocation) + } + + followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, redirectLocation)) + followBody := framework.ReadAllAndClose(t, followResp) + if followResp.StatusCode != http.StatusOK { + t.Fatalf("following redirect expected 200, got %d", followResp.StatusCode) + } + if string(followBody) != string(payload) { + t.Fatalf("redirect-follow body mismatch: got %q want %q", string(followBody), string(payload)) + } +} + +func TestReadModeLocalMissingLocalVolumeReturnsNotFound(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P1() + profile.ReadMode = "local" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + + const volumeID = uint32(103) + framework.AllocateVolume(t, grpc0, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 120003, 0x0102BEEF) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, []byte("local-read-mode-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + readResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(1), fid) + _ = framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusNotFound { + t.Fatalf("local read mode expected 404 on non-owning server, got %d", readResp.StatusCode) + } +} + +func TestReadDeletedProxyModeOnMissingLocalVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P1() + profile.ReadMode = "proxy" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + + const volumeID = uint32(104) + framework.AllocateVolume(t, grpc0, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 120004, 0x0102CAFE) + payload := []byte("proxy-readDeleted-missing-local-content") + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL(0)+"/"+fid)) + _ = framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode) + } + + readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?readDeleted=true" + var proxiedBody []byte + if !waitForHTTPStatus(t, client, readURL, http.StatusOK, 10*time.Second, func(resp *http.Response) { + proxiedBody = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("proxy readDeleted path did not return 200 from non-owning volume server within deadline") + } + if string(proxiedBody) != string(payload) { + t.Fatalf("proxy readDeleted body mismatch: got %q want %q", string(proxiedBody), string(payload)) + } +} + +func TestReadDeletedRedirectModeDropsQueryParameterParity(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P1() + profile.ReadMode = "redirect" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + + const volumeID = uint32(105) + framework.AllocateVolume(t, grpc0, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 120005, 0x0102FACE) + payload := []byte("redirect-readDeleted-query-drop-parity") + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL(0)+"/"+fid)) + _ = framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode) + } + + noRedirectClient := &http.Client{ + Timeout: 10 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + redirectURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?readDeleted=true" + var location string + if !waitForHTTPStatus(t, noRedirectClient, redirectURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) { + location = resp.Header.Get("Location") + _ = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("redirect readDeleted path did not return 301 from non-owning volume server within deadline") + } + if location == "" { + t.Fatalf("redirect readDeleted response missing Location header") + } + if !strings.Contains(location, "proxied=true") { + t.Fatalf("redirect readDeleted Location should include proxied=true, got %q", location) + } + if strings.Contains(location, "readDeleted=true") { + t.Fatalf("redirect readDeleted Location should reflect current query-drop behavior, got %q", location) + } + + followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, location)) + _ = framework.ReadAllAndClose(t, followResp) + if followResp.StatusCode != http.StatusNotFound { + t.Fatalf("redirect-follow without readDeleted query expected 404 for deleted needle, got %d", followResp.StatusCode) + } +} + +func TestReadModeRedirectPreservesCollectionQuery(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P1() + profile.ReadMode = "redirect" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + + const volumeID = uint32(109) + const collection = "redirect-collection" + framework.AllocateVolume(t, grpc0, volumeID, collection) + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 120006, 0x0102F00D) + payload := []byte("redirect-collection-preserve-content") + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + noRedirectClient := &http.Client{ + Timeout: 10 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + redirectURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?collection=" + collection + var location string + if !waitForHTTPStatus(t, noRedirectClient, redirectURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) { + location = resp.Header.Get("Location") + _ = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("redirect collection path did not return 301 from non-owning volume server within deadline") + } + if location == "" { + t.Fatalf("redirect collection response missing Location header") + } + if !strings.Contains(location, "proxied=true") { + t.Fatalf("redirect collection Location should include proxied=true, got %q", location) + } + if !strings.Contains(location, "collection="+collection) { + t.Fatalf("redirect collection Location should preserve collection query, got %q", location) + } + + followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, location)) + followBody := framework.ReadAllAndClose(t, followResp) + if followResp.StatusCode != http.StatusOK { + t.Fatalf("redirect-follow expected 200, got %d", followResp.StatusCode) + } + if string(followBody) != string(payload) { + t.Fatalf("redirect-follow body mismatch: got %q want %q", string(followBody), string(payload)) + } +} + +func waitForHTTPStatus(t testing.TB, client *http.Client, url string, expectedStatus int, timeout time.Duration, onMatch func(resp *http.Response)) bool { + t.Helper() + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, url)) + if resp.StatusCode == expectedStatus { + onMatch(resp) + return true + } + _ = framework.ReadAllAndClose(t, resp) + time.Sleep(200 * time.Millisecond) + } + + return false +} diff --git a/test/volume_server/http/read_path_variants_test.go b/test/volume_server/http/read_path_variants_test.go new file mode 100644 index 000000000..97a7ac628 --- /dev/null +++ b/test/volume_server/http/read_path_variants_test.go @@ -0,0 +1,191 @@ +package volume_server_http_test + +import ( + "fmt" + "net/http" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestReadPathShapesAndIfModifiedSince(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(93) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fullFileID := framework.NewFileID(volumeID, 771234, 0xBEEFCACE) + uploadPayload := []byte("read-path-shape-content") + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fullFileID, uploadPayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + parts := strings.SplitN(fullFileID, ",", 2) + if len(parts) != 2 { + t.Fatalf("unexpected file id format: %q", fullFileID) + } + fidOnly := parts[1] + + readByVidFid := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, fmt.Sprintf("%s/%d/%s", clusterHarness.VolumeAdminURL(), volumeID, fidOnly))) + readByVidFidBody := framework.ReadAllAndClose(t, readByVidFid) + if readByVidFid.StatusCode != http.StatusOK { + t.Fatalf("GET /{vid}/{fid} expected 200, got %d", readByVidFid.StatusCode) + } + if string(readByVidFidBody) != string(uploadPayload) { + t.Fatalf("GET /{vid}/{fid} body mismatch: got %q want %q", string(readByVidFidBody), string(uploadPayload)) + } + + readWithFilename := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, fmt.Sprintf("%s/%d/%s/%s", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "named.bin"))) + readWithFilenameBody := framework.ReadAllAndClose(t, readWithFilename) + if readWithFilename.StatusCode != http.StatusOK { + t.Fatalf("GET /{vid}/{fid}/{filename} expected 200, got %d", readWithFilename.StatusCode) + } + if string(readWithFilenameBody) != string(uploadPayload) { + t.Fatalf("GET /{vid}/{fid}/{filename} body mismatch: got %q want %q", string(readWithFilenameBody), string(uploadPayload)) + } + + lastModified := readWithFilename.Header.Get("Last-Modified") + if lastModified == "" { + t.Fatalf("expected Last-Modified header on read response") + } + + ifModifiedSinceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fullFileID) + ifModifiedSinceReq.Header.Set("If-Modified-Since", lastModified) + ifModifiedSinceResp := framework.DoRequest(t, client, ifModifiedSinceReq) + _ = framework.ReadAllAndClose(t, ifModifiedSinceResp) + if ifModifiedSinceResp.StatusCode != http.StatusNotModified { + t.Fatalf("If-Modified-Since expected 304, got %d", ifModifiedSinceResp.StatusCode) + } + + headIfModifiedSinceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fullFileID) + headIfModifiedSinceReq.Header.Set("If-Modified-Since", lastModified) + headIfModifiedSinceResp := framework.DoRequest(t, client, headIfModifiedSinceReq) + headIfModifiedSinceBody := framework.ReadAllAndClose(t, headIfModifiedSinceResp) + if headIfModifiedSinceResp.StatusCode != http.StatusNotModified { + t.Fatalf("HEAD If-Modified-Since expected 304, got %d", headIfModifiedSinceResp.StatusCode) + } + if len(headIfModifiedSinceBody) != 0 { + t.Fatalf("HEAD If-Modified-Since expected empty body, got %d bytes", len(headIfModifiedSinceBody)) + } +} + +func TestMalformedVidFidPathReturnsBadRequest(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + client := framework.NewHTTPClient() + + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/not-a-vid/not-a-fid")) + _ = framework.ReadAllAndClose(t, resp) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("malformed /{vid}/{fid} expected 400, got %d", resp.StatusCode) + } +} + +func TestReadWrongCookieReturnsNotFound(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(95) + const needleID = uint64(771235) + const cookie = uint32(0xBEEFCACF) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, cookie) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("read-cookie-mismatch-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + wrongCookieFid := framework.NewFileID(volumeID, needleID, cookie+1) + getResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), wrongCookieFid) + _ = framework.ReadAllAndClose(t, getResp) + if getResp.StatusCode != http.StatusNotFound { + t.Fatalf("GET with wrong cookie expected 404, got %d", getResp.StatusCode) + } + + headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+wrongCookieFid)) + headBody := framework.ReadAllAndClose(t, headResp) + if headResp.StatusCode != http.StatusNotFound { + t.Fatalf("HEAD with wrong cookie expected 404, got %d", headResp.StatusCode) + } + if len(headBody) != 0 { + t.Fatalf("HEAD wrong-cookie response body should be empty, got %d bytes", len(headBody)) + } +} + +func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(99) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 772002, 0x2B3C4D5E) + payload := []byte("conditional-precedence-content") + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + baselineResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, baselineResp) + if baselineResp.StatusCode != http.StatusOK { + t.Fatalf("baseline read expected 200, got %d", baselineResp.StatusCode) + } + lastModified := baselineResp.Header.Get("Last-Modified") + if lastModified == "" { + t.Fatalf("baseline read expected Last-Modified header") + } + + precedenceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + precedenceReq.Header.Set("If-Modified-Since", lastModified) + precedenceReq.Header.Set("If-None-Match", "\"definitely-different-etag\"") + precedenceResp := framework.DoRequest(t, client, precedenceReq) + precedenceBody := framework.ReadAllAndClose(t, precedenceResp) + if precedenceResp.StatusCode != http.StatusNotModified { + t.Fatalf("conditional precedence expected 304, got %d", precedenceResp.StatusCode) + } + if len(precedenceBody) != 0 { + t.Fatalf("conditional precedence expected empty body, got %d bytes", len(precedenceBody)) + } + + invalidIMSReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + invalidIMSReq.Header.Set("If-Modified-Since", "not-a-valid-http-date") + invalidIMSReq.Header.Set("If-None-Match", "\"definitely-different-etag\"") + invalidIMSResp := framework.DoRequest(t, client, invalidIMSReq) + invalidIMSBody := framework.ReadAllAndClose(t, invalidIMSResp) + if invalidIMSResp.StatusCode != http.StatusOK { + t.Fatalf("invalid If-Modified-Since with mismatched etag expected 200, got %d", invalidIMSResp.StatusCode) + } + if string(invalidIMSBody) != string(payload) { + t.Fatalf("invalid If-Modified-Since fallback body mismatch: got %q want %q", string(invalidIMSBody), string(payload)) + } +} diff --git a/test/volume_server/http/read_write_delete_test.go b/test/volume_server/http/read_write_delete_test.go new file mode 100644 index 000000000..b122d697c --- /dev/null +++ b/test/volume_server/http/read_write_delete_test.go @@ -0,0 +1,123 @@ +package volume_server_http_test + +import ( + "net/http" + "strconv" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestUploadReadRangeHeadDeleteRoundTrip(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(7) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 123456, 0xA1B2C3D4) + data := []byte("hello-volume-server-integration") + client := framework.NewHTTPClient() + + uploadResp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(), fid, data) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode) + } + + getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid) + getBody := framework.ReadAllAndClose(t, getResp) + if getResp.StatusCode != http.StatusOK { + t.Fatalf("get status: expected 200, got %d", getResp.StatusCode) + } + if string(getBody) != string(data) { + t.Fatalf("get body mismatch: got %q want %q", string(getBody), string(data)) + } + etag := getResp.Header.Get("ETag") + if etag == "" { + t.Fatalf("expected ETag header from GET response") + } + + notModifiedReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid) + notModifiedReq.Header.Set("If-None-Match", etag) + notModifiedResp := framework.DoRequest(t, client, notModifiedReq) + _ = framework.ReadAllAndClose(t, notModifiedResp) + if notModifiedResp.StatusCode != http.StatusNotModified { + t.Fatalf("if-none-match expected 304, got %d", notModifiedResp.StatusCode) + } + + rangeReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid) + rangeReq.Header.Set("Range", "bytes=0-4") + rangeResp := framework.DoRequest(t, client, rangeReq) + rangeBody := framework.ReadAllAndClose(t, rangeResp) + if rangeResp.StatusCode != http.StatusPartialContent { + t.Fatalf("range status: expected 206, got %d", rangeResp.StatusCode) + } + if got, want := string(rangeBody), "hello"; got != want { + t.Fatalf("range body mismatch: got %q want %q", got, want) + } + + invalidRangeReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid) + invalidRangeReq.Header.Set("Range", "bytes=9999-10000") + invalidRangeResp := framework.DoRequest(t, client, invalidRangeReq) + _ = framework.ReadAllAndClose(t, invalidRangeResp) + if invalidRangeResp.StatusCode != http.StatusRequestedRangeNotSatisfiable { + t.Fatalf("invalid range expected 416, got %d", invalidRangeResp.StatusCode) + } + + headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, cluster.VolumeAdminURL()+"/"+fid)) + headBody := framework.ReadAllAndClose(t, headResp) + if headResp.StatusCode != http.StatusOK { + t.Fatalf("head status: expected 200, got %d", headResp.StatusCode) + } + if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(data)) { + t.Fatalf("head content-length mismatch: got %q want %d", got, len(data)) + } + if len(headBody) != 0 { + t.Fatalf("head body should be empty, got %d bytes", len(headBody)) + } + + headNotModifiedReq := mustNewRequest(t, http.MethodHead, cluster.VolumeAdminURL()+"/"+fid) + headNotModifiedReq.Header.Set("If-None-Match", etag) + headNotModifiedResp := framework.DoRequest(t, client, headNotModifiedReq) + headNotModifiedBody := framework.ReadAllAndClose(t, headNotModifiedResp) + if headNotModifiedResp.StatusCode != http.StatusNotModified { + t.Fatalf("head if-none-match expected 304, got %d", headNotModifiedResp.StatusCode) + } + if len(headNotModifiedBody) != 0 { + t.Fatalf("head if-none-match body should be empty, got %d bytes", len(headNotModifiedBody)) + } + + deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, cluster.VolumeAdminURL()+"/"+fid)) + _ = framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("delete status: expected 202, got %d", deleteResp.StatusCode) + } + + notFoundResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, notFoundResp) + if notFoundResp.StatusCode != http.StatusNotFound { + t.Fatalf("read after delete: expected 404, got %d", notFoundResp.StatusCode) + } +} + +func TestInvalidReadPathReturnsBadRequest(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + client := framework.NewHTTPClient() + + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/invalid,needle")) + _ = framework.ReadAllAndClose(t, resp) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("invalid read expected 400, got %d", resp.StatusCode) + } +} diff --git a/test/volume_server/http/throttling_test.go b/test/volume_server/http/throttling_test.go new file mode 100644 index 000000000..7a66e9ebb --- /dev/null +++ b/test/volume_server/http/throttling_test.go @@ -0,0 +1,730 @@ +package volume_server_http_test + +import ( + "bytes" + "context" + "io" + "net/http" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" +) + +type pausableReader struct { + remaining int64 + pauseAfter int64 + paused bool + unblock <-chan struct{} +} + +func (r *pausableReader) Read(p []byte) (int, error) { + if r.remaining <= 0 { + return 0, io.EOF + } + if !r.paused && r.pauseAfter > 0 { + n := int64(len(p)) + if n > r.pauseAfter { + n = r.pauseAfter + } + for i := int64(0); i < n; i++ { + p[i] = 'a' + } + r.remaining -= n + r.pauseAfter -= n + if r.pauseAfter == 0 { + r.paused = true + } + return int(n), nil + } + if r.paused { + <-r.unblock + r.paused = false + } + n := int64(len(p)) + if n > r.remaining { + n = r.remaining + } + for i := int64(0); i < n; i++ { + p[i] = 'b' + } + r.remaining -= n + return int(n), nil +} + +func TestUploadLimitTimeoutAndReplicateBypass(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(98) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + const blockedUploadSize = 2 * 1024 * 1024 // over 1MB P8 upload limit + + unblockFirstUpload := make(chan struct{}) + firstUploadDone := make(chan error, 1) + firstFID := framework.NewFileID(volumeID, 880001, 0x1A2B3C4D) + go func() { + req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{ + remaining: blockedUploadSize, + pauseAfter: 1, + unblock: unblockFirstUpload, + }) + if err != nil { + firstUploadDone <- err + return + } + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = blockedUploadSize + + resp, err := (&http.Client{}).Do(req) + if resp != nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } + firstUploadDone <- err + }() + + // Give the first upload time to pass limit checks and block in body processing. + time.Sleep(300 * time.Millisecond) + + replicateFID := framework.NewFileID(volumeID, 880002, 0x5E6F7A8B) + replicateReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+replicateFID+"?type=replicate", bytes.NewReader([]byte("replicate"))) + if err != nil { + t.Fatalf("create replicate request: %v", err) + } + replicateReq.Header.Set("Content-Type", "application/octet-stream") + replicateReq.ContentLength = int64(len("replicate")) + replicateResp, err := framework.NewHTTPClient().Do(replicateReq) + if err != nil { + t.Fatalf("replicate request failed: %v", err) + } + _ = framework.ReadAllAndClose(t, replicateResp) + if replicateResp.StatusCode != http.StatusCreated { + t.Fatalf("replicate request expected 201 bypassing limit, got %d", replicateResp.StatusCode) + } + + normalFID := framework.NewFileID(volumeID, 880003, 0x9C0D1E2F) + normalReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+normalFID, bytes.NewReader([]byte("normal"))) + if err != nil { + t.Fatalf("create normal request: %v", err) + } + normalReq.Header.Set("Content-Type", "application/octet-stream") + normalReq.ContentLength = int64(len("normal")) + + timeoutClient := &http.Client{Timeout: 10 * time.Second} + normalResp, err := timeoutClient.Do(normalReq) + if err != nil { + t.Fatalf("normal upload request failed: %v", err) + } + _ = framework.ReadAllAndClose(t, normalResp) + if normalResp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("normal upload expected 429 while limit blocked, got %d", normalResp.StatusCode) + } + + close(unblockFirstUpload) + select { + case <-firstUploadDone: + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for blocked upload to finish") + } +} + +func TestUploadLimitWaitThenProceed(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(111) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + const blockedUploadSize = 2 * 1024 * 1024 + + unblockFirstUpload := make(chan struct{}) + firstUploadDone := make(chan error, 1) + firstFID := framework.NewFileID(volumeID, 880601, 0x6A2B3C4D) + go func() { + req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{ + remaining: blockedUploadSize, + pauseAfter: 1, + unblock: unblockFirstUpload, + }) + if err != nil { + firstUploadDone <- err + return + } + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = blockedUploadSize + + resp, err := (&http.Client{}).Do(req) + if resp != nil { + _ = framework.ReadAllAndClose(t, resp) + } + firstUploadDone <- err + }() + + time.Sleep(300 * time.Millisecond) + + type uploadResult struct { + resp *http.Response + err error + } + secondUploadDone := make(chan uploadResult, 1) + secondFID := framework.NewFileID(volumeID, 880602, 0x6A2B3C4E) + go func() { + req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+secondFID, bytes.NewReader([]byte("wait-then-proceed"))) + if err != nil { + secondUploadDone <- uploadResult{err: err} + return + } + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = int64(len("wait-then-proceed")) + resp, err := (&http.Client{Timeout: 10 * time.Second}).Do(req) + secondUploadDone <- uploadResult{resp: resp, err: err} + }() + + time.Sleep(500 * time.Millisecond) + close(unblockFirstUpload) + + select { + case firstErr := <-firstUploadDone: + if firstErr != nil { + t.Fatalf("first blocked upload failed: %v", firstErr) + } + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for first upload completion") + } + + select { + case result := <-secondUploadDone: + if result.err != nil { + t.Fatalf("second upload failed: %v", result.err) + } + _ = framework.ReadAllAndClose(t, result.resp) + if result.resp.StatusCode != http.StatusCreated { + t.Fatalf("second upload expected 201 after waiting for slot, got %d", result.resp.StatusCode) + } + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for second upload completion") + } +} + +func TestUploadLimitTimeoutThenRecovery(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(113) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + const blockedUploadSize = 2 * 1024 * 1024 + + unblockFirstUpload := make(chan struct{}) + firstUploadDone := make(chan error, 1) + firstFID := framework.NewFileID(volumeID, 880801, 0x7A2B3C4D) + go func() { + req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{ + remaining: blockedUploadSize, + pauseAfter: 1, + unblock: unblockFirstUpload, + }) + if err != nil { + firstUploadDone <- err + return + } + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = blockedUploadSize + resp, err := (&http.Client{}).Do(req) + if resp != nil { + _ = framework.ReadAllAndClose(t, resp) + } + firstUploadDone <- err + }() + + time.Sleep(300 * time.Millisecond) + + timeoutFID := framework.NewFileID(volumeID, 880802, 0x7A2B3C4E) + timeoutResp := framework.UploadBytes(t, &http.Client{Timeout: 10 * time.Second}, clusterHarness.VolumeAdminURL(), timeoutFID, []byte("should-timeout")) + _ = framework.ReadAllAndClose(t, timeoutResp) + if timeoutResp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("second upload under blocked pressure expected 429, got %d", timeoutResp.StatusCode) + } + + close(unblockFirstUpload) + select { + case firstErr := <-firstUploadDone: + if firstErr != nil { + t.Fatalf("first blocked upload failed: %v", firstErr) + } + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for first upload completion") + } + + recoveryFID := framework.NewFileID(volumeID, 880803, 0x7A2B3C4F) + recoveryResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), recoveryFID, []byte("recovered-upload")) + _ = framework.ReadAllAndClose(t, recoveryResp) + if recoveryResp.StatusCode != http.StatusCreated { + t.Fatalf("recovery upload expected 201, got %d", recoveryResp.StatusCode) + } +} + +func TestDownloadLimitTimeoutReturnsTooManyRequests(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(99) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + largePayload := make([]byte, 12*1024*1024) // over 1MB P8 download limit + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + downloadFID := framework.NewFileID(volumeID, 880101, 0x10203040) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), downloadFID, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+downloadFID)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + defer firstResp.Body.Close() + + // Keep first response body unread so server write path stays in-flight. + time.Sleep(300 * time.Millisecond) + + secondClient := &http.Client{Timeout: 10 * time.Second} + secondResp, err := secondClient.Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+downloadFID)) + if err != nil { + t.Fatalf("second GET failed: %v", err) + } + _ = framework.ReadAllAndClose(t, secondResp) + if secondResp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("second GET expected 429 while first download holds limit, got %d", secondResp.StatusCode) + } +} + +func TestDownloadLimitWaitThenProceedWithoutReplica(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(112) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + largePayload := make([]byte, 12*1024*1024) + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + fid := framework.NewFileID(volumeID, 880701, 0x60708090) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + + type readResult struct { + resp *http.Response + err error + } + secondReadDone := make(chan readResult, 1) + go func() { + resp, readErr := (&http.Client{Timeout: 10 * time.Second}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) + secondReadDone <- readResult{resp: resp, err: readErr} + }() + + time.Sleep(500 * time.Millisecond) + _ = firstResp.Body.Close() + + select { + case result := <-secondReadDone: + if result.err != nil { + t.Fatalf("second GET failed: %v", result.err) + } + secondBody := framework.ReadAllAndClose(t, result.resp) + if result.resp.StatusCode != http.StatusOK { + t.Fatalf("second GET expected 200 after waiting for slot, got %d", result.resp.StatusCode) + } + if len(secondBody) != len(largePayload) { + t.Fatalf("second GET body size mismatch: got %d want %d", len(secondBody), len(largePayload)) + } + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for second GET completion") + } +} + +func TestDownloadLimitTimeoutThenRecovery(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(114) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + largePayload := make([]byte, 12*1024*1024) + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + fid := framework.NewFileID(volumeID, 880901, 0x708090A0) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + + time.Sleep(300 * time.Millisecond) + + timeoutResp := framework.ReadBytes(t, &http.Client{Timeout: 10 * time.Second}, clusterHarness.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, timeoutResp) + if timeoutResp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("second GET under blocked pressure expected 429, got %d", timeoutResp.StatusCode) + } + + _ = firstResp.Body.Close() + + recoveryResp := framework.ReadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid) + recoveryBody := framework.ReadAllAndClose(t, recoveryResp) + if recoveryResp.StatusCode != http.StatusOK { + t.Fatalf("recovery GET expected 200, got %d", recoveryResp.StatusCode) + } + if len(recoveryBody) != len(largePayload) { + t.Fatalf("recovery GET body size mismatch: got %d want %d", len(recoveryBody), len(largePayload)) + } +} + +func TestDownloadLimitOverageProxiesToReplica(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P8() + profile.ReadMode = "proxy" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + conn1, grpc1 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) + defer conn1.Close() + + const volumeID = uint32(100) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + req := &volume_server_pb.AllocateVolumeRequest{ + VolumeId: volumeID, + Replication: "001", + Version: uint32(needle.GetCurrentVersion()), + } + if _, err := grpc0.AllocateVolume(ctx, req); err != nil { + t.Fatalf("allocate replicated volume on node0: %v", err) + } + if _, err := grpc1.AllocateVolume(ctx, req); err != nil { + t.Fatalf("allocate replicated volume on node1: %v", err) + } + + largePayload := make([]byte, 12*1024*1024) + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + fid := framework.NewFileID(volumeID, 880201, 0x0A0B0C0D) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(0), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("replicated large upload expected 201, got %d", uploadResp.StatusCode) + } + + replicaReadURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + if !waitForHTTPStatus(t, framework.NewHTTPClient(), replicaReadURL, http.StatusOK, 10*time.Second, func(resp *http.Response) { + _ = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("replica did not become readable within deadline") + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + defer firstResp.Body.Close() + + time.Sleep(300 * time.Millisecond) + + secondResp, err := framework.NewHTTPClient().Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid)) + if err != nil { + t.Fatalf("second GET failed: %v", err) + } + secondBody := framework.ReadAllAndClose(t, secondResp) + if secondResp.StatusCode != http.StatusOK { + t.Fatalf("second GET expected 200 via replica proxy fallback, got %d", secondResp.StatusCode) + } + if len(secondBody) != len(largePayload) { + t.Fatalf("second GET proxied body size mismatch: got %d want %d", len(secondBody), len(largePayload)) + } +} + +func TestDownloadLimitProxiedRequestSkipsReplicaFallbackAndTimesOut(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P8() + profile.ReadMode = "proxy" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + conn1, grpc1 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) + defer conn1.Close() + + const volumeID = uint32(106) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + req := &volume_server_pb.AllocateVolumeRequest{ + VolumeId: volumeID, + Replication: "001", + Version: uint32(needle.GetCurrentVersion()), + } + if _, err := grpc0.AllocateVolume(ctx, req); err != nil { + t.Fatalf("allocate replicated volume on node0: %v", err) + } + if _, err := grpc1.AllocateVolume(ctx, req); err != nil { + t.Fatalf("allocate replicated volume on node1: %v", err) + } + + largePayload := make([]byte, 12*1024*1024) + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + fid := framework.NewFileID(volumeID, 880202, 0x0A0B0D0E) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(0), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("replicated large upload expected 201, got %d", uploadResp.StatusCode) + } + + // Ensure replica path is actually available, so a non-proxied request would proxy. + replicaReadURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + if !waitForHTTPStatus(t, framework.NewHTTPClient(), replicaReadURL, http.StatusOK, 10*time.Second, func(resp *http.Response) { + _ = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("replica did not become readable within deadline") + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + defer firstResp.Body.Close() + + time.Sleep(300 * time.Millisecond) + + // proxied=true should bypass replica fallback and hit wait/timeout branch. + secondResp, err := framework.NewHTTPClient().Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid+"?proxied=true")) + if err != nil { + t.Fatalf("second GET failed: %v", err) + } + _ = framework.ReadAllAndClose(t, secondResp) + if secondResp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("second GET with proxied=true expected 429 timeout path, got %d", secondResp.StatusCode) + } +} + +func TestUploadLimitDisabledAllowsConcurrentUploads(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(107) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + const blockedUploadSize = 2 * 1024 * 1024 + unblockFirstUpload := make(chan struct{}) + firstUploadDone := make(chan error, 1) + firstFID := framework.NewFileID(volumeID, 880301, 0x1A2B3C5D) + go func() { + req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{ + remaining: blockedUploadSize, + pauseAfter: 1, + unblock: unblockFirstUpload, + }) + if err != nil { + firstUploadDone <- err + return + } + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = blockedUploadSize + + resp, err := (&http.Client{}).Do(req) + if resp != nil { + _ = framework.ReadAllAndClose(t, resp) + } + firstUploadDone <- err + }() + + time.Sleep(300 * time.Millisecond) + + secondFID := framework.NewFileID(volumeID, 880302, 0x1A2B3C5E) + secondResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), secondFID, []byte("no-limit-second-upload")) + _ = framework.ReadAllAndClose(t, secondResp) + if secondResp.StatusCode != http.StatusCreated { + t.Fatalf("second upload with disabled limit expected 201, got %d", secondResp.StatusCode) + } + + close(unblockFirstUpload) + select { + case <-firstUploadDone: + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for first upload completion") + } +} + +func TestDownloadLimitDisabledAllowsConcurrentDownloads(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(108) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + largePayload := make([]byte, 12*1024*1024) + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + fid := framework.NewFileID(volumeID, 880401, 0x20304050) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + defer firstResp.Body.Close() + + time.Sleep(300 * time.Millisecond) + + secondResp := framework.ReadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid) + secondBody := framework.ReadAllAndClose(t, secondResp) + if secondResp.StatusCode != http.StatusOK { + t.Fatalf("second GET with disabled limit expected 200, got %d", secondResp.StatusCode) + } + if len(secondBody) != len(largePayload) { + t.Fatalf("second GET body size mismatch: got %d want %d", len(secondBody), len(largePayload)) + } +} + +func TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(110) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + largePayload := make([]byte, 12*1024*1024) + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + fid := framework.NewFileID(volumeID, 880501, 0x50607080) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + defer firstResp.Body.Close() + + time.Sleep(300 * time.Millisecond) + + invalidReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/not-a-vid,1234567890ab") + invalidResp := framework.DoRequest(t, framework.NewHTTPClient(), invalidReq) + _ = framework.ReadAllAndClose(t, invalidResp) + if invalidResp.StatusCode != http.StatusBadRequest { + t.Fatalf("invalid vid while over limit expected 400, got %d", invalidResp.StatusCode) + } +} diff --git a/test/volume_server/http/write_delete_variants_test.go b/test/volume_server/http/write_delete_variants_test.go new file mode 100644 index 000000000..3355e7778 --- /dev/null +++ b/test/volume_server/http/write_delete_variants_test.go @@ -0,0 +1,118 @@ +package volume_server_http_test + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestWriteUnchangedAndDeleteEdgeVariants(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(87) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + const key = uint64(999001) + const cookie = uint32(0xDEADBEEF) + fid := framework.NewFileID(volumeID, key, cookie) + client := framework.NewHTTPClient() + payload := []byte("unchanged-write-content") + + firstUpload := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + firstUploadResp := framework.DoRequest(t, client, firstUpload) + _ = framework.ReadAllAndClose(t, firstUploadResp) + if firstUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("first upload expected 201, got %d", firstUploadResp.StatusCode) + } + + secondUpload := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + secondUploadResp := framework.DoRequest(t, client, secondUpload) + _ = framework.ReadAllAndClose(t, secondUploadResp) + if secondUploadResp.StatusCode != http.StatusNoContent { + t.Fatalf("second unchanged upload expected 204, got %d", secondUploadResp.StatusCode) + } + if secondUploadResp.Header.Get("ETag") == "" { + t.Fatalf("second unchanged upload expected ETag header") + } + + wrongCookieFid := framework.NewFileID(volumeID, key, cookie+1) + wrongCookieDelete := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+wrongCookieFid)) + _ = framework.ReadAllAndClose(t, wrongCookieDelete) + if wrongCookieDelete.StatusCode != http.StatusBadRequest { + t.Fatalf("delete with mismatched cookie expected 400, got %d", wrongCookieDelete.StatusCode) + } + + missingDelete := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+framework.NewFileID(volumeID, key+1, cookie))) + missingDeleteBody := framework.ReadAllAndClose(t, missingDelete) + if missingDelete.StatusCode != http.StatusNotFound { + t.Fatalf("delete missing needle expected 404, got %d", missingDelete.StatusCode) + } + + var payloadMap map[string]int64 + if err := json.Unmarshal(missingDeleteBody, &payloadMap); err != nil { + t.Fatalf("decode delete missing response: %v", err) + } + if payloadMap["size"] != 0 { + t.Fatalf("delete missing needle expected size=0, got %d", payloadMap["size"]) + } +} + +func TestDeleteTimestampOverrideKeepsReadDeletedLastModifiedParity(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(88) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 999002, 0xABCD1234) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("delete-ts-override")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + beforeDeleteResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, beforeDeleteResp) + if beforeDeleteResp.StatusCode != http.StatusOK { + t.Fatalf("pre-delete read expected 200, got %d", beforeDeleteResp.StatusCode) + } + lastModifiedBeforeDelete := beforeDeleteResp.Header.Get("Last-Modified") + if lastModifiedBeforeDelete == "" { + t.Fatalf("expected Last-Modified before delete") + } + + deleteReq := mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid+"?ts=1700000000") + deleteResp := framework.DoRequest(t, client, deleteReq) + _ = framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("delete with ts override expected 202, got %d", deleteResp.StatusCode) + } + + readDeletedResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true")) + _ = framework.ReadAllAndClose(t, readDeletedResp) + if readDeletedResp.StatusCode != http.StatusOK { + t.Fatalf("readDeleted after ts override expected 200, got %d", readDeletedResp.StatusCode) + } + lastModified := readDeletedResp.Header.Get("Last-Modified") + if lastModified == "" { + t.Fatalf("expected Last-Modified header on readDeleted response") + } + if lastModified != lastModifiedBeforeDelete { + t.Fatalf("expected readDeleted Last-Modified parity with pre-delete header, got %q want %q", lastModified, lastModifiedBeforeDelete) + } +} diff --git a/test/volume_server/http/write_error_variants_test.go b/test/volume_server/http/write_error_variants_test.go new file mode 100644 index 000000000..ead11ed6c --- /dev/null +++ b/test/volume_server/http/write_error_variants_test.go @@ -0,0 +1,74 @@ +package volume_server_http_test + +import ( + "net/http" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestWriteInvalidVidAndFidReturnBadRequest(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + client := framework.NewHTTPClient() + + invalidVidReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/invalid,12345678", []byte("x")) + invalidVidResp := framework.DoRequest(t, client, invalidVidReq) + _ = framework.ReadAllAndClose(t, invalidVidResp) + if invalidVidResp.StatusCode != http.StatusBadRequest { + t.Fatalf("write with invalid vid expected 400, got %d", invalidVidResp.StatusCode) + } + + invalidFidReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/1,bad", []byte("x")) + invalidFidResp := framework.DoRequest(t, client, invalidFidReq) + _ = framework.ReadAllAndClose(t, invalidFidResp) + if invalidFidResp.StatusCode != http.StatusBadRequest { + t.Fatalf("write with invalid fid expected 400, got %d", invalidFidResp.StatusCode) + } +} + +func TestWriteMalformedMultipartAndMD5Mismatch(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(98) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 772001, 0x1A2B3C4D) + + malformedMultipartReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+fid, strings.NewReader("not-a-valid-multipart-body")) + if err != nil { + t.Fatalf("create malformed multipart request: %v", err) + } + malformedMultipartReq.Header.Set("Content-Type", "multipart/form-data") + malformedMultipartResp := framework.DoRequest(t, client, malformedMultipartReq) + malformedMultipartBody := framework.ReadAllAndClose(t, malformedMultipartResp) + if malformedMultipartResp.StatusCode != http.StatusBadRequest { + t.Fatalf("malformed multipart write expected 400, got %d", malformedMultipartResp.StatusCode) + } + if !strings.Contains(strings.ToLower(string(malformedMultipartBody)), "boundary") { + t.Fatalf("malformed multipart response should mention boundary parse failure, got %q", string(malformedMultipartBody)) + } + + md5MismatchReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, []byte("content-md5-mismatch-body")) + md5MismatchReq.Header.Set("Content-MD5", "AAAAAAAAAAAAAAAAAAAAAA==") + md5MismatchResp := framework.DoRequest(t, client, md5MismatchReq) + md5MismatchBody := framework.ReadAllAndClose(t, md5MismatchResp) + if md5MismatchResp.StatusCode != http.StatusBadRequest { + t.Fatalf("content-md5 mismatch write expected 400, got %d", md5MismatchResp.StatusCode) + } + if !strings.Contains(string(md5MismatchBody), "Content-MD5") { + t.Fatalf("content-md5 mismatch response should mention Content-MD5, got %q", string(md5MismatchBody)) + } +} diff --git a/test/volume_server/matrix/config_profiles.go b/test/volume_server/matrix/config_profiles.go new file mode 100644 index 000000000..c359eb029 --- /dev/null +++ b/test/volume_server/matrix/config_profiles.go @@ -0,0 +1,63 @@ +package matrix + +import "time" + +// Profile describes one runtime test matrix configuration. +type Profile struct { + Name string + + ReadMode string + SplitPublicPort bool + + EnableJWT bool + JWTSigningKey string + JWTReadKey string + EnableMaintain bool + + ConcurrentUploadLimitMB int + ConcurrentDownloadLimitMB int + InflightUploadTimeout time.Duration + InflightDownloadTimeout time.Duration + + ReplicatedLayout bool + HasErasureCoding bool + HasRemoteTier bool +} + +// P1 is the baseline profile: one volume server, no JWT, proxy read mode. +func P1() Profile { + return Profile{ + Name: "P1", + ReadMode: "proxy", + SplitPublicPort: false, + } +} + +// P2 uses split public/admin ports to verify public read-only behavior. +func P2() Profile { + p := P1() + p.Name = "P2" + p.SplitPublicPort = true + return p +} + +// P3 enables JWT verification for read/write flows. +func P3() Profile { + p := P1() + p.Name = "P3" + p.EnableJWT = true + p.JWTSigningKey = "volume-server-write-key" + p.JWTReadKey = "volume-server-read-key" + return p +} + +// P8 enables upload/download throttling branches. +func P8() Profile { + p := P1() + p.Name = "P8" + p.ConcurrentUploadLimitMB = 1 + p.ConcurrentDownloadLimitMB = 1 + p.InflightUploadTimeout = 2 * time.Second + p.InflightDownloadTimeout = 2 * time.Second + return p +} diff --git a/weed/iam/sts/credential_prefix_test.go b/weed/iam/sts/credential_prefix_test.go new file mode 100644 index 000000000..b3bd0d331 --- /dev/null +++ b/weed/iam/sts/credential_prefix_test.go @@ -0,0 +1,68 @@ +package sts + +import ( + "encoding/base64" + "encoding/hex" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestTemporaryCredentialPrefix verifies that temporary credentials use ASIA prefix +// (not AKIA which is for permanent IAM user credentials) +func TestTemporaryCredentialPrefix(t *testing.T) { + sessionId := "test-session-for-prefix" + expiration := time.Now().Add(time.Hour) + + credGen := NewCredentialGenerator() + cred, err := credGen.GenerateTemporaryCredentials(sessionId, expiration) + + assert.NoError(t, err) + assert.NotNil(t, cred) + + // Verify ASIA prefix for temporary credentials + assert.True(t, strings.HasPrefix(cred.AccessKeyId, "ASIA"), + "Temporary credentials must use ASIA prefix, got: %s", cred.AccessKeyId) + + // Verify it's NOT using AKIA (permanent credentials) + assert.False(t, strings.HasPrefix(cred.AccessKeyId, "AKIA"), + "Temporary credentials must NOT use AKIA prefix (that's for permanent IAM keys)") +} + +// TestTemporaryCredentialFormat verifies the full format of temporary credentials +func TestTemporaryCredentialFormat(t *testing.T) { + sessionId := "format-test-session" + expiration := time.Now().Add(time.Hour) + + credGen := NewCredentialGenerator() + cred, err := credGen.GenerateTemporaryCredentials(sessionId, expiration) + + assert.NoError(t, err) + assert.NotNil(t, cred) + + // AWS temporary access key format: ASIA + 16 hex characters = 20 chars total + assert.Equal(t, 20, len(cred.AccessKeyId), + "Access key ID should be 20 characters (ASIA + 16 hex chars)") + + // Verify it starts with ASIA + assert.True(t, strings.HasPrefix(cred.AccessKeyId, "ASIA"), + "Access key must start with ASIA prefix") + + // Verify the rest is hex (after ASIA prefix) + hexPart := cred.AccessKeyId[4:] + assert.Equal(t, 16, len(hexPart), "Hex part should be 16 characters") + _, err = hex.DecodeString(hexPart) + assert.NoError(t, err, "The part after ASIA prefix should be valid hex") + + // Verify secret key is not empty and is a valid base64-encoded SHA256 hash + assert.NotEmpty(t, cred.SecretAccessKey) + assert.Equal(t, 44, len(cred.SecretAccessKey), + "SecretAccessKey should be 44 characters for a base64-encoded 32-byte hash") + _, err = base64.StdEncoding.DecodeString(cred.SecretAccessKey) + assert.NoError(t, err, "SecretAccessKey should be a valid base64 string") + + // Verify session token is not empty + assert.NotEmpty(t, cred.SessionToken) +} diff --git a/weed/iam/sts/token_utils.go b/weed/iam/sts/token_utils.go index a788287d8..69ab170ed 100644 --- a/weed/iam/sts/token_utils.go +++ b/weed/iam/sts/token_utils.go @@ -170,7 +170,7 @@ func NewCredentialGenerator() *CredentialGenerator { // GenerateTemporaryCredentials creates temporary AWS credentials func (c *CredentialGenerator) GenerateTemporaryCredentials(sessionId string, expiration time.Time) (*Credentials, error) { - accessKeyId, err := c.generateAccessKeyId(sessionId) + accessKeyId, err := c.generateTemporaryAccessKeyId(sessionId) if err != nil { return nil, fmt.Errorf("failed to generate access key ID: %w", err) } @@ -193,11 +193,12 @@ func (c *CredentialGenerator) GenerateTemporaryCredentials(sessionId string, exp }, nil } -// generateAccessKeyId generates an AWS-style access key ID -func (c *CredentialGenerator) generateAccessKeyId(sessionId string) (string, error) { +// generateTemporaryAccessKeyId generates an AWS-style access key ID for temporary STS credentials +func (c *CredentialGenerator) generateTemporaryAccessKeyId(sessionId string) (string, error) { // Create a deterministic but unique access key ID based on session hash := sha256.Sum256([]byte("access-key:" + sessionId)) - return "AKIA" + hex.EncodeToString(hash[:8]), nil // AWS format: AKIA + 16 chars + // Use ASIA prefix for temporary credentials (STS), not AKIA (permanent IAM keys) + return "ASIA" + hex.EncodeToString(hash[:8]), nil // AWS format: ASIA + 16 chars } // generateSecretAccessKey generates a deterministic secret access key based on sessionId diff --git a/weed/pb/server_address.go b/weed/pb/server_address.go index 88cadbb81..e1f5da74d 100644 --- a/weed/pb/server_address.go +++ b/weed/pb/server_address.go @@ -57,7 +57,7 @@ func (sa ServerAddress) ToHttpAddress() string { sepIndex := strings.LastIndex(string(ports), ".") if sepIndex >= 0 { host := string(sa[0:portsSepIndex]) - return net.JoinHostPort(host, ports[0:sepIndex]) + return util.JoinHostPortStr(host, ports[0:sepIndex]) } return string(sa) } @@ -74,7 +74,7 @@ func (sa ServerAddress) ToGrpcAddress() string { sepIndex := strings.LastIndex(ports, ".") if sepIndex >= 0 { host := string(sa[0:portsSepIndex]) - return net.JoinHostPort(host, ports[sepIndex+1:]) + return util.JoinHostPortStr(host, ports[sepIndex+1:]) } return ServerToGrpcAddress(string(sa)) } diff --git a/weed/pb/server_address_test.go b/weed/pb/server_address_test.go index f5a12427a..933a873c8 100644 --- a/weed/pb/server_address_test.go +++ b/weed/pb/server_address_test.go @@ -34,3 +34,36 @@ func TestServerAddresses_ToAddressMapOrSrv_shouldHandleIPPortList(t *testing.T) t.Fatalf(`Expected %q, got %q`, expected, d.list) } } + +func TestIPv6ServerAddressFormatting(t *testing.T) { + testCases := []struct { + name string + sa ServerAddress + expectedHttp string + expectedGrpc string + }{ + { + name: "unbracketed IPv6", + sa: NewServerAddress("2001:db8::1", 8080, 18080), + expectedHttp: "[2001:db8::1]:8080", + expectedGrpc: "[2001:db8::1]:18080", + }, + { + name: "bracketed IPv6", + sa: NewServerAddressWithGrpcPort("[2001:db8::1]:8080", 18080), + expectedHttp: "[2001:db8::1]:8080", + expectedGrpc: "[2001:db8::1]:18080", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if httpAddr := tc.sa.ToHttpAddress(); httpAddr != tc.expectedHttp { + t.Errorf("%s: ToHttpAddress() = %s, want %s", tc.name, httpAddr, tc.expectedHttp) + } + if grpcAddr := tc.sa.ToGrpcAddress(); grpcAddr != tc.expectedGrpc { + t.Errorf("%s: ToGrpcAddress() = %s, want %s", tc.name, grpcAddr, tc.expectedGrpc) + } + }) + } +} diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index d305f8b46..822b2cbf3 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -100,6 +100,9 @@ type Account struct { Id string } +// Default account ID for all automated SeaweedFS accounts and fallback +const defaultAccountID = "000000000000" + // Predefined Accounts var ( // AccountAdmin is used as the default account for IAM-Credentials access without Account configured @@ -809,7 +812,6 @@ func (iam *IdentityAccessManagement) MergeS3ApiConfiguration(config *iam_pb.S3Ap iam.nameToIdentity = nameToIdentity iam.accessKeyIdent = accessKeyIdent iam.policies = policies - iam.accessKeyIdent = accessKeyIdent // Update authentication state based on whether identities exist // Once enabled, keep it enabled (one-way toggle) authJustEnabled := iam.updateAuthenticationState(len(identities)) @@ -1010,11 +1012,11 @@ func generatePrincipalArn(identityName string) string { // Handle special cases switch identityName { case AccountAnonymous.Id: - return "arn:aws:iam::user/anonymous" + return "*" // Use universal wildcard for anonymous allowed by bucket policy case AccountAdmin.Id: - return "arn:aws:iam::user/admin" + return fmt.Sprintf("arn:aws:iam::%s:user/admin", defaultAccountID) default: - return fmt.Sprintf("arn:aws:iam::user/%s", identityName) + return fmt.Sprintf("arn:aws:iam::%s:user/%s", defaultAccountID, identityName) } } @@ -1271,6 +1273,7 @@ func (iam *IdentityAccessManagement) authRequestWithAuthType(r *http.Request, ac // the specific IAM action (e.g., self-service vs admin operations). // Returns the authenticated identity and any signature verification error. func (iam *IdentityAccessManagement) AuthSignatureOnly(r *http.Request) (*Identity, s3err.ErrorCode) { + var identity *Identity var s3Err s3err.ErrorCode var authType string @@ -1405,7 +1408,12 @@ func buildPrincipalARN(identity *Identity, r *http.Request) string { return "*" // Anonymous } - // Check if this is the anonymous user identity (authenticated as anonymous) + // Priority 1: Use principal ARN if explicitly set (from STS JWT or IAM user) + if identity.PrincipalArn != "" { + return identity.PrincipalArn + } + + // Priority 2: Check if this is the anonymous user identity (authenticated as anonymous) // S3 policies expect Principal: "*" for anonymous access if identity.Name == s3_constants.AccountAnonymousId || (identity.Account != nil && identity.Account.Id == s3_constants.AccountAnonymousId) { @@ -1414,9 +1422,9 @@ func buildPrincipalARN(identity *Identity, r *http.Request) string { // Build an AWS-compatible principal ARN // Format: arn:aws:iam::account-id:user/user-name - accountId := identity.Account.Id - if accountId == "" { - accountId = "000000000000" // Default account ID + accountID := defaultAccountID // Default account ID + if identity.Account != nil && identity.Account.Id != "" { + accountID = identity.Account.Id } userName := identity.Name @@ -1424,7 +1432,7 @@ func buildPrincipalARN(identity *Identity, r *http.Request) string { userName = "unknown" } - return fmt.Sprintf("arn:aws:iam::%s:user/%s", accountId, userName) + return fmt.Sprintf("arn:aws:iam::%s:user/%s", accountID, userName) } // GetCredentialManager returns the credential manager instance @@ -1434,7 +1442,6 @@ func (iam *IdentityAccessManagement) GetCredentialManager() *credential.Credenti // LoadS3ApiConfigurationFromCredentialManager loads configuration using the credential manager func (iam *IdentityAccessManagement) LoadS3ApiConfigurationFromCredentialManager() error { - glog.V(1).Infof("IAM: reloading configuration from credential manager") glog.V(1).Infof("Loading S3 API configuration from credential manager") s3ApiConfiguration, err := iam.credentialManager.LoadConfiguration(context.Background()) diff --git a/weed/s3api/auth_credentials_test.go b/weed/s3api/auth_credentials_test.go index 224d1956f..92cad7812 100644 --- a/weed/s3api/auth_credentials_test.go +++ b/weed/s3api/auth_credentials_test.go @@ -1,6 +1,7 @@ package s3api import ( + "fmt" "os" "reflect" "sync" @@ -294,7 +295,7 @@ func TestLoadS3ApiConfiguration(t *testing.T) { expectIdent: &Identity{ Name: "notSpecifyAccountId", Account: &AccountAdmin, - PrincipalArn: "arn:aws:iam::user/notSpecifyAccountId", + PrincipalArn: fmt.Sprintf("arn:aws:iam::%s:user/notSpecifyAccountId", defaultAccountID), Actions: []Action{ "Read", "Write", @@ -320,7 +321,7 @@ func TestLoadS3ApiConfiguration(t *testing.T) { expectIdent: &Identity{ Name: "specifiedAccountID", Account: &specifiedAccount, - PrincipalArn: "arn:aws:iam::user/specifiedAccountID", + PrincipalArn: fmt.Sprintf("arn:aws:iam::%s:user/specifiedAccountID", defaultAccountID), Actions: []Action{ "Read", "Write", @@ -338,7 +339,7 @@ func TestLoadS3ApiConfiguration(t *testing.T) { expectIdent: &Identity{ Name: "anonymous", Account: &AccountAnonymous, - PrincipalArn: "arn:aws:iam::user/anonymous", + PrincipalArn: "*", Actions: []Action{ "Read", "Write", diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index d30b5bf17..9e58daf47 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -79,7 +79,7 @@ func streamHashRequestBody(r *http.Request, sizeLimit int64) (string, error) { return "", err } - r.Body = io.NopCloser(&bodyBuffer) + r.Body = io.NopCloser(bytes.NewReader(bodyBuffer.Bytes())) if bodyBuffer.Len() == 0 { return emptySHA256, nil diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 2c4a1a884..526b5162c 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -21,6 +21,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" @@ -832,6 +833,28 @@ func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWr writeSuccessResponseXML(w, r, response) } +// resolveLifecycleDefaultsFromFilerConf returns replication and volumeGrowthCount for use when adding a lifecycle TTL rule. +// S3 does not set DataCenter/Rack/DataNode so placement is not pinned to a specific DC/rack. +// Precedence: parent path rule first, then filer global. If volumeGrowthCount is 0 but replication is set, +// use replication's copy count so the rule is valid (volumeGrowthCount must be divisible by copy count). +func resolveLifecycleDefaultsFromFilerConf(fc *filer.FilerConf, filerConfigReplication, bucketsPath, bucket string) (replication string, volumeGrowthCount uint32, err error) { + bucketPath := fmt.Sprintf("%s/%s/", bucketsPath, bucket) + parentRule := fc.MatchStorageRule(bucketPath) + replication = parentRule.Replication + if replication == "" { + replication = filerConfigReplication + } + volumeGrowthCount = parentRule.VolumeGrowthCount + if volumeGrowthCount == 0 && replication != "" { + var rp *super_block.ReplicaPlacement + rp, err = super_block.NewReplicaPlacementFromString(replication) + if err == nil { + volumeGrowthCount = uint32(rp.GetCopyCount()) + } + } + return +} + // PutBucketLifecycleConfigurationHandler Put Bucket Lifecycle configuration // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) { @@ -857,6 +880,24 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } + + // Resolve replication so lifecycle rules do not create filer.conf entries with empty replication. + var filerConfigReplication string + if filerErr := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + resp, err := client.GetFilerConfiguration(r.Context(), &filer_pb.GetFilerConfigurationRequest{}) + if err != nil { + return err + } + filerConfigReplication = resp.GetReplication() + return nil + }); filerErr != nil { + glog.V(2).Infof("PutBucketLifecycleConfigurationHandler: could not get filer config: %v", filerErr) + } + defaultReplication, defaultVolumeGrowthCount, err := resolveLifecycleDefaultsFromFilerConf(fc, filerConfigReplication, s3a.option.BucketsPath, bucket) + if err != nil { + glog.Warningf("PutBucketLifecycleConfigurationHandler bucket %s: invalid replication %q: %v", bucket, defaultReplication, err) + } + collectionName := s3a.getCollectionName(bucket) collectionTtls := fc.GetCollectionTtls(collectionName) changed := false @@ -881,9 +922,13 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr } locationPrefix := fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, rulePrefix) locConf := &filer_pb.FilerConf_PathConf{ - LocationPrefix: locationPrefix, - Collection: collectionName, - Ttl: fmt.Sprintf("%dd", rule.Expiration.Days), + LocationPrefix: locationPrefix, + Collection: collectionName, + Ttl: fmt.Sprintf("%dd", rule.Expiration.Days), + Replication: defaultReplication, + VolumeGrowthCount: defaultVolumeGrowthCount, + // DataCenter/Rack/DataNode intentionally not set: S3 is not tied to a specific DC/rack, + // requests can hit any filer; setting them would pin placement unnecessarily. } if ttl, ok := collectionTtls[locConf.LocationPrefix]; ok && ttl == locConf.Ttl { continue diff --git a/weed/s3api/s3api_bucket_handlers_lifecycle_test.go b/weed/s3api/s3api_bucket_handlers_lifecycle_test.go new file mode 100644 index 000000000..bc89ee782 --- /dev/null +++ b/weed/s3api/s3api_bucket_handlers_lifecycle_test.go @@ -0,0 +1,80 @@ +package s3api + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/stretchr/testify/assert" +) + +func TestResolveLifecycleDefaultsFromFilerConf(t *testing.T) { + // Precedence: global (lowest), then path rules top-down (parent overrides global), then query (highest). + // So parent path rule has priority over filer global config. + + t.Run("parent_rule_replication_takes_precedence_over_filer_config", func(t *testing.T) { + fc := filer.NewFilerConf() + fc.SetLocationConf(&filer_pb.FilerConf_PathConf{ + LocationPrefix: "/buckets/", + Replication: "001", + }) + repl, vgc, err := resolveLifecycleDefaultsFromFilerConf(fc, "010", "/buckets", "mybucket") + assert.NoError(t, err) + assert.Equal(t, "001", repl, "parent path rule must override filer global config") + assert.Equal(t, uint32(2), vgc, "volumeGrowthCount derived from replication 001 copy count (SameRackCount=1 -> 2 copies)") + }) + + t.Run("falls_back_to_filer_config_when_parent_rule_replication_empty", func(t *testing.T) { + fc := filer.NewFilerConf() + fc.SetLocationConf(&filer_pb.FilerConf_PathConf{ + LocationPrefix: "/buckets/", + Replication: "", // no replication on parent + }) + repl, vgc, err := resolveLifecycleDefaultsFromFilerConf(fc, "010", "/buckets", "mybucket") + assert.NoError(t, err) + assert.Equal(t, "010", repl, "replication should come from filer config when parent rule has none") + assert.Equal(t, uint32(2), vgc, "volumeGrowthCount derived from replication 010 copy count") + }) + + t.Run("parent_rule_empty_when_no_matching_prefix_uses_filer_config", func(t *testing.T) { + fc := filer.NewFilerConf() + // no rules; parent path /buckets/mybucket/ matches nothing + repl, vgc, err := resolveLifecycleDefaultsFromFilerConf(fc, "010", "/buckets", "mybucket") + assert.NoError(t, err) + assert.Equal(t, "010", repl, "when no path rule, use filer config replication") + assert.Equal(t, uint32(2), vgc, "volumeGrowthCount derived from replication 010") + }) + + t.Run("all_empty_when_no_parent_rule_and_no_filer_config", func(t *testing.T) { + fc := filer.NewFilerConf() + repl, vgc, err := resolveLifecycleDefaultsFromFilerConf(fc, "", "/buckets", "mybucket") + assert.NoError(t, err) + assert.Empty(t, repl) + assert.Equal(t, uint32(0), vgc) + }) + + t.Run("parent_rule_volume_growth_count_used_when_set", func(t *testing.T) { + fc := filer.NewFilerConf() + fc.SetLocationConf(&filer_pb.FilerConf_PathConf{ + LocationPrefix: "/buckets/", + Replication: "010", + VolumeGrowthCount: 4, + }) + repl, vgc, err := resolveLifecycleDefaultsFromFilerConf(fc, "010", "/buckets", "mybucket") + assert.NoError(t, err) + assert.Equal(t, "010", repl) + assert.Equal(t, uint32(4), vgc, "parent VolumeGrowthCount must be used when set") + }) + + t.Run("invalid_replication_returns_error", func(t *testing.T) { + fc := filer.NewFilerConf() + fc.SetLocationConf(&filer_pb.FilerConf_PathConf{ + LocationPrefix: "/buckets/", + Replication: "0x1", // invalid: non-digit + }) + repl, vgc, err := resolveLifecycleDefaultsFromFilerConf(fc, "", "/buckets", "mybucket") + assert.Error(t, err) + assert.Equal(t, "0x1", repl, "replication string is still returned") + assert.Equal(t, uint32(0), vgc, "volumeGrowthCount remains 0 when parse fails") + }) +} diff --git a/weed/s3api/s3api_bucket_policy_arn_test.go b/weed/s3api/s3api_bucket_policy_arn_test.go index 3f9b890e4..bb5284ef3 100644 --- a/weed/s3api/s3api_bucket_policy_arn_test.go +++ b/weed/s3api/s3api_bucket_policy_arn_test.go @@ -1,6 +1,7 @@ package s3api import ( + "fmt" "testing" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" @@ -62,6 +63,14 @@ func TestBuildPrincipalARN(t *testing.T) { identity: nil, expected: "*", }, + { + name: "explicit principal ARN", + identity: &Identity{ + Name: "test-user", + PrincipalArn: "arn:aws:iam::123456789012:role/MyRole", + }, + expected: "arn:aws:iam::123456789012:role/MyRole", + }, { name: "anonymous user by name", identity: &Identity{ @@ -100,7 +109,7 @@ func TestBuildPrincipalARN(t *testing.T) { Id: "", }, }, - expected: "arn:aws:iam::000000000000:user/test-user", + expected: fmt.Sprintf("arn:aws:iam::%s:user/test-user", defaultAccountID), }, { name: "identity without name", diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index faa0fb8d5..4a3fa4554 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -1,9 +1,11 @@ package s3api import ( + "bytes" "context" "encoding/json" "fmt" + "io" "net" "net/http" "os" @@ -424,6 +426,93 @@ func (s3a *S3ApiServer) handleCORSOriginValidation(w http.ResponseWriter, r *htt return true } +// UnifiedPostHandler handles authenticated POST requests to the root path +// It inspects the Action parameter to dispatch to either STS or IAM handlers +func (s3a *S3ApiServer) UnifiedPostHandler(w http.ResponseWriter, r *http.Request) { + // 1. Authenticate (preserves body) + identity, errCode := s3a.iam.AuthSignatureOnly(r) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + + // 2. Parse Form to get Action + // Save the body first so we can restore it for STS handler signature verification + var bodyBytes []byte + if r.Body != nil { + // Limit body size to prevent DoS attacks + r.Body = http.MaxBytesReader(w, r.Body, iamRequestBodyLimit) + var err error + bodyBytes, err = io.ReadAll(r.Body) + if err != nil { + glog.Errorf("failed to read request body: %v", err) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) + return + } + r.Body.Close() + // Restore body for ParseForm + r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) + } + + if err := r.ParseForm(); err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) + return + } + + // Restore body again for downstream handlers (STS needs it for signature verification) + if bodyBytes != nil { + r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) + } + + // 3. Dispatch + action := r.Form.Get("Action") + if strings.HasPrefix(action, "AssumeRole") { + // STS + if s3a.stsHandlers == nil { + s3err.WriteErrorResponse(w, r, s3err.ErrServiceUnavailable) + return + } + s3a.stsHandlers.HandleSTSRequest(w, r) + } else { + // IAM + // IAM API requests must be authenticated - reject nil identity + if identity == nil { + s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied) + return + } + + // Store identity in context + // Always set identity in context when non-nil to ensure downstream handlers have access + ctx := r.Context() + if identity.Name != "" { + ctx = SetIdentityNameInContext(ctx, identity.Name) + } + ctx = SetIdentityInContext(ctx, identity) + r = r.WithContext(ctx) + + targetUserName := r.Form.Get("UserName") + + // Check permissions based on action type + isSelfServiceAction := iamRequiresAdminForOthers(action) + isActingOnSelf := targetUserName == "" || targetUserName == identity.Name + + // Permission check is required for all actions except for self-service actions + // performed on the user's own identity. + if !(isSelfServiceAction && isActingOnSelf) { + if !identity.isAdmin() { + if s3a.iam.VerifyActionPermission(r, identity, Action("iam:"+action), "arn:aws:iam:::*", "") != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied) + return + } + } + } + + // Call Limit middleware + DoActions + handler, _ := s3a.cb.Limit(s3a.embeddedIam.DoActions, ACTION_WRITE) + handler.ServeHTTP(w, r) + } +} + func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // API Router apiRouter := router.PathPrefix("/").Subrouter() @@ -685,38 +774,31 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // POST / (without specific query parameters) // Uses AuthIam for granular permission checking if s3a.embeddedIam != nil { - // 2. Authenticated IAM requests + + // 2. Authenticated IAM/STS Post requests // Only match if the request appears to be authenticated (AWS Signature) - // AND is not an STS request (which should be handled by STS handlers) + // We use a UnifiedPostHandler to dispatch based on Action (STS vs IAM) iamMatcher := func(r *http.Request, rm *mux.RouteMatch) bool { if getRequestAuthType(r) == authTypeAnonymous { return false } - // IMPORTANT: Do NOT call r.ParseForm() here! - // ParseForm() consumes the request body, which breaks AWS Signature V4 verification - // for IAM requests. The signature must be calculated on the original body. - // Instead, check only the query string for the Action parameter. - - // For IAM requests, the Action is typically in the POST body, not query string - // So we match all authenticated POST / requests and let AuthIam validate them - // This is safe because: - // 1. STS actions are excluded (handled by separate STS routes) - // 2. S3 operations don't POST to / (they use / or //) - // 3. IAM operations all POST to / + // IMPORTANT: We do NOT parse the body here. + // UnifiedPostHandler will handle authentication and body parsing. + // We only filter out requests that are explicitly targeted at STS via Query params + // to avoid double-handling, although UnifiedPostHandler would handle them correctly anyway. - // Only exclude STS actions which might be in query string + // Action in Query String is handled by explicit STS routes above action := r.URL.Query().Get("Action") if action == "AssumeRole" || action == "AssumeRoleWithWebIdentity" || action == "AssumeRoleWithLDAPIdentity" { return false } - // Match all other authenticated POST / requests (IAM operations) return true } apiRouter.Methods(http.MethodPost).Path("/").MatcherFunc(iamMatcher). - HandlerFunc(track(s3a.embeddedIam.AuthIam(s3a.cb.Limit(s3a.embeddedIam.DoActions, ACTION_WRITE)), "IAM")) + HandlerFunc(track(s3a.UnifiedPostHandler, "IAM-Unified")) glog.V(1).Infof("Embedded IAM API enabled on S3 port") } diff --git a/weed/s3api/s3api_sts.go b/weed/s3api/s3api_sts.go index 943e67929..0a5c565c7 100644 --- a/weed/s3api/s3api_sts.go +++ b/weed/s3api/s3api_sts.go @@ -5,8 +5,6 @@ package s3api // AWS SDKs to obtain temporary credentials using OIDC/JWT tokens. import ( - "crypto/rand" - "encoding/base64" "encoding/xml" "errors" "fmt" @@ -46,9 +44,6 @@ const ( const ( minDurationSeconds = int64(900) // 15 minutes maxDurationSeconds = int64(43200) // 12 hours - - // Default account ID for federated users - defaultAccountId = "111122223333" ) // parseDurationSeconds parses and validates the DurationSeconds parameter @@ -90,6 +85,13 @@ func NewSTSHandlers(stsService *sts.STSService, iam *IdentityAccessManagement) * } } +func (h *STSHandlers) getAccountID() string { + if h.stsService != nil && h.stsService.Config != nil && h.stsService.Config.AccountId != "" { + return h.stsService.Config.AccountId + } + return defaultAccountID +} + // HandleSTSRequest is the main entry point for STS requests // It routes requests based on the Action parameter func (h *STSHandlers) HandleSTSRequest(w http.ResponseWriter, r *http.Request) { @@ -289,7 +291,7 @@ func (h *STSHandlers) handleAssumeRole(w http.ResponseWriter, r *http.Request) { } // Generate common STS components - stsCreds, assumedUser, err := h.prepareSTSCredentials(roleArn, roleSessionName, identity.PrincipalArn, durationSeconds, nil) + stsCreds, assumedUser, err := h.prepareSTSCredentials(roleArn, roleSessionName, durationSeconds, nil) if err != nil { h.writeSTSErrorResponse(w, r, STSErrInternalError, err) return @@ -398,14 +400,7 @@ func (h *STSHandlers) handleAssumeRoleWithLDAPIdentity(w http.ResponseWriter, r glog.V(2).Infof("AssumeRoleWithLDAPIdentity: user %s authenticated successfully, groups=%v", ldapUsername, identity.Groups) - // Verify that the identity is allowed to assume the role - // We create a temporary identity to represent the LDAP user for permission checking - // The checking logic will verify if the role's trust policy allows this principal - // Use configured account ID or default to "111122223333" for federated users - accountId := defaultAccountId - if h.stsService != nil && h.stsService.Config != nil && h.stsService.Config.AccountId != "" { - accountId = h.stsService.Config.AccountId - } + accountID := h.getAccountID() ldapUserIdentity := &Identity{ Name: identity.UserID, @@ -414,7 +409,7 @@ func (h *STSHandlers) handleAssumeRoleWithLDAPIdentity(w http.ResponseWriter, r EmailAddress: identity.Email, Id: identity.UserID, }, - PrincipalArn: fmt.Sprintf("arn:aws:iam::%s:user/%s", accountId, identity.UserID), + PrincipalArn: fmt.Sprintf("arn:aws:iam::%s:user/%s", accountID, identity.UserID), } // Verify that the identity is allowed to assume the role by checking the Trust Policy @@ -430,7 +425,7 @@ func (h *STSHandlers) handleAssumeRoleWithLDAPIdentity(w http.ResponseWriter, r claims.WithIdentityProvider("ldap", identity.UserID, identity.Provider) } - stsCreds, assumedUser, err := h.prepareSTSCredentials(roleArn, roleSessionName, ldapUserIdentity.PrincipalArn, durationSeconds, modifyClaims) + stsCreds, assumedUser, err := h.prepareSTSCredentials(roleArn, roleSessionName, durationSeconds, modifyClaims) if err != nil { h.writeSTSErrorResponse(w, r, STSErrInternalError, err) return @@ -449,7 +444,7 @@ func (h *STSHandlers) handleAssumeRoleWithLDAPIdentity(w http.ResponseWriter, r } // prepareSTSCredentials extracts common shared logic for credential generation -func (h *STSHandlers) prepareSTSCredentials(roleArn, roleSessionName, principalArn string, +func (h *STSHandlers) prepareSTSCredentials(roleArn, roleSessionName string, durationSeconds *int64, modifyClaims func(*sts.STSSessionClaims)) (STSCredentials, *AssumedRoleUser, error) { // Calculate duration @@ -472,10 +467,17 @@ func (h *STSHandlers) prepareSTSCredentials(roleArn, roleSessionName, principalA roleName = roleArn // Fallback to full ARN if extraction fails } + accountID := h.getAccountID() + + // Construct AssumedRoleUser ARN - this will be used as the principal for the vended token + assumedRoleArn := fmt.Sprintf("arn:aws:sts::%s:assumed-role/%s/%s", accountID, roleName, roleSessionName) + // Create session claims with role information + // SECURITY: Use the assumedRoleArn as the principal in the token. + // This ensures that subsequent requests using this token are correctly identified as the assumed role. claims := sts.NewSTSSessionClaims(sessionId, h.stsService.Config.Issuer, expiration). WithSessionName(roleSessionName). - WithRoleInfo(roleArn, fmt.Sprintf("%s:%s", roleName, roleSessionName), principalArn) + WithRoleInfo(roleArn, fmt.Sprintf("%s:%s", roleName, roleSessionName), assumedRoleArn) // Apply custom claims if provided (e.g., LDAP identity) if modifyClaims != nil { @@ -488,30 +490,14 @@ func (h *STSHandlers) prepareSTSCredentials(roleArn, roleSessionName, principalA return STSCredentials{}, nil, fmt.Errorf("failed to generate session token: %w", err) } - // Generate temporary credentials (cryptographically secure) - // AccessKeyId: ASIA + 16 chars hex - // SecretAccessKey: 40 chars base64 - randBytes := make([]byte, 30) // Sufficient for both - if _, err := rand.Read(randBytes); err != nil { - return STSCredentials{}, nil, fmt.Errorf("failed to generate random bytes: %w", err) - } - - // Generate AccessKeyId (ASIA + 16 upper-hex chars) - // We use 8 bytes (16 hex chars) - accessKeyId := "ASIA" + fmt.Sprintf("%X", randBytes[:8]) - - // Generate SecretAccessKey: 30 random bytes, base64-encoded to a 40-character string - secretBytes := make([]byte, 30) - if _, err := rand.Read(secretBytes); err != nil { - return STSCredentials{}, nil, fmt.Errorf("failed to generate secret bytes: %w", err) - } - secretAccessKey := base64.StdEncoding.EncodeToString(secretBytes) - - // Get account ID from STS config or use default - accountId := defaultAccountId - if h.stsService != nil && h.stsService.Config != nil && h.stsService.Config.AccountId != "" { - accountId = h.stsService.Config.AccountId + // Generate temporary credentials (deterministic based on sessionId) + stsCredGen := sts.NewCredentialGenerator() + stsCredsDet, err := stsCredGen.GenerateTemporaryCredentials(sessionId, expiration) + if err != nil { + return STSCredentials{}, nil, fmt.Errorf("failed to generate temporary credentials: %w", err) } + accessKeyId := stsCredsDet.AccessKeyId + secretAccessKey := stsCredsDet.SecretAccessKey stsCreds := STSCredentials{ AccessKeyId: accessKeyId, @@ -522,7 +508,7 @@ func (h *STSHandlers) prepareSTSCredentials(roleArn, roleSessionName, principalA assumedUser := &AssumedRoleUser{ AssumedRoleId: fmt.Sprintf("%s:%s", roleName, roleSessionName), - Arn: fmt.Sprintf("arn:aws:sts::%s:assumed-role/%s/%s", accountId, roleName, roleSessionName), + Arn: assumedRoleArn, } return stsCreds, assumedUser, nil diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index b782e9356..215d90262 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -116,6 +116,7 @@ const ( ErrTooManyRequest ErrRequestBytesExceed + ErrServiceUnavailable OwnershipControlsNotFoundError ErrNoSuchTagSet @@ -512,6 +513,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Simultaneous request bytes exceed limitations", HTTPStatusCode: http.StatusServiceUnavailable, }, + ErrServiceUnavailable: { + Code: "ServiceUnavailable", + Description: "Service Unavailable", + HTTPStatusCode: http.StatusServiceUnavailable, + }, OwnershipControlsNotFoundError: { Code: "OwnershipControlsNotFoundError", diff --git a/weed/s3api/sts_params_test.go b/weed/s3api/sts_params_test.go new file mode 100644 index 000000000..d7e04fa58 --- /dev/null +++ b/weed/s3api/sts_params_test.go @@ -0,0 +1,200 @@ +package s3api + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/iam/sts" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/stretchr/testify/assert" +) + +// Minimal mock implementation of AuthenticateJWT needed for testing +type mockIAMIntegration struct{} + +func (m *mockIAMIntegration) AuthenticateJWT(ctx context.Context, r *http.Request) (*IAMIdentity, s3err.ErrorCode) { + return &IAMIdentity{ + Name: "test-user", + Account: &Account{ + Id: "test-account", + DisplayName: "test-account", + EmailAddress: "test@example.com", + }, + Principal: "arn:aws:iam::test-account:user/test-user", + SessionToken: "mock-session-token", + }, s3err.ErrNone +} +func (m *mockIAMIntegration) AuthorizeAction(ctx context.Context, identity *IAMIdentity, action Action, bucket, object string, r *http.Request) s3err.ErrorCode { + return s3err.ErrNone +} +func (m *mockIAMIntegration) ValidateTrustPolicyForPrincipal(ctx context.Context, roleArn, principalArn string) error { + return nil +} +func (m *mockIAMIntegration) ValidateSessionToken(ctx context.Context, token string) (*sts.SessionInfo, error) { + return nil, nil +} + +func TestSTSAssumeRolePostBody(t *testing.T) { + // Setup S3ApiServer with IAM enabled + option := &S3ApiServerOption{ + DomainName: "localhost", + EnableIam: true, + Filers: []pb.ServerAddress{"localhost:8888"}, + } + + // Create IAM instance that we can control + // We need to bypass the file/store loading logic in NewIdentityAccessManagement + // So we construct it manually similarly to how it's done for tests + iam := &IdentityAccessManagement{ + identities: []*Identity{{Name: "test-user"}}, + isAuthEnabled: true, + accessKeyIdent: make(map[string]*Identity), + nameToIdentity: make(map[string]*Identity), + iamIntegration: &mockIAMIntegration{}, + } + + // Pre-populate an identity for testing + ident := &Identity{ + Name: "test-user", + Credentials: []*Credential{ + {AccessKey: "test", SecretKey: "test", Status: "Active"}, + }, + Actions: nil, // Admin + IsStatic: true, + } + iam.identities[0] = ident + iam.accessKeyIdent["test"] = ident + iam.nameToIdentity["test-user"] = ident + + s3a := &S3ApiServer{ + option: option, + iam: iam, + embeddedIam: &EmbeddedIamApi{iam: iam, getS3ApiConfigurationFunc: func(cfg *iam_pb.S3ApiConfiguration) error { return nil }}, + stsHandlers: NewSTSHandlers(nil, iam), // STS service nil -> will return STSErrSTSNotReady (503) + credentialManager: nil, // Not needed for this test as we pre-populated IAM + cb: &CircuitBreaker{ + counters: make(map[string]*int64), + limitations: make(map[string]int64), + }, + } + s3a.cb.s3a = s3a + s3a.inFlightDataLimitCond = sync.NewCond(&sync.Mutex{}) + + // Create router and register routes + router := mux.NewRouter() + s3a.registerRouter(router) + + // Test Case 1: STS Action in Query String (Should work - routed to STS) + t.Run("ActionInQuery", func(t *testing.T) { + req := httptest.NewRequest("POST", "/?Action=AssumeRole", nil) + // We aren't signing requests, so we expect STSErrAccessDenied (403) from STS handler + // due to invalid signature, OR STSErrSTSNotReady (503) if it gets past auth. + // The key is it should NOT be 501 Not Implemented (which comes from IAM handler) + + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + + // If routed to STS, we expect 400 (Bad Request) - MissingParameter + // because we didn't provide RoleArn/RoleSessionName etc. + // Or 503 if it checks STS service readiness first. + + // Let's see what we get. The STS handler checks parameters first. + // "RoleArn is required" -> 400 Bad Request + + assert.NotEqual(t, http.StatusNotImplemented, rr.Code, "Should not return 501 (IAM handler)") + assert.Equal(t, http.StatusBadRequest, rr.Code, "Should return 400 (STS handler) for missing params") + }) + + // Test Case 2: STS Action in Body (Should FAIL current implementation - routed to IAM) + t.Run("ActionInBody", func(t *testing.T) { + form := url.Values{} + form.Add("Action", "AssumeRole") + form.Add("RoleArn", "arn:aws:iam::123:role/test") + form.Add("RoleSessionName", "session") + + req := httptest.NewRequest("POST", "/", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + // We need an Authorization header to trigger the IAM matcher + // The matcher checks: getRequestAuthType(r) != authTypeAnonymous + // So we provide a dummy auth header + + req.Header.Set("Authorization", "Bearer test-token") + + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + + // CURRENT BEHAVIOR: + // The Router does not match "/" for STS because Action is not in query. + // The Router matches "/" for IAM because it has Authorization header. + // IAM handler (AuthIam) calls DoActions. + // DoActions switches on "AssumeRole" -> default -> Not Implemented (501). + + // DESIRED BEHAVIOR (after fix): + // Should be routed to UnifiedPostHandler (or similar), detected as STS action, + // and routed to STS handler. + // STS handler should return 403 Forbidden (Access Denied) or 400 Bad Request + // because of signature mismatch (since we provided dummy auth). + // It should NOT be 501. + + // For verification of fix, we assert it IS 503 (STS Service Not Initialized). + // This confirms it was routed to STS handler. + if rr.Code != http.StatusServiceUnavailable { + t.Logf("Unexpected status code: %d", rr.Code) + t.Logf("Response body: %s", rr.Body.String()) + } + // Confirm it routed to STS + assert.Equal(t, http.StatusServiceUnavailable, rr.Code, "Fixed behavior: Should return 503 from STS handler (service not ready)") + }) + + // Test Case 3: STS Action in Body with SigV4-style Authorization (Real-world scenario) + // This test validates that requests with AWS SigV4 Authorization headers and POST body + // parameters are correctly routed to the STS handler. + t.Run("ActionInBodyWithSigV4Style", func(t *testing.T) { + form := url.Values{} + form.Add("Action", "AssumeRole") + form.Add("RoleArn", "arn:aws:iam::123:role/test") + form.Add("RoleSessionName", "session") + + bodyContent := form.Encode() + req := httptest.NewRequest("POST", "/", strings.NewReader(bodyContent)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + // Set AWS SigV4-style Authorization header + // This simulates a real SigV4-signed request without needing perfect signature + // The key is to validate that UnifiedPostHandler correctly routes based on Action + req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=test/20260212/us-east-1/sts/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=dummy") + req.Header.Set("x-amz-date", "20260212T000000Z") + + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + + // With SigV4-style Authorization header, the request should: + // 1. Be recognized as authenticated (not anonymous) + // 2. Be routed to UnifiedPostHandler + // 3. UnifiedPostHandler should parse Action=AssumeRole from body + // 4. Route to STS handler (which returns 503 because stsService is nil) + // OR return 403 if signature validation fails (which is acceptable) + + // The key validation is that it should NOT return 501 (IAM handler's "Not Implemented") + // This confirms the routing fix works for SigV4-signed requests with POST body params + + if rr.Code != http.StatusServiceUnavailable && rr.Code != http.StatusForbidden { + t.Logf("Unexpected status code: %d", rr.Code) + t.Logf("Response body: %s", rr.Body.String()) + } + + // Accept either 503 (routed to STS, service unavailable) or 403 (signature failed) + // Both indicate correct routing to STS handler, not IAM handler + assert.NotEqual(t, http.StatusNotImplemented, rr.Code, "Should not return 501 (IAM handler)") + assert.Contains(t, []int{http.StatusServiceUnavailable, http.StatusForbidden}, rr.Code, + "Should return 503 (STS unavailable) or 403 (auth failed), confirming STS routing") + }) +} diff --git a/weed/util/network.go b/weed/util/network.go index f7dbeebb7..62716d869 100644 --- a/weed/util/network.go +++ b/weed/util/network.go @@ -58,11 +58,14 @@ func selectIpV4(netInterfaces []net.Interface, isIpV4 bool) string { } func JoinHostPort(host string, port int) string { - portStr := strconv.Itoa(port) + return JoinHostPortStr(host, strconv.Itoa(port)) +} + +func JoinHostPortStr(host string, port string) string { if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { - return host + ":" + portStr + return host + ":" + port } - return net.JoinHostPort(host, portStr) + return net.JoinHostPort(host, port) } // GetVolumeServerId returns the volume server ID.