From beeb375a88e7082d39520b5a7bedabe416208e0a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Feb 2026 00:40:56 -0800 Subject: [PATCH] Add volume server integration test suite and CI workflow (#8322) * docs(volume_server): add integration test development plan * test(volume_server): add integration harness and profile matrix * test(volume_server/http): add admin and options integration coverage * test(volume_server/grpc): add state and status integration coverage * test(volume_server): auto-build weed binary and harden cluster startup * test(volume_server/http): add upload read range head delete coverage * test(volume_server/grpc): expand admin lifecycle and state coverage * docs(volume_server): update progress tracker for implemented tests * test(volume_server/http): cover if-none-match and invalid-range branches * test(volume_server/grpc): add batch delete integration coverage * docs(volume_server): log latest HTTP and gRPC test coverage * ci(volume_server): run volume server integration tests in github actions * test(volume_server/grpc): add needle status configure ping and leave coverage * docs(volume_server): record additional grpc coverage progress * test(volume_server/grpc): add vacuum integration coverage * docs(volume_server): record vacuum test coverage progress * test(volume_server/grpc): add read and write needle blob error-path coverage * docs(volume_server): record data rw grpc coverage progress * test(volume_server/http): add jwt auth integration coverage * test(volume_server/grpc): add sync copy and stream error-path coverage * docs(volume_server): record jwt and sync/copy test coverage * test(volume_server/grpc): add scrub and query integration coverage * test(volume_server/grpc): add volume tail sender and receiver coverage * docs(volume_server): record scrub query and tail test progress * test(volume_server/grpc): add readonly writable and collection lifecycle coverage * test(volume_server/http): add public-port cors and method parity coverage * test(volume_server/grpc): add blob meta and read-all success path coverage * test(volume_server/grpc): expand scrub and query variation coverage * test(volume_server/grpc): add tiering and remote fetch error-path coverage * test(volume_server/http): add unchanged write and delete edge-case coverage * test(volume_server/grpc): add ping unknown and unreachable target coverage * test(volume_server/grpc): add volume delete only-empty variation coverage * test(volume_server/http): add jwt fid-mismatch auth coverage * test(volume_server/grpc): add scrub ec auto-select empty coverage * test(volume_server/grpc): stabilize ping timestamp assertion * docs(volume_server): update integration coverage progress log * test(volume_server/grpc): add tier remote backend and config variation coverage * docs(volume_server): record tier remote variation progress * test(volume_server/grpc): add incremental copy and receive-file protocol coverage * test(volume_server/http): add read path shape and if-modified-since coverage * test(volume_server/grpc): add copy-file compaction and receive-file success coverage * test(volume_server/http): add passthrough headers and static asset coverage * test(volume_server/grpc): add ping filer unreachable coverage * docs(volume_server): record copy receive and http variant progress * test(volume_server/grpc): add erasure coding maintenance and missing-path coverage * docs(volume_server): record initial erasure coding rpc coverage * test(volume_server/http): add multi-range multipart response coverage * docs(volume_server): record multi-range http coverage progress * test(volume_server/grpc): add query empty-stripe no-match coverage * docs(volume_server): record query no-match stream behavior coverage * test(volume_server/http): add upload throttling timeout and replicate bypass coverage * docs(volume_server): record upload throttling coverage progress * test(volume_server/http): add download throttling timeout coverage * docs(volume_server): record download throttling coverage progress * test(volume_server/http): add jwt wrong-cookie fid mismatch coverage * docs(volume_server): record jwt wrong-cookie mismatch coverage * test(volume_server/http): add jwt expired-token rejection coverage * docs(volume_server): record jwt expired-token coverage * test(volume_server/http): add jwt query and cookie transport coverage * docs(volume_server): record jwt token transport coverage * test(volume_server/http): add jwt token-source precedence coverage * docs(volume_server): record jwt token-source precedence coverage * test(volume_server/http): add jwt header-over-cookie precedence coverage * docs(volume_server): record jwt header cookie precedence coverage * test(volume_server/http): add jwt query-over-cookie precedence coverage * docs(volume_server): record jwt query cookie precedence coverage * test(volume_server/grpc): add setstate version mismatch and nil-state coverage * docs(volume_server): record setstate validation coverage * test(volume_server/grpc): add readonly persist-true lifecycle coverage * docs(volume_server): record readonly persist variation coverage * test(volume_server/http): add options origin cors header coverage * docs(volume_server): record options origin cors coverage * test(volume_server/http): add trace unsupported-method parity coverage * docs(volume_server): record trace method parity coverage * test(volume_server/grpc): add batch delete cookie-check variation coverage * docs(volume_server): record batch delete cookie-check coverage * test(volume_server/grpc): add admin lifecycle missing and maintenance variants * docs(volume_server): record admin lifecycle edge-case coverage * test(volume_server/grpc): add mixed batch delete status matrix coverage * docs(volume_server): record mixed batch delete matrix coverage * test(volume_server/http): add jwt-profile ui access gating coverage * docs(volume_server): record jwt ui-gating http coverage * test(volume_server/http): add propfind unsupported-method parity coverage * docs(volume_server): record propfind method parity coverage * test(volume_server/grpc): add volume configure success and rollback-path coverage * docs(volume_server): record volume configure branch coverage * test(volume_server/grpc): add volume needle status missing-path coverage * docs(volume_server): record volume needle status error-path coverage * test(volume_server/http): add readDeleted query behavior coverage * docs(volume_server): record readDeleted http behavior coverage * test(volume_server/http): add delete ts override parity coverage * docs(volume_server): record delete ts parity coverage * test(volume_server/grpc): add invalid blob/meta offset coverage * docs(volume_server): record invalid blob/meta offset coverage * test(volume_server/grpc): add read-all mixed volume abort coverage * docs(volume_server): record read-all mixed-volume abort coverage * test(volume_server/http): assert head response body parity * docs(volume_server): record head body parity assertion * test(volume_server/grpc): assert status state and memory payload completeness * docs(volume_server): record volume server status payload coverage * test(volume_server/grpc): add batch delete chunk-manifest rejection coverage * docs(volume_server): record batch delete chunk-manifest coverage * test(volume_server/grpc): add query cookie-mismatch eof parity coverage * docs(volume_server): record query cookie-mismatch parity coverage * test(volume_server/grpc): add ping master success target coverage * docs(volume_server): record ping master success coverage * test(volume_server/http): add head if-none-match conditional parity * docs(volume_server): record head if-none-match parity coverage * test(volume_server/http): add head if-modified-since parity coverage * docs(volume_server): record head if-modified-since parity coverage * test(volume_server/http): add connect unsupported-method parity coverage * docs(volume_server): record connect method parity coverage * test(volume_server/http): assert options allow-headers cors parity * docs(volume_server): record options allow-headers coverage * test(volume_server/framework): add dual volume cluster integration harness * test(volume_server/http): add missing-local read mode proxy redirect local coverage * docs(volume_server): record read mode missing-local matrix coverage * test(volume_server/http): add download over-limit replica proxy fallback coverage * docs(volume_server): record download replica fallback coverage * test(volume_server/http): add missing-local readDeleted proxy redirect parity coverage * docs(volume_server): record missing-local readDeleted mode coverage * test(volume_server/framework): add single-volume cluster with filer harness * test(volume_server/grpc): add ping filer success target coverage * docs(volume_server): record ping filer success coverage * test(volume_server/http): add proxied-loop guard download timeout coverage * docs(volume_server): record proxied-loop download coverage * test(volume_server/http): add disabled upload and download limit coverage * docs(volume_server): record disabled throttling path coverage * test(volume_server/grpc): add idempotent volume server leave coverage * docs(volume_server): record leave idempotence coverage * test(volume_server/http): add redirect collection query preservation coverage * docs(volume_server): record redirect collection query coverage * test(volume_server/http): assert admin server headers on status and health * docs(volume_server): record admin server header coverage * test(volume_server/http): assert healthz request-id echo parity * docs(volume_server): record healthz request-id parity coverage * test(volume_server/http): add over-limit invalid-vid download branch coverage * docs(volume_server): record over-limit invalid-vid branch coverage * test(volume_server/http): add public-port static asset coverage * docs(volume_server): record public static endpoint coverage * test(volume_server/http): add public head method parity coverage * docs(volume_server): record public head parity coverage * test(volume_server/http): add throttling wait-then-proceed path coverage * docs(volume_server): record throttling wait-then-proceed coverage * test(volume_server/http): add read cookie-mismatch not-found coverage * docs(volume_server): record read cookie-mismatch coverage * test(volume_server/http): add throttling timeout-recovery coverage * docs(volume_server): record throttling timeout-recovery coverage * test(volume_server/grpc): add ec generate mount info unmount lifecycle coverage * docs(volume_server): record ec positive lifecycle coverage * test(volume_server/grpc): add ec shard read and blob delete lifecycle coverage * docs(volume_server): record ec shard read/blob delete lifecycle coverage * test(volume_server/grpc): add ec rebuild and to-volume error branch coverage * docs(volume_server): record ec rebuild and to-volume branch coverage * test(volume_server/grpc): add ec shards-to-volume success roundtrip coverage * docs(volume_server): record ec shards-to-volume success coverage * test(volume_server/grpc): add ec receive and copy-file missing-source coverage * docs(volume_server): record ec receive and copy-file coverage * test(volume_server/grpc): add ec last-shard delete cleanup coverage * docs(volume_server): record ec last-shard delete cleanup coverage * test(volume_server/grpc): add volume copy success path coverage * docs(volume_server): record volume copy success coverage * test(volume_server/grpc): add volume copy overwrite-destination coverage * docs(volume_server): record volume copy overwrite coverage * test(volume_server/http): add write error-path variant coverage * docs(volume_server): record http write error-path coverage * test(volume_server/http): add conditional header precedence coverage * docs(volume_server): record conditional header precedence coverage * test(volume_server/http): add oversized combined range guard coverage * docs(volume_server): record oversized range guard coverage * test(volume_server/http): add image resize and crop read coverage * docs(volume_server): record image transform coverage * test(volume_server/http): add chunk-manifest expansion and bypass coverage * docs(volume_server): record chunk-manifest read coverage * test(volume_server/http): add compressed read encoding matrix coverage * docs(volume_server): record compressed read matrix coverage * test(volume_server/grpc): add tail receiver source replication coverage * docs(volume_server): record tail receiver replication coverage * test(volume_server/grpc): add tail sender large-needle chunking coverage * docs(volume_server): record tail sender chunking coverage * test(volume_server/grpc): add ec-backed volume needle status coverage * docs(volume_server): record ec-backed needle status coverage * test(volume_server/grpc): add ec shard copy from peer success coverage * docs(volume_server): record ec shard copy success coverage * test(volume_server/http): add chunk-manifest delete child cleanup coverage * docs(volume_server): record chunk-manifest delete cleanup coverage * test(volume_server/http): add chunk-manifest delete failure-path coverage * docs(volume_server): record chunk-manifest delete failure coverage * test(volume_server/grpc): add ec shard copy source-unavailable coverage * docs(volume_server): record ec shard copy source-unavailable coverage * parallel --- .../volume-server-integration-tests.yml | 122 ++ test/volume_server/DEV_PLAN.md | 1129 +++++++++++++++++ test/volume_server/Makefile | 7 + test/volume_server/README.md | 27 + test/volume_server/framework/cluster.go | 442 +++++++ test/volume_server/framework/cluster_dual.go | 293 +++++ .../framework/cluster_with_filer.go | 91 ++ .../framework/fault_injection.go | 8 + test/volume_server/framework/grpc_client.go | 28 + test/volume_server/framework/http_client.go | 34 + .../volume_server/framework/volume_fixture.go | 56 + test/volume_server/grpc/admin_extra_test.go | 445 +++++++ .../grpc/admin_lifecycle_test.go | 215 ++++ .../grpc/admin_readonly_collection_test.go | 177 +++ test/volume_server/grpc/batch_delete_test.go | 264 ++++ .../grpc/copy_receive_variants_test.go | 431 +++++++ test/volume_server/grpc/copy_sync_test.go | 284 +++++ test/volume_server/grpc/data_rw_test.go | 146 +++ .../grpc/data_stream_success_test.go | 273 ++++ .../volume_server/grpc/erasure_coding_test.go | 777 ++++++++++++ test/volume_server/grpc/health_state_test.go | 139 ++ test/volume_server/grpc/scrub_query_test.go | 385 ++++++ test/volume_server/grpc/tail_test.go | 206 +++ .../volume_server/grpc/tiering_remote_test.go | 236 ++++ test/volume_server/grpc/vacuum_test.go | 87 ++ test/volume_server/http/admin_test.go | 174 +++ test/volume_server/http/auth_test.go | 419 ++++++ .../volume_server/http/chunk_manifest_test.go | 232 ++++ .../http/compressed_read_test.go | 97 ++ .../volume_server/http/headers_static_test.go | 102 ++ .../http/image_transform_test.go | 92 ++ .../http/public_cors_methods_test.go | 287 +++++ .../volume_server/http/range_variants_test.go | 82 ++ test/volume_server/http/read_deleted_test.go | 54 + .../http/read_mode_proxy_redirect_test.go | 319 +++++ .../http/read_path_variants_test.go | 191 +++ .../http/read_write_delete_test.go | 123 ++ test/volume_server/http/throttling_test.go | 730 +++++++++++ .../http/write_delete_variants_test.go | 118 ++ .../http/write_error_variants_test.go | 74 ++ test/volume_server/matrix/config_profiles.go | 63 + 41 files changed, 9459 insertions(+) create mode 100644 .github/workflows/volume-server-integration-tests.yml create mode 100644 test/volume_server/DEV_PLAN.md create mode 100644 test/volume_server/Makefile create mode 100644 test/volume_server/README.md create mode 100644 test/volume_server/framework/cluster.go create mode 100644 test/volume_server/framework/cluster_dual.go create mode 100644 test/volume_server/framework/cluster_with_filer.go create mode 100644 test/volume_server/framework/fault_injection.go create mode 100644 test/volume_server/framework/grpc_client.go create mode 100644 test/volume_server/framework/http_client.go create mode 100644 test/volume_server/framework/volume_fixture.go create mode 100644 test/volume_server/grpc/admin_extra_test.go create mode 100644 test/volume_server/grpc/admin_lifecycle_test.go create mode 100644 test/volume_server/grpc/admin_readonly_collection_test.go create mode 100644 test/volume_server/grpc/batch_delete_test.go create mode 100644 test/volume_server/grpc/copy_receive_variants_test.go create mode 100644 test/volume_server/grpc/copy_sync_test.go create mode 100644 test/volume_server/grpc/data_rw_test.go create mode 100644 test/volume_server/grpc/data_stream_success_test.go create mode 100644 test/volume_server/grpc/erasure_coding_test.go create mode 100644 test/volume_server/grpc/health_state_test.go create mode 100644 test/volume_server/grpc/scrub_query_test.go create mode 100644 test/volume_server/grpc/tail_test.go create mode 100644 test/volume_server/grpc/tiering_remote_test.go create mode 100644 test/volume_server/grpc/vacuum_test.go create mode 100644 test/volume_server/http/admin_test.go create mode 100644 test/volume_server/http/auth_test.go create mode 100644 test/volume_server/http/chunk_manifest_test.go create mode 100644 test/volume_server/http/compressed_read_test.go create mode 100644 test/volume_server/http/headers_static_test.go create mode 100644 test/volume_server/http/image_transform_test.go create mode 100644 test/volume_server/http/public_cors_methods_test.go create mode 100644 test/volume_server/http/range_variants_test.go create mode 100644 test/volume_server/http/read_deleted_test.go create mode 100644 test/volume_server/http/read_mode_proxy_redirect_test.go create mode 100644 test/volume_server/http/read_path_variants_test.go create mode 100644 test/volume_server/http/read_write_delete_test.go create mode 100644 test/volume_server/http/throttling_test.go create mode 100644 test/volume_server/http/write_delete_variants_test.go create mode 100644 test/volume_server/http/write_error_variants_test.go create mode 100644 test/volume_server/matrix/config_profiles.go diff --git a/.github/workflows/volume-server-integration-tests.yml b/.github/workflows/volume-server-integration-tests.yml new file mode 100644 index 000000000..5c278d13d --- /dev/null +++ b/.github/workflows/volume-server-integration-tests.yml @@ -0,0 +1,122 @@ +name: "Volume Server Integration Tests" + +on: + pull_request: + branches: [ master ] + paths: + - 'test/volume_server/**' + - 'weed/server/**' + - 'weed/storage/**' + - 'weed/pb/volume_server.proto' + - 'weed/pb/volume_server_pb/**' + - '.github/workflows/volume-server-integration-tests.yml' + push: + branches: [ master, main ] + paths: + - 'test/volume_server/**' + - 'weed/server/**' + - 'weed/storage/**' + - 'weed/pb/volume_server.proto' + - 'weed/pb/volume_server_pb/**' + - '.github/workflows/volume-server-integration-tests.yml' + +concurrency: + group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests + cancel-in-progress: true + +permissions: + contents: read + +env: + GO_VERSION: '1.24' + TEST_TIMEOUT: '30m' + +jobs: + volume-server-integration-tests: + name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }}) + runs-on: ubuntu-22.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + test-type: [grpc, http] + shard: [1, 2, 3] + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Build SeaweedFS binary + run: | + cd weed + go build -o weed . + chmod +x weed + ./weed version + + - name: Run volume server integration tests + env: + WEED_BINARY: ${{ github.workspace }}/weed/weed + run: | + if [ "${{ matrix.test-type }}" == "grpc" ]; then + if [ "${{ matrix.shard }}" == "1" ]; then + TEST_PATTERN="^Test[A-H]" + elif [ "${{ matrix.shard }}" == "2" ]; then + TEST_PATTERN="^Test[I-S]" + else + TEST_PATTERN="^Test[T-Z]" + fi + else + if [ "${{ matrix.shard }}" == "1" ]; then + TEST_PATTERN="^Test[A-G]" + elif [ "${{ matrix.shard }}" == "2" ]; then + TEST_PATTERN="^Test[H-R]" + else + TEST_PATTERN="^Test[S-Z]" + fi + fi + echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..." + go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}" + + - name: Collect logs on failure + if: failure() + run: | + mkdir -p /tmp/volume-server-it-logs + find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-it-logs/ \; || true + + - name: Archive logs on failure + if: failure() + uses: actions/upload-artifact@v6 + with: + name: volume-server-integration-test-logs + path: /tmp/volume-server-it-logs/ + if-no-files-found: warn + retention-days: 7 + + - name: Test summary + if: always() + run: | + if [ "${{ matrix.test-type }}" == "grpc" ]; then + if [ "${{ matrix.shard }}" == "1" ]; then + TEST_PATTERN="^Test[A-H]" + elif [ "${{ matrix.shard }}" == "2" ]; then + TEST_PATTERN="^Test[I-S]" + else + TEST_PATTERN="^Test[T-Z]" + fi + else + if [ "${{ matrix.shard }}" == "1" ]; then + TEST_PATTERN="^Test[A-G]" + elif [ "${{ matrix.shard }}" == "2" ]; then + TEST_PATTERN="^Test[H-R]" + else + TEST_PATTERN="^Test[S-Z]" + fi + fi + echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY" + echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY" + echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY" diff --git a/test/volume_server/DEV_PLAN.md b/test/volume_server/DEV_PLAN.md new file mode 100644 index 000000000..73f81e758 --- /dev/null +++ b/test/volume_server/DEV_PLAN.md @@ -0,0 +1,1129 @@ +# Volume Server Integration Test Dev Plan (Go) + +## Goal +Create a Go integration test suite under `test/volume_server` that validates **drop-in behavior parity** for the Volume Server HTTP and gRPC APIs, so a Rust rewrite can be verified against the current Go behavior. + +## Hard Requirements +- Tests live under `test/volume_server`. +- Tests are written in Go. +- HTTP + gRPC APIs are both covered. +- Coverage targets execution-path parity (happy path + edge/failure/state variants), not only API reachability. +- During implementation, commit each logic change separately for reviewability. + +## Ground Truth (API Surface) +- HTTP handlers: + - `weed/server/volume_server.go` + - `weed/server/volume_server_handlers.go` + - `weed/server/volume_server_handlers_read.go` + - `weed/server/volume_server_handlers_write.go` + - `weed/server/volume_server_handlers_admin.go` + - `weed/server/common.go` (path parsing and range handling) +- gRPC service and handlers: + - `weed/pb/volume_server.proto` + - `weed/server/volume_grpc_*.go` + +## Proposed Test Directory Tree + +```text +test/volume_server/ + DEV_PLAN.md + README.md + Makefile + framework/ + cluster.go # test cluster lifecycle, ports, process control + volume_fixture.go # test data provisioning and seed volumes + http_client.go # request helpers, auth helpers, assertions + grpc_client.go # grpc dial + common call wrappers + fault_injection.go # kill/restart peers, network/error simulation hooks + matrix/ + config_profiles.go # runtime matrix profiles (jwt, readMode, public port split, etc.) + http/ + admin_test.go # /status, /healthz, /ui, static resources + read_test.go # GET/HEAD read behaviors + variations + write_test.go # POST/PUT write behaviors + variations + delete_test.go # DELETE behaviors + variations + cors_and_options_test.go + throttling_test.go # upload/download in-flight limit paths + auth_test.go # jwt/no-jwt/bad-jwt/fid mismatch + grpc/ + admin_lifecycle_test.go + vacuum_test.go + data_rw_test.go + copy_sync_test.go + tail_test.go + erasure_coding_test.go + tiering_test.go + remote_fetch_test.go + scrub_test.go + query_test.go + health_state_test.go + compatibility/ + golden_behavior_test.go # protocol-level parity checks for selected canonical flows +``` + +## Environment/Matrix Profiles (Execution Dimensions) +Each API should be exercised across the smallest set of profiles that still covers behavior divergence: + +- `P1`: Single volume server, no JWT, `readMode=proxy`, single HTTP port. +- `P2`: Public/admin port split (`port.public != port`) to verify public read-only behavior. +- `P3`: JWT enabled (`jwt.signing.key` and `jwt.signing.read.key`) with valid/invalid/missing tokens. +- `P4`: Replicated volume layout (>=2 volume servers) to cover proxy/redirect/replicate paths. +- `P5`: Erasure coding volumes present. +- `P6`: Remote tier backend configured. +- `P7`: Maintenance mode enabled. +- `P8`: Upload/download throttling limits enabled. + +## HTTP API Test Case Tree + +### 1. Admin and service endpoints +- [ ] `GET /status` + - [ ] baseline payload fields (`Version`, `DiskStatuses`, `Volumes`) + - [ ] response headers (`Server`, request ID) +- [ ] `GET /healthz` + - [ ] healthy -> `200` + - [ ] server stopping -> `503` + - [ ] heartbeat disabled -> `503` +- [ ] `GET /ui/index.html` + - [ ] enabled path renders page + - [ ] disabled/secured behavior matches current config +- [ ] static assets (`/favicon.ico`, `/seaweedfsstatic/*`) reachable + +### 2. Data read endpoints (`GET`/`HEAD` on `/...`) +- [ ] URL shape variants + - [ ] `/{vid},{fid}` + - [ ] `/{vid}/{fid}` + - [ ] `/{vid}/{fid}/{filename}` + - [ ] malformed vid/fid -> `400` +- [ ] auth variants + - [ ] no JWT required + - [ ] read JWT missing/invalid/fid-mismatch -> `401` + - [ ] valid JWT -> success +- [ ] locality/read mode variants + - [ ] local volume read success + - [ ] missing local volume with `readMode=local` -> `404` + - [ ] missing local volume with `readMode=proxy` -> proxied response + - [ ] missing local volume with `readMode=redirect` -> `301` + - [ ] proxied-loop guard (`proxied=true`) behavior +- [ ] object state variants + - [ ] not found/deleted -> `404` + - [ ] cookie mismatch -> `404` + - [ ] internal read failure -> `500` + - [ ] `readDeleted=true` behavior +- [ ] conditional headers + - [ ] `If-Modified-Since` -> `304` + - [ ] `If-None-Match` -> `304` + - [ ] normal ETag emission +- [ ] range handling + - [ ] full read (no `Range`) + - [ ] single range -> `206`, `Content-Range` + - [ ] multi-range -> `206`, multipart body + - [ ] invalid range -> `416` + - [ ] oversized-range-sum behavior parity +- [ ] content transformations + - [ ] compressed data + `Accept-Encoding=gzip` + - [ ] compressed data without accepted encoding (decompress path) + - [ ] image resize (`width`,`height`,`mode`) + - [ ] image crop (`crop_*`) +- [ ] chunk manifest behavior + - [ ] manifest auto-expansion path + - [ ] `cm=false` bypass path +- [ ] response header passthrough queries + - [ ] `response-*` overrides and `dl` content-disposition behavior +- [ ] `HEAD` parity + - [ ] headers same as `GET` minus body + - [ ] content-length behavior parity + +### 3. Data write endpoints (`PUT`/`POST` on `/...`) +- [ ] URL shape and parse validation +- [ ] write JWT variants (missing/invalid/mismatch) +- [ ] payload variants + - [ ] standard upload success -> `201` + - [ ] unchanged write -> `204` with ETag + - [ ] oversize file rejected (file-size limit) + - [ ] malformed multipart/form data -> `400` +- [ ] metadata variants (name/mime/pairs/md5 headers) +- [ ] replication path + - [ ] `type=replicate` bypasses upload throttling + - [ ] replication write failure behavior +- [ ] throttling paths + - [ ] limit disabled + - [ ] over limit wait then proceed + - [ ] timeout -> `429` + - [ ] canceled request -> `499` + +### 4. Data delete endpoint (`DELETE` on `/...`) +- [ ] normal volume delete success (`202` + size) +- [ ] non-existing needle (`404` size=0) +- [ ] EC volume delete path +- [ ] cookie mismatch -> `400` +- [ ] chunk manifest delete (children first) + - [ ] chunk delete success + - [ ] chunk delete failure -> `500` +- [ ] `ts` override behavior +- [ ] auth variants as in write path + +### 5. Method/CORS behavior +- [ ] private port `OPTIONS` -> allow `PUT,POST,GET,DELETE,OPTIONS` +- [ ] public port `OPTIONS` -> allow `GET,OPTIONS` +- [ ] CORS origin headers on requests with `Origin` +- [ ] unsupported method behavior parity (private and public ports) + +### 6. Concurrency-limit and replica fallback behavior +- [ ] download over-limit + replica available -> proxy/redirect fallback +- [ ] download over-limit + no replica -> wait/timeout/cancel outcomes +- [ ] upload/download inflight counters update and release (no leaks) + +## gRPC API Test Case Tree + +For each RPC below: cover baseline success, validation/argument errors, state preconditions (maintenance, missing volume), and stream interruption where applicable. + +### A. Admin/Lifecycle +- [ ] `DeleteCollection` + - [ ] existing collection + - [ ] non-existing collection idempotence/error parity +- [ ] `AllocateVolume` + - [ ] success + - [ ] maintenance mode reject + - [ ] duplicate/invalid allocation parity +- [ ] `VolumeMount` + - [ ] success + - [ ] missing/not-mountable volume +- [ ] `VolumeUnmount` + - [ ] success + - [ ] missing/not-mounted volume +- [ ] `VolumeDelete` + - [ ] `only_empty=true` and `false` + - [ ] maintenance mode reject +- [ ] `VolumeConfigure` + - [ ] success + - [ ] invalid replication string -> `resp.Error` path + - [ ] unmount failure path + - [ ] configure failure + remount rollback path + - [ ] mount failure path +- [ ] `VolumeMarkReadonly` + - [ ] success with `persist=false/true` + - [ ] volume not found + - [ ] notify-master failure (pre/post local transition) +- [ ] `VolumeMarkWritable` + - [ ] success + - [ ] volume not found + - [ ] notify-master failure +- [ ] `VolumeStatus` + - [ ] success + - [ ] volume not found + - [ ] data backend missing +- [ ] `VolumeServerStatus` + - [ ] payload completeness (`State`, `MemoryStatus`, disk statuses) +- [ ] `VolumeServerLeave` + - [ ] heartbeat stopped effect +- [ ] `GetState` +- [ ] `SetState` + - [ ] state transition success + - [ ] invalid update error path +- [ ] `Ping` + - [ ] target type: filer / volume / master + - [ ] unreachable target error wrapping + - [ ] unknown target type behavior parity + +### B. Vacuum / compaction +- [ ] `VacuumVolumeCheck` + - [ ] success + garbage ratio + - [ ] missing volume/error path +- [ ] `VacuumVolumeCompact` (stream) + - [ ] progress events emitted + - [ ] maintenance mode reject + - [ ] compact failure + - [ ] client stream receive interruption +- [ ] `VacuumVolumeCommit` + - [ ] success (readonly + size fields) + - [ ] maintenance mode reject + - [ ] commit failure +- [ ] `VacuumVolumeCleanup` + - [ ] success + - [ ] maintenance mode reject + - [ ] cleanup failure + +### C. Data read/write +- [ ] `ReadNeedleBlob` + - [ ] success + - [ ] missing volume + - [ ] invalid offset/size path +- [ ] `ReadNeedleMeta` + - [ ] success + - [ ] missing volume + - [ ] EC-only volume unsupported path + - [ ] read metadata failure +- [ ] `WriteNeedleBlob` + - [ ] success + - [ ] maintenance mode reject + - [ ] missing volume + - [ ] write failure +- [ ] `VolumeNeedleStatus` + - [ ] normal volume success + - [ ] EC volume success + - [ ] volume missing + - [ ] needle missing/read error + +### D. Batch and scan +- [ ] `BatchDelete` + - [ ] `skip_cookie_check=true/false` + - [ ] invalid fid parse + - [ ] not found + - [ ] cookie mismatch path + - [ ] chunk manifest reject path (`406` in result) + - [ ] regular and EC delete paths +- [ ] `ReadAllNeedles` (stream) + - [ ] multiple volumes success + - [ ] one missing volume abort behavior + +### E. Copy/sync/replication streams +- [ ] `VolumeSyncStatus` + - [ ] success + - [ ] missing volume +- [ ] `VolumeIncrementalCopy` (stream) + - [ ] data streamed from `since_ns` + - [ ] `isLastOne` no-data path + - [ ] missing volume/error path +- [ ] `VolumeCopy` (stream) + - [ ] full copy success (`.dat/.idx/.vif`) + mount + - [ ] existing destination volume delete-before-copy path + - [ ] source unavailable / read status failure + - [ ] no free location + - [ ] remote-dat-file branch + - [ ] copy integrity mismatch failures + - [ ] final progress/append timestamp behavior +- [ ] `ReadVolumeFileStatus` + - [ ] success field validation + - [ ] missing volume +- [ ] `CopyFile` (stream) + - [ ] normal volume path + - [ ] EC volume path + - [ ] compaction revision mismatch + - [ ] missing source file with ignore flag true/false + - [ ] zero-byte and stop-offset edge cases +- [ ] `ReceiveFile` (client stream) + - [ ] happy path regular volume + - [ ] happy path EC file target + - [ ] info-first protocol violation + - [ ] unknown message type + - [ ] maintenance mode reject + - [ ] write/create failure cleanup behavior + +### F. Tailing +- [ ] `VolumeTailSender` (stream) + - [ ] volume not found + - [ ] heartbeat chunks when no updates + - [ ] idle-timeout drain completion + - [ ] large needle chunking behavior +- [ ] `VolumeTailReceiver` + - [ ] success applies streamed writes + - [ ] destination volume missing + - [ ] source stream/connect failure + +### G. Erasure coding +- [ ] `VolumeEcShardsGenerate` + - [ ] success default config + - [ ] success with existing `.vif` EC config + - [ ] maintenance mode reject + - [ ] collection mismatch + - [ ] generate/write cleanup on failure +- [ ] `VolumeEcShardsRebuild` + - [ ] rebuild missing shards success + - [ ] no shards found path + - [ ] rebuild failures +- [ ] `VolumeEcShardsCopy` + - [ ] shard copy success + - [ ] `copy_ecx`/`copy_ecj`/`copy_vif` toggles + - [ ] explicit `disk_id` valid/invalid + - [ ] no-space/source-copy failure +- [ ] `VolumeEcShardsDelete` + - [ ] delete selected shard ids + - [ ] delete-last-shard cleanup (`.ecx/.ecj` + optional `.vif`) + - [ ] missing shard no-op parity +- [ ] `VolumeEcShardsMount` + - [ ] multi-shard success + - [ ] per-shard failure abort behavior +- [ ] `VolumeEcShardsUnmount` + - [ ] multi-shard success + - [ ] per-shard failure abort behavior +- [ ] `VolumeEcShardRead` (stream) + - [ ] success + - [ ] not-found volume/shard + - [ ] deleted file key returns `IsDeleted` + - [ ] chunked streaming for large reads +- [ ] `VolumeEcBlobDelete` + - [ ] delete existing blob + - [ ] already deleted idempotence + - [ ] locate failure +- [ ] `VolumeEcShardsToVolume` + - [ ] success path from EC -> normal + - [ ] missing EC volume/shard + - [ ] invalid data-shard config + - [ ] no-live-entries failed-precondition path + - [ ] write dat/idx failures +- [ ] `VolumeEcShardsInfo` + - [ ] success counts (including deleted) + - [ ] missing EC volume + - [ ] walk-index failure + +### H. Tiering and remote +- [ ] `VolumeTierMoveDatToRemote` (stream) + - [ ] success with progress events + - [ ] maintenance mode reject + - [ ] volume missing / collection mismatch + - [ ] destination backend missing + - [ ] destination exists already + - [ ] keep-local true/false branches +- [ ] `VolumeTierMoveDatFromRemote` (stream) + - [ ] success with progress events + - [ ] volume missing / collection mismatch + - [ ] already-local path + - [ ] backend missing/download failure + - [ ] keep-remote true/false branches +- [ ] `FetchAndWriteNeedle` + - [ ] success without replicas + - [ ] success with replica fanout + - [ ] maintenance mode reject + - [ ] missing volume + - [ ] remote client/read failure + - [ ] local write failure + - [ ] one replica write failure behavior + +### I. Query and scrub +- [ ] `Query` (stream) + - [ ] JSON input selection/filter success + - [ ] malformed fid parse failure + - [ ] read/cookie mismatch failure + - [ ] CSV-input current behavior parity +- [ ] `ScrubVolume` + - [ ] auto-select all volumes when request empty + - [ ] mode `INDEX` + - [ ] mode `LOCAL` (not-implemented detail reporting) + - [ ] mode `FULL` (not-implemented detail reporting) + - [ ] unsupported mode error +- [ ] `ScrubEcVolume` + - [ ] auto-select all EC volumes when request empty + - [ ] mode `INDEX` + - [ ] mode `LOCAL` + - [ ] mode `FULL` (not-implemented detail reporting) + - [ ] unsupported mode error + +## Phased Implementation Plan (Tracking) + +### Phase 0: Harness and scaffolding +- [x] Create `framework/` cluster bootstrap and teardown +- [x] Add profile-based environment builder (P1..P8) +- [x] Add common assertion helpers (HTTP and gRPC) +- [x] Add `README.md` with run instructions +- [x] Add `Makefile` targets (`test-volume-server`, profile filters) + +### Phase 1: HTTP parity suites +- [ ] Admin/status/health/UI/static +- [ ] Read path variants and headers/range/transforms +- [ ] Write/delete/auth/throttling/public-port behavior + +### Phase 2: gRPC parity suites (core) +- [x] Admin/lifecycle/state/ping +- [ ] Vacuum + batch + data rw +- [ ] Copy/sync/tail + +### Phase 3: gRPC parity suites (advanced) +- [ ] Erasure coding family +- [ ] Tiering + remote fetch +- [ ] Query + scrub + +### Phase 4: Compatibility hardening +- [ ] Golden behavior assertions for canonical flows +- [ ] Flake reduction and deterministic retries/timeouts +- [ ] CI runtime tuning and sharding + +## Commit Strategy (for implementation) +Use one commit per logical change set. Suggested sequence: + +1. `test(volume_server): add integration framework and cluster lifecycle` +2. `test(volume_server): add config profile matrix and test utilities` +3. `test(volume_server/http): add admin and health endpoint coverage` +4. `test(volume_server/http): add read-path matrix coverage` +5. `test(volume_server/http): add write/delete/auth/throttling coverage` +6. `test(volume_server/grpc): add admin lifecycle and state/ping coverage` +7. `test(volume_server/grpc): add vacuum batch and data rw coverage` +8. `test(volume_server/grpc): add copy sync and tail coverage` +9. `test(volume_server/grpc): add erasure coding coverage` +10. `test(volume_server/grpc): add tiering and remote fetch coverage` +11. `test(volume_server/grpc): add query and scrub coverage` +12. `test(volume_server): add compatibility golden scenarios and docs` + +## Progress Log +Update this section during implementation: + +- Date: 2026-02-12 +- Change: Added initial dev plan and project scaffold. +- APIs covered: Planning only. +- Profiles covered: Planning only. +- Gaps introduced/remaining: Implementation not started yet. +- Commit: `21c10a9ec` + +- Date: 2026-02-12 +- Change: Added integration harness, profile matrix, and auto-build support for missing `weed` binary. Master now starts with `-peers=none` and low `-volumeSizeLimitMB`. +- APIs covered: Harness only. +- Profiles covered: P1, P2, P3, P8 definitions in place. +- Gaps introduced/remaining: Full API case matrix still pending. +- Commit: `5e9c437e0`, `1be296139` + +- Date: 2026-02-12 +- Change: Added HTTP integration coverage for admin endpoints, method options, and upload/read/range/head/delete roundtrip. +- APIs covered: `/status`, `/healthz`, `/ui/index.html`, `OPTIONS /`, `POST/GET/HEAD/DELETE /{fid}`. +- Profiles covered: P1, P2. +- Gaps introduced/remaining: Remaining HTTP branch variants (JWT/proxy/redirect/throttling/etc.) still pending. +- Commit: `038e9161e`, `1df1e3812` + +- Date: 2026-02-12 +- Change: Added gRPC integration coverage for state/status/ping and admin lifecycle/maintenance checks. +- APIs covered: `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `AllocateVolume`, `VolumeStatus`, `VolumeMount`, `VolumeUnmount`, `VolumeDelete`. +- Profiles covered: P1. +- Gaps introduced/remaining: Remaining gRPC methods and advanced branches still pending. +- Commit: `9a9b8c500`, `3c562a64c` + +- Date: 2026-02-12 +- Change: Added HTTP cache/range branch tests. +- APIs covered: `GET /{fid}` with `If-None-Match` (`304`) and invalid `Range` (`416`). +- Profiles covered: P1. +- Gaps introduced/remaining: Remaining HTTP auth/proxy/redirect/throttling branches pending. +- Commit: `317346b51` + +- Date: 2026-02-12 +- Change: Added gRPC `BatchDelete` integration checks for invalid fid mapping and maintenance-mode rejection. +- APIs covered: `BatchDelete`. +- Profiles covered: P1. +- Gaps introduced/remaining: Remaining gRPC method families still pending. +- Commit: `7a8aed127` + +- Date: 2026-02-12 +- Change: Added gRPC integration tests for needle status, configure validation branch, ping volume-target branch, and leave/health interaction. +- APIs covered: `VolumeNeedleStatus`, `VolumeConfigure` (invalid replication response path), `Ping` (`volumeServer` target), `VolumeServerLeave`. +- Profiles covered: P1. +- Gaps introduced/remaining: Still pending large RPC groups (vacuum/copy/tail/ec/tiering/query/scrub). +- Commit: `59a571a10` + +- Date: 2026-02-12 +- Change: Added gRPC vacuum integration coverage for success/missing-volume and maintenance-mode rejection branches. +- APIs covered: `VacuumVolumeCheck`, `VacuumVolumeCompact`, `VacuumVolumeCommit`, `VacuumVolumeCleanup`. +- Profiles covered: P1. +- Gaps introduced/remaining: Copy/sync/tail, EC, tiering, query, scrub, and many HTTP matrix branches still pending. +- Commit: `0f7cc53dd` + +- Date: 2026-02-12 +- Change: Added gRPC data read/write error-path coverage for missing-volume and maintenance-mode branches. +- APIs covered: `ReadNeedleBlob`, `ReadNeedleMeta`, `WriteNeedleBlob`. +- Profiles covered: P1. +- Gaps introduced/remaining: Positive-path blob/meta and stream/copy/tail/EC/tiering/query/scrub families remain. +- Commit: `f83ad41b5` + +- Date: 2026-02-12 +- Change: Added HTTP JWT integration coverage for missing/invalid/valid token behavior across write and read paths. +- APIs covered: HTTP `POST /{fid}` and `GET /{fid}` auth paths with read/write signing keys. +- Profiles covered: P3. +- Gaps introduced/remaining: Remaining HTTP proxy/redirect/throttling branches still pending. +- Commit: `def509acb` + +- Date: 2026-02-12 +- Change: Added gRPC sync/copy family tests for success and missing-volume or maintenance-mode stream error paths. +- APIs covered: `VolumeSyncStatus`, `VolumeIncrementalCopy`, `ReadAllNeedles`, `ReadVolumeFileStatus`, `CopyFile`, `VolumeCopy`, `ReceiveFile`. +- Profiles covered: P1. +- Gaps introduced/remaining: Tail/EC/tiering/query/scrub and positive-path copy/tail flows still pending. +- Commit: `b13642838` + +- Date: 2026-02-12 +- Change: Added gRPC scrub and query integration coverage for supported/unsupported modes and invalid/missing fid paths. +- APIs covered: `ScrubVolume`, `ScrubEcVolume`, `Query`. +- Profiles covered: P1. +- Gaps introduced/remaining: Query success path and broader scrub mode matrix remain pending. +- Commit: `a2ab4cde8` + +- Date: 2026-02-12 +- Change: Added gRPC tail integration coverage for sender heartbeat/EOF behavior and sender/receiver missing-volume errors. +- APIs covered: `VolumeTailSender`, `VolumeTailReceiver`. +- Profiles covered: P1. +- Gaps introduced/remaining: Tail success replication path and large-needle chunking remain pending. +- Commit: `fd582ba58` + +- Date: 2026-02-12 +- Change: Expanded gRPC admin/lifecycle coverage with readonly/writable transitions, collection delete behavior, non-empty delete `only_empty` branch, and ping unknown/unreachable target variants. +- APIs covered: `VolumeMarkReadonly`, `VolumeMarkWritable`, `DeleteCollection`, `VolumeDelete` (`only_empty=true/false`), `Ping` (unknown and unreachable master target). +- Profiles covered: P1. +- Gaps introduced/remaining: Notify-master failure paths for readonly/writable and additional admin error branches still pending. +- Commit: `2e6d577f7`, `a3a2da791`, `9f887f25c`, `724bbe2d9` + +- Date: 2026-02-12 +- Change: Added gRPC positive-path data coverage for blob/meta read/write roundtrip and stream-all-needles payload validation. +- APIs covered: `ReadNeedleBlob`, `ReadNeedleMeta`, `WriteNeedleBlob`, `ReadAllNeedles`. +- Profiles covered: P1. +- Gaps introduced/remaining: Additional blob/meta offset/size corruption branches remain. +- Commit: `21e94d1d2` + +- Date: 2026-02-12 +- Change: Expanded gRPC scrub/query coverage for auto-select volume behavior, local/full scrub detail branches, JSON query success filtering, CSV no-output behavior, and EC auto-select empty result. +- APIs covered: `ScrubVolume` (`INDEX`, `LOCAL`, `FULL`, auto-select), `ScrubEcVolume` (missing-volume + auto-select empty), `Query` (JSON success + CSV no-output + invalid paths). +- Profiles covered: P1. +- Gaps introduced/remaining: EC scrub full/local positive paths need EC fixture setup. +- Commit: `8cdf3589a`, `12150a9a2` + +- Date: 2026-02-12 +- Change: Added gRPC tiering/remote early-branch error coverage. +- APIs covered: `FetchAndWriteNeedle` (maintenance + missing volume), `VolumeTierMoveDatToRemote` (missing volume, collection mismatch, maintenance), `VolumeTierMoveDatFromRemote` (missing volume, collection mismatch, already-local path). +- Profiles covered: P1. +- Gaps introduced/remaining: Tier upload/download success flows with real remote backend remain. +- Commit: `51e6fa749` + +- Date: 2026-02-12 +- Change: Expanded HTTP behavior coverage for split public port semantics, CORS on origin requests, unsupported-method parity, unchanged-write `204`, delete edge branches, and JWT fid-mismatch auth rejection. +- APIs covered: public/admin method divergence (`GET/HEAD/POST/DELETE/PATCH`), CORS headers, write unchanged response path, delete cookie-mismatch/missing-needle paths, JWT fid mismatch for write/read. +- Profiles covered: P1, P2, P3. +- Gaps introduced/remaining: Remaining HTTP proxy/redirect/throttling and transformation branches still pending. +- Commit: `2de39c548`, `9998d19dd`, `ea5d8b7b3` + +- Date: 2026-02-12 +- Change: Expanded tier/remote gRPC variation coverage with invalid remote config and missing destination backend branches. +- APIs covered: `FetchAndWriteNeedle` (invalid `RemoteConf`), `VolumeTierMoveDatToRemote` (destination backend not found). +- Profiles covered: P1. +- Gaps introduced/remaining: Tier upload/download success flows with an actual remote backend and replica fanout behavior remain. +- Commit: `855c84f31` + +- Date: 2026-02-12 +- Change: Expanded copy/receive stream coverage with incremental-copy data/no-data branches and receive-file protocol violation handling. +- APIs covered: `VolumeIncrementalCopy` (stream data + EOF no-data), `CopyFile` (ignore missing source + `stop_offset=0`), `ReceiveFile` (content-before-info and unknown message type response errors). +- Profiles covered: P1. +- Gaps introduced/remaining: Full `VolumeCopy` happy path with a real source volume node remains. +- Commit: `1e99407e1` + +- Date: 2026-02-12 +- Change: Added additional copy/receive branches for compaction mismatch and regular-volume receive-file success with byte-for-byte verification via `CopyFile`. +- APIs covered: `CopyFile` (compaction revision mismatch), `ReceiveFile` (successful regular volume write path). +- Profiles covered: P1. +- Gaps introduced/remaining: EC receive-file success path and cleanup failure branches remain. +- Commit: `4c710463e` + +- Date: 2026-02-12 +- Change: Added HTTP read-path variants and conditional request coverage. +- APIs covered: `GET /{vid}/{fid}`, `GET /{vid}/{fid}/{filename}`, malformed `/{vid}/{fid}` parse error path, `If-Modified-Since` (`304`) behavior. +- Profiles covered: P1. +- Gaps introduced/remaining: Proxy/redirect read mode matrix and image/chunk-manifest transformation branches remain. +- Commit: `1f64ebe1d` + +- Date: 2026-02-12 +- Change: Added HTTP passthrough header and static resource coverage. +- APIs covered: query-based `response-*` header passthrough, `dl=true` content-disposition attachment handling, `/favicon.ico`, `/seaweedfsstatic/seaweed50x50.png`. +- Profiles covered: P1. +- Gaps introduced/remaining: Additional static resource variants and multi-range response formatting checks remain. +- Commit: `f1ad1ec50` + +- Date: 2026-02-12 +- Change: Added gRPC ping branch coverage for unreachable filer target. +- APIs covered: `Ping` (`target_type=filer` unreachable target path). +- Profiles covered: P1. +- Gaps introduced/remaining: Successful ping path for filer/master targets in multi-service integration setup remains. +- Commit: `c6ace0331` + +- Date: 2026-02-12 +- Change: Added initial erasure-coding RPC integration coverage for maintenance-gate, missing-volume, invalid-disk, and no-op behaviors. +- APIs covered: `VolumeEcShardsGenerate`, `VolumeEcShardsRebuild`, `VolumeEcShardsCopy`, `VolumeEcShardsDelete`, `VolumeEcShardsMount`, `VolumeEcShardsUnmount`, `VolumeEcShardRead`, `VolumeEcBlobDelete`, `VolumeEcShardsToVolume`, `VolumeEcShardsInfo`. +- Profiles covered: P1. +- Gaps introduced/remaining: Positive EC data-path flows (generate/copy/mount/read/delete/to-volume/info with actual shard files) still require EC fixture setup. +- Commit: `c7592d118` + +- Date: 2026-02-12 +- Change: Added HTTP multi-range response coverage for multipart `206` behavior. +- APIs covered: `GET /{fid}` with multi-range header (`Range: bytes=0-1,4-5`) and multipart response validation. +- Profiles covered: P1. +- Gaps introduced/remaining: Oversized multi-range sum behavior and deeper range-edge normalization remain. +- Commit: `39c68c679` + +- Date: 2026-02-12 +- Change: Added query no-match parity coverage to lock current stream semantics. +- APIs covered: `Query` JSON filter no-match path (returns one empty stripe, then EOF). +- Profiles covered: P1. +- Gaps introduced/remaining: CSV parsing behavior beyond current no-output branch still pending. +- Commit: `39895cb84` + +- Date: 2026-02-12 +- Change: Added HTTP upload throttling integration coverage with deterministic timeout and replicate-bypass behavior. +- APIs covered: upload limit timeout path (`429`) and `type=replicate` bypass branch under concurrent upload pressure. +- Profiles covered: P8 (with short inflight timeout in test profile). +- Gaps introduced/remaining: download throttling wait/proxy branches remain. +- Commit: `464d0b2b6` + +- Date: 2026-02-12 +- Change: Added HTTP download throttling timeout coverage under concurrent large-read pressure. +- APIs covered: download limit timeout path (`429`) when another large response keeps in-flight download data above limit. +- Profiles covered: P8 (short inflight download timeout). +- Gaps introduced/remaining: download replica proxy fallback branch (`proxied=true`/replica redirect) remains. +- Commit: `a929e6ddc` + +- Date: 2026-02-12 +- Change: Expanded JWT auth mismatch variations for same-needle wrong-cookie tokens. +- APIs covered: write/read JWT rejection when token fid differs only by cookie from requested fid. +- Profiles covered: P3. +- Gaps introduced/remaining: token expiry boundary behavior remains untested. +- Commit: `61fe52398` + +- Date: 2026-02-12 +- Change: Added JWT expired-token rejection coverage for both write and read auth paths. +- APIs covered: write/read auth rejection when token signature is valid but `exp` is in the past. +- Profiles covered: P3. +- Gaps introduced/remaining: additional JWT transport variants (query/cookie token sources) remain. +- Commit: `6e808623f` + +- Date: 2026-02-12 +- Change: Added JWT token transport coverage via query parameter and HTTP-only cookie. +- APIs covered: write auth using `?jwt=` token and read auth using `AT` cookie token. +- Profiles covered: P3. +- Gaps introduced/remaining: JWT precedence rules when multiple token sources are present remain. +- Commit: `ccefdfe8d` + +- Date: 2026-02-12 +- Change: Added JWT token-source precedence coverage when both query and header tokens are present. +- APIs covered: query-token precedence over header-token for write/read auth checks. +- Profiles covered: P3. +- Gaps introduced/remaining: explicit query-vs-cookie precedence combination remains. +- Commit: `605054e5d` + +- Date: 2026-02-12 +- Change: Added JWT token-source precedence coverage when both header and cookie tokens are present. +- APIs covered: header-token precedence over cookie-token for write/read auth checks. +- Profiles covered: P3. +- Gaps introduced/remaining: JWT transport precedence matrix for query/header/cookie is now covered for tested combinations. +- Commit: `3fcaf845c` + +- Date: 2026-02-12 +- Change: Added JWT token-source precedence coverage when both query and cookie tokens are present. +- APIs covered: query-token precedence over cookie-token for write/read auth checks, including positive path when query is valid and cookie is invalid. +- Profiles covered: P3. +- Gaps introduced/remaining: none in current JWT token source precedence matrix. +- Commit: `4ea552973` + +- Date: 2026-02-12 +- Change: Added gRPC state update validation coverage for optimistic versioning and nil-state requests. +- APIs covered: `SetState` stale-version mismatch error path and nil-state no-op path. +- Profiles covered: P1. +- Gaps introduced/remaining: persistent state save failure branch remains environment-dependent. +- Commit: `34ff97996` + +- Date: 2026-02-12 +- Change: Added readonly lifecycle variation for persisted readonly flag path. +- APIs covered: `VolumeMarkReadonly` success path with `persist=true`. +- Profiles covered: P1. +- Gaps introduced/remaining: notify-master failure branches remain untested. +- Commit: `c37e6cd95` + +- Date: 2026-02-12 +- Change: Added CORS header validation on `OPTIONS` requests with `Origin` for admin and public ports. +- APIs covered: `OPTIONS /` CORS headers (`Access-Control-Allow-Origin`, `Access-Control-Allow-Credentials`) for split-port profile. +- Profiles covered: P2. +- Gaps introduced/remaining: unsupported-method parity for additional verbs beyond `PATCH`/`TRACE` remains. +- Commit: `ca08af7ba` + +- Date: 2026-02-12 +- Change: Added unsupported-method parity coverage for `TRACE` on admin/public split ports. +- APIs covered: admin `TRACE` error (`400`) vs public `TRACE` passthrough (`200`) behavior. +- Profiles covered: P2. +- Gaps introduced/remaining: broader unsupported verb matrix remains. +- Commit: `b03ddf855` + +- Date: 2026-02-12 +- Change: Added gRPC batch-delete cookie-check variation coverage. +- APIs covered: `BatchDelete` mismatch-cookie rejection path (`skip_cookie_check=false`) and skip-cookie-check acceptance/deletion path (`skip_cookie_check=true`). +- Profiles covered: P1. +- Gaps introduced/remaining: batch-delete malformed entry combinations are partially covered; mixed per-entry status permutations can be expanded. +- Commit: `87d75e786` + +- Date: 2026-02-12 +- Change: Expanded gRPC admin lifecycle variants for allocate/mount/unmount/delete edge cases. +- APIs covered: duplicate `AllocateVolume` rejection, missing-volume `VolumeMount` error, idempotent `VolumeUnmount` behavior for missing/already-unmounted volumes, and `VolumeDelete` maintenance-mode rejection. +- Profiles covered: P1. +- Gaps introduced/remaining: `VolumeConfigure` rollback/mount-failure branches still need dedicated fault-path coverage. +- Commit: `bc1faec8e` + +- Date: 2026-02-12 +- Change: Added mixed gRPC `BatchDelete` result-matrix coverage including early-stop behavior on cookie mismatch. +- APIs covered: per-entry status matrix in one request (`400` invalid fid, `202` accepted delete, `404` missing fid) and early break semantics when cookie mismatch occurs before later entries. +- Profiles covered: P1. +- Gaps introduced/remaining: chunk-manifest rejection (`406`) and EC batch-delete success path still require dedicated fixtures. +- Commit: `450f63ac4` + +- Date: 2026-02-12 +- Change: Added secured-UI HTTP behavior coverage under JWT-enabled profile. +- APIs covered: `/ui/index.html` route behavior when admin UI is not exposed due signing key; verified fallback auth-gated response path (`401`). +- Profiles covered: P3. +- Gaps introduced/remaining: explicit `access.ui=true` override scenario remains untested. +- Commit: `9c10ccb38` + +- Date: 2026-02-12 +- Change: Expanded split-port unsupported HTTP method matrix with non-standard verb coverage. +- APIs covered: admin/public parity for `PROPFIND` (`400` on admin, passthrough `200` on public) with post-call data-integrity verification. +- Profiles covered: P2. +- Gaps introduced/remaining: remaining unsupported-verb breadth now primarily around less common methods (e.g., `CONNECT`) and proxy-specific edge semantics. +- Commit: `1d7afd11e` + +- Date: 2026-02-12 +- Change: Expanded gRPC `VolumeConfigure` coverage for both success and configure-failure rollback reporting. +- APIs covered: valid replication success path and missing-volume configure failure path with remount-restore failure detail propagation. +- Profiles covered: P1. +- Gaps introduced/remaining: explicit unmount-failure and mount-failure branches via injected I/O faults are still pending. +- Commit: `287a60197` + +- Date: 2026-02-12 +- Change: Added `VolumeNeedleStatus` error-path coverage. +- APIs covered: missing-volume error path and missing-needle error path on existing normal volumes. +- Profiles covered: P1. +- Gaps introduced/remaining: EC-backed positive/error status permutations still require dedicated EC fixture state. +- Commit: `bf0c609a7` + +- Date: 2026-02-12 +- Change: Added HTTP deleted-needle read recovery coverage. +- APIs covered: `GET` with `readDeleted=true` returning deleted needle content, alongside normal post-delete `404` behavior. +- Profiles covered: P1. +- Gaps introduced/remaining: proxy/redirect interactions with `readDeleted` remain unverified. +- Commit: `2ed9434cf` + +- Date: 2026-02-12 +- Change: Added HTTP delete `ts` query parity coverage for deleted-read metadata behavior. +- APIs covered: `DELETE ?ts=` followed by `GET ?readDeleted=true`, asserting current Last-Modified parity with pre-delete reads. +- Profiles covered: P1. +- Gaps introduced/remaining: explicit externally visible timestamp override effects remain limited in current API responses. +- Commit: `225b8e800` + +- Date: 2026-02-12 +- Change: Added gRPC invalid-offset coverage for needle blob/meta reads. +- APIs covered: `ReadNeedleBlob` and `ReadNeedleMeta` failure paths on existing volumes with out-of-range offsets. +- Profiles covered: P1. +- Gaps introduced/remaining: low-level corrupted-size and backend I/O fault branches still require fault-injection hooks. +- Commit: `33ed77ad6` + +- Date: 2026-02-12 +- Change: Added mixed-volume `ReadAllNeedles` stream abort coverage. +- APIs covered: stream progression from an existing volume followed by missing-volume abort error in the same request. +- Profiles covered: P1. +- Gaps introduced/remaining: multi-volume happy-path ordering/volume-boundary assertions can be expanded further. +- Commit: `7799b28b1` + +- Date: 2026-02-12 +- Change: Tightened HTTP `HEAD` parity assertions for read path. +- APIs covered: `HEAD` behavior now verifies empty response body while retaining `Content-Length` parity expectations. +- Profiles covered: P1. +- Gaps introduced/remaining: additional conditional-header parity checks on `HEAD` can still be expanded. +- Commit: `9499e5400` + +- Date: 2026-02-12 +- Change: Expanded `VolumeServerStatus` payload assertions. +- APIs covered: `VolumeServerStatus` now validates presence of `State` and `MemoryStatus` (including non-zero goroutine count), in addition to version/disk payload checks. +- Profiles covered: P1. +- Gaps introduced/remaining: heartbeat-disabled and stopping-state transitions are still exercised indirectly rather than in a dedicated status-payload transition test. +- Commit: `374411418` + +- Date: 2026-02-12 +- Change: Added gRPC `BatchDelete` chunk-manifest rejection coverage. +- APIs covered: `BatchDelete` result status/error path for chunk-manifest needles (`406`, `ChunkManifest` message) and non-deletion parity after rejection. +- Profiles covered: P1. +- Gaps introduced/remaining: EC-backed `BatchDelete` positive path still pending dedicated EC fixture setup. +- Commit: `326be22a9` + +- Date: 2026-02-12 +- Change: Added gRPC `Query` cookie-mismatch branch parity coverage. +- APIs covered: `Query` behavior when fid id exists but cookie mismatches; verified current EOF/no-record stream outcome. +- Profiles covered: P1. +- Gaps introduced/remaining: CSV parsing behavior beyond current no-output path remains pending. +- Commit: `2aaf0a339` + +- Date: 2026-02-12 +- Change: Added positive gRPC `Ping` coverage for master targets. +- APIs covered: `Ping` success path for `target_type=master` with non-zero remote timestamp and valid timing envelope. +- Profiles covered: P1. +- Gaps introduced/remaining: positive filer-target ping path still requires a filer fixture in the integration harness. +- Commit: `fa5cad6dc` + +- Date: 2026-02-12 +- Change: Expanded HTTP conditional-header parity for `HEAD`. +- APIs covered: `HEAD` with `If-None-Match` now verifies `304` behavior and empty-body semantics. +- Profiles covered: P1. +- Gaps introduced/remaining: explicit `HEAD` + `If-Modified-Since` parity remains expandable. +- Commit: `9984f2ec4` + +- Date: 2026-02-12 +- Change: Added HTTP `HEAD` + `If-Modified-Since` conditional parity coverage. +- APIs covered: `HEAD` conditional path returning `304` with empty-body semantics when `If-Modified-Since` matches Last-Modified. +- Profiles covered: P1. +- Gaps introduced/remaining: deeper conditional-header combinations (`If-None-Match` + `If-Modified-Since` precedence) remain expandable. +- Commit: `e87563a3c` + +- Date: 2026-02-12 +- Change: Expanded split-port unsupported-method matrix with `CONNECT` parity coverage. +- APIs covered: admin/public behavior for `CONNECT` (`400` on admin, passthrough `200` on public) with post-call data-integrity verification. +- Profiles covered: P2. +- Gaps introduced/remaining: unsupported-method parity now covers `PATCH`, `TRACE`, `PROPFIND`, and `CONNECT`; additional uncommon verbs can still be sampled as needed. +- Commit: `2a893d10d` + +- Date: 2026-02-12 +- Change: Added explicit CORS `Access-Control-Allow-Headers` assertions for `OPTIONS`. +- APIs covered: admin/public `OPTIONS` now verify `Access-Control-Allow-Headers: *` in addition to allowed-method matrices. +- Profiles covered: P2. +- Gaps introduced/remaining: CORS method/header semantics are now covered for baseline split-port flows. +- Commit: `6fcb9fa9c` + +- Date: 2026-02-12 +- Change: Added dual-volume integration harness and read-mode matrix tests for missing-local volume behavior. +- APIs covered: HTTP read path when volume is missing locally across `readMode=proxy` (forward success), `readMode=redirect` (`301` + `proxied=true`), and `readMode=local` (`404`). +- Profiles covered: custom P1-derived profiles with `ReadMode` overrides. +- Gaps introduced/remaining: throttling-specific proxy fallback (`checkDownloadLimit` replica path) is still pending targeted pressure setup. +- Commits: `74b04a3f8`, `70ce0c8b8` + +- Date: 2026-02-12 +- Change: Added HTTP download-throttling replica fallback coverage under over-limit pressure. +- APIs covered: `checkDownloadLimit` replica-proxy branch (`download over limit + replica available -> proxy fallback`) with replicated dual-node setup. +- Profiles covered: P8-derived profile (`readMode=proxy`) with dual volume servers. +- Gaps introduced/remaining: cancellation branch (`499`) for download-limit waiting remains pending. +- Commit: `316cfb7a3` + +- Date: 2026-02-12 +- Change: Expanded missing-local deleted-read parity across proxy and redirect modes. +- APIs covered: `readDeleted=true` behavior from non-owning servers in `readMode=proxy` (forwarded success) and `readMode=redirect` (redirect query-drop parity leading to `404` on follow). +- Profiles covered: custom P1-derived profiles with `ReadMode` overrides. +- Gaps introduced/remaining: explicit proxied-loop edge behavior remains pending dedicated setup. +- Commit: `0164a383d` + +- Date: 2026-02-12 +- Change: Added filer-enabled harness and positive gRPC `Ping` coverage for filer targets. +- APIs covered: `Ping` success path for `target_type=filer` with non-zero remote timestamp and valid timing envelope. +- Profiles covered: P1 (single volume + filer auxiliary process). +- Gaps introduced/remaining: no additional ping target-type gaps remain in current harness scope. +- Commits: `5f09d86a8`, `2fc1dde3f` + +- Date: 2026-02-12 +- Change: Added download-limit proxied-loop guard coverage. +- APIs covered: over-limit download path with `proxied=true` now verifies replica fallback is skipped and timeout returns `429`. +- Profiles covered: P8-derived profile (`readMode=proxy`) with dual volume servers. +- Gaps introduced/remaining: explicit cancellation (`499`) branch for wait loops remains difficult to assert over HTTP transport semantics. +- Commit: `6d532eddc` + +- Date: 2026-02-12 +- Change: Added explicit no-limit throttling coverage for baseline profile. +- APIs covered: upload/download limit-disabled branches (`concurrent*Limit=0`) under concurrent pressure, verifying requests proceed (`200`/`201`) without throttling. +- Profiles covered: P1. +- Gaps introduced/remaining: cancellation (`499`) path remains pending due client-transport observability constraints. +- Commit: `2cd9a9c6f` + +- Date: 2026-02-12 +- Change: Added gRPC `VolumeServerLeave` idempotence coverage. +- APIs covered: repeated `VolumeServerLeave` calls (already-stopped heartbeat path) with persistent `healthz=503` verification. +- Profiles covered: P1. +- Gaps introduced/remaining: none for leave semantics in current harness. +- Commit: `0fd666916` + +- Date: 2026-02-12 +- Change: Expanded redirect read-mode query handling coverage for collection-aware redirects. +- APIs covered: non-owning redirect path now verifies `collection` query parameter preservation in `Location` alongside `proxied=true`. +- Profiles covered: P1-derived profile with `ReadMode=redirect` using dual volume servers. +- Gaps introduced/remaining: redirect branch currently preserves only `collection`; broader query propagation is intentionally untested for parity with current behavior. +- Commit: `ad287b392` + +- Date: 2026-02-12 +- Change: Tightened HTTP admin endpoint header parity checks. +- APIs covered: `/status` and `/healthz` now assert `Server` header format (`SeaweedFS Volume ...`) in addition to status and payload checks. +- Profiles covered: P1. +- Gaps introduced/remaining: none for baseline admin header checks. +- Commit: `cad34314b` + +- Date: 2026-02-12 +- Change: Expanded admin middleware parity checks for request-id propagation. +- APIs covered: `/healthz` now explicitly verifies request-id echo behavior via `x-amz-request-id` response header. +- Profiles covered: P1. +- Gaps introduced/remaining: none for request-id propagation on covered admin endpoints. +- Commit: `e0268a5b7` + +- Date: 2026-02-12 +- Change: Added over-limit invalid-vid branch coverage in download throttling proxy path. +- APIs covered: `checkDownloadLimit` -> `tryProxyToReplica` invalid volume-id parse path now explicitly verified as `400` under over-limit pressure. +- Profiles covered: P8. +- Gaps introduced/remaining: cancellation (`499`) branch remains pending due client-side transport observability limits. +- Commit: `b4984b335` + +- Date: 2026-02-12 +- Change: Expanded static-resource coverage to split public-port topology. +- APIs covered: public-port static endpoints (`/favicon.ico`, `/seaweedfsstatic/seaweed50x50.png`) under P2. +- Profiles covered: P2. +- Gaps introduced/remaining: static asset baseline coverage is now present for both admin and public ports. +- Commit: `e4c329811` + +- Date: 2026-02-12 +- Change: Added split public-port `HEAD` method parity coverage. +- APIs covered: public-port `HEAD` read behavior (`200`, content-length parity, empty-body semantics) for existing files. +- Profiles covered: P2. +- Gaps introduced/remaining: none for baseline public-port `GET/HEAD/OPTIONS` method coverage. +- Commit: `127c43b1a` + +- Date: 2026-02-12 +- Change: Added throttling wait-then-proceed branch coverage for both upload and download paths. +- APIs covered: over-limit `wait then proceed` behavior (`waitForUploadSlot` and `waitForDownloadSlot`) when in-flight pressure is released before timeout. +- Profiles covered: P8. +- Gaps introduced/remaining: explicit cancellation (`499`) path remains pending due client-side transport observability limits. +- Commit: `f7b362a2a` + +- Date: 2026-02-12 +- Change: Added HTTP read cookie-mismatch parity coverage. +- APIs covered: `GET` and `HEAD` wrong-cookie reads now explicitly verify `404` not-found semantics. +- Profiles covered: P1. +- Gaps introduced/remaining: internal read-failure (`500`) branch remains fault-injection dependent. +- Commit: `8fc192827` + +- Date: 2026-02-12 +- Change: Added throttling timeout-recovery coverage to validate in-flight counter release behavior. +- APIs covered: upload/download timeout (`429`) scenarios followed by successful recovery requests after pressure release. +- Profiles covered: P8. +- Gaps introduced/remaining: explicit cancellation (`499`) path remains pending due client-side transport observability limits. +- Commit: `3214972fa` + +- Date: 2026-02-12 +- Change: Added positive erasure-coding lifecycle coverage for generate/mount/info/unmount flows. +- APIs covered: `VolumeEcShardsGenerate` success, `VolumeEcShardsMount` success, `VolumeEcShardsInfo` success after mount, and expected not-found after unmount. +- Profiles covered: P1. +- Gaps introduced/remaining: multi-shard, rebuild/copy/blob-delete/to-volume positive permutations remain for broader EC matrix completeness. +- Commit: `80dce7c5b` + +- Date: 2026-02-12 +- Change: Expanded EC positive-path coverage for shard-read and blob-delete behavior. +- APIs covered: `VolumeEcShardRead` success, `VolumeEcBlobDelete` first-delete + idempotent second-delete, and `VolumeEcShardRead` deleted-marker (`IsDeleted`) path. +- Profiles covered: P1. +- Gaps introduced/remaining: EC copy/rebuild/to-volume multi-node success permutations remain for broader matrix completeness. +- Commit: `1f405f52d` + +- Date: 2026-02-13 +- Change: Added EC rebuild and `ShardsToVolume` branch coverage for missing-shard and no-live-entry conditions. +- APIs covered: `VolumeEcShardsRebuild` (missing-shard regenerate path), `VolumeEcShardsToVolume` (missing data shard error and no-live-entries `FailedPrecondition` path). +- Profiles covered: P1. +- Gaps introduced/remaining: `VolumeEcShardsToVolume` full success conversion path and EC copy/rebuild multi-node permutations remain pending. +- Commit: `e8c449c16` + +- Date: 2026-02-13 +- Change: Added positive `VolumeEcShardsToVolume` conversion coverage with post-conversion read verification. +- APIs covered: `VolumeEcShardsToVolume` success path (mounted data shards -> convert -> readable needle payload parity). +- Profiles covered: P1. +- Gaps introduced/remaining: EC multi-node shard-copy/rebuild permutations and last-shard cleanup edge cases remain pending. +- Commit: `6d223338a` + +- Date: 2026-02-13 +- Change: Added EC file-stream integration coverage for `ReceiveFile` and `CopyFile`. +- APIs covered: `ReceiveFile` success with `is_ec_volume=true`, and `CopyFile` EC missing-source behavior with `ignore_source_file_not_found=true/false`. +- Profiles covered: P1. +- Gaps introduced/remaining: multi-node EC shard copy/rebuild permutations and last-shard cleanup edge assertions remain pending. +- Commit: `a78290d56` + +- Date: 2026-02-13 +- Change: Added EC last-shard deletion cleanup coverage. +- APIs covered: `VolumeEcShardsDelete` branch where deleting the final shard set removes EC index files (`.ecx` cleanup), validated via EC `CopyFile` pre/post behavior. +- Profiles covered: P1. +- Gaps introduced/remaining: EC multi-node shard copy/rebuild permutations remain pending for broader distributed parity. +- Commit: `e8ef35346` + +- Date: 2026-02-13 +- Change: Added dual-node `VolumeCopy` integration success coverage. +- APIs covered: `VolumeCopy` full happy path (source status read, stream copy, destination mount) with post-copy HTTP read parity from destination node. +- Profiles covered: P4-derived dual-volume topology. +- Gaps introduced/remaining: additional distributed copy branches (e.g., existing-destination overwrite and remote-dat variations) remain pending. +- Commit: `7538653ad` + +- Date: 2026-02-13 +- Change: Added dual-node `VolumeCopy` overwrite coverage when destination volume already exists. +- APIs covered: existing-destination delete-before-copy branch, with pre-copy destination payload assertion and post-copy source parity validation. +- Profiles covered: P4-derived dual-volume topology. +- Gaps introduced/remaining: remote-dat-file and no-space source-copy branches remain pending. +- Commit: `2d49019e9` + +- Date: 2026-02-13 +- Change: Added HTTP write error-path integration coverage. +- APIs covered: `POST` invalid vid/fid parse rejections (`400`), malformed multipart form parse failure (`400`), and `Content-MD5` mismatch validation failure (`400`). +- Profiles covered: P1. +- Gaps introduced/remaining: file-size limit rejection and replicated-write failure branches remain pending for write-path breadth. +- Commit: `046390e54` + +- Date: 2026-02-13 +- Change: Added HTTP conditional-header precedence coverage. +- APIs covered: `GET` with combined `If-Modified-Since` + mismatched `If-None-Match` (`304` by current precedence), plus invalid `If-Modified-Since` fallback behavior (`200` body path). +- Profiles covered: P1. +- Gaps introduced/remaining: range-sum oversized-request behavior and image transform/chunk-manifest read branches remain pending. +- Commit: `31d59f0b8` + +- Date: 2026-02-13 +- Change: Added HTTP oversized-multi-range guard coverage. +- APIs covered: `GET` `Range` requests where combined ranges exceed total object size (`sumRangesSize > totalSize`) with current parity response (`200` + empty body). +- Profiles covered: P1. +- Gaps introduced/remaining: image resize/crop and chunk-manifest read branches remain pending. +- Commit: `612e5f61c` + +- Date: 2026-02-13 +- Change: Added HTTP image transformation integration coverage. +- APIs covered: read-path image resize (`width`,`height`) and crop (`crop_*`) branches with decoded output-dimension assertions. +- Profiles covered: P1. +- Gaps introduced/remaining: chunk-manifest read/`cm=false` parity and compressed-content encoding matrix remain pending. +- Commit: `d68803ecc` + +- Date: 2026-02-13 +- Change: Added HTTP chunk-manifest integration coverage for expansion and bypass. +- APIs covered: chunk-manifest auto-expansion read path (`X-File-Store: chunked`) and `cm=false` raw-manifest bypass parity. +- Profiles covered: P1. +- Gaps introduced/remaining: compressed-content encoding matrix and some write/delete failure-injection branches remain pending. +- Commit: `8ecf427c4` + +- Date: 2026-02-13 +- Change: Added HTTP compressed-read encoding matrix coverage. +- APIs covered: compressed needle read branch with `Accept-Encoding=gzip` passthrough (`Content-Encoding: gzip`) and `Accept-Encoding=identity` decompression parity. +- Profiles covered: P1. +- Gaps introduced/remaining: write/delete failure-injection and request-cancel (`499`) paths remain pending due transport/fault observability constraints. +- Commit: `e17604c57` + +- Date: 2026-02-13 +- Change: Added dual-node gRPC tail receiver success-path coverage. +- APIs covered: `VolumeTailReceiver` end-to-end replication from source server stream into destination volume, with post-tail HTTP payload parity verification. +- Profiles covered: P4-derived dual-volume topology. +- Gaps introduced/remaining: sender large-needle chunking specifics and transport-interruption branches remain pending. +- Commit: `3989bc6e5` + +- Date: 2026-02-13 +- Change: Added gRPC tail sender large-needle chunking coverage. +- APIs covered: `VolumeTailSender` stream chunk-splitting path for oversized needle bodies (multiple chunks with non-last and final `IsLastChunk=true` markers). +- Profiles covered: P1. +- Gaps introduced/remaining: sender/receiver transport interruption branches remain pending. +- Commit: `514c05131` + +- Date: 2026-02-13 +- Change: Added EC-backed `VolumeNeedleStatus` integration coverage. +- APIs covered: `VolumeNeedleStatus` EC execution path (normal volume unmounted + EC shards mounted), including missing-needle error behavior. +- Profiles covered: P1. +- Gaps introduced/remaining: transport-interruption and deep fault-injection branches remain pending. +- Commit: `4ef666791` + +- Date: 2026-02-13 +- Change: Added dual-node EC shard copy success coverage. +- APIs covered: `VolumeEcShardsCopy` positive path from source peer (including `copy_ecx_file`/`copy_vif_file`) with copied artifact verification via EC `CopyFile` on destination. +- Profiles covered: P4-derived dual-volume topology. +- Gaps introduced/remaining: EC rebuild/copy distributed failure injection permutations remain pending. +- Commit: `c9f6710c2` + +- Date: 2026-02-13 +- Change: Added HTTP chunk-manifest delete cleanup coverage. +- APIs covered: chunk-manifest delete success path with child-chunk deletion verification and expected delete response size parity. +- Profiles covered: P1. +- Gaps introduced/remaining: chunk-manifest delete failure-injection branch remains pending. +- Commit: `c6df98a02` + +- Date: 2026-02-13 +- Change: Added HTTP chunk-manifest delete failure-path coverage. +- APIs covered: chunk-manifest child-delete error branch (`500`) and non-deletion parity of manifest metadata after failed delete. +- Profiles covered: P1. +- Gaps introduced/remaining: request-cancel (`499`) and deep transport-interruption branches remain pending. +- Commit: `38a1f4f4f` + +- Date: 2026-02-13 +- Change: Added EC shard-copy source-unavailable error coverage. +- APIs covered: `VolumeEcShardsCopy` network/source-unreachable failure path with wrapped RPC error propagation. +- Profiles covered: P1. +- Gaps introduced/remaining: `499` cancellation and transport-interruption branches remain pending. +- Commit: `d1e5f390a` diff --git a/test/volume_server/Makefile b/test/volume_server/Makefile new file mode 100644 index 000000000..1801734a9 --- /dev/null +++ b/test/volume_server/Makefile @@ -0,0 +1,7 @@ +.PHONY: test-volume-server test-volume-server-short + +test-volume-server: + go test ./test/volume_server/... -v + +test-volume-server-short: + go test ./test/volume_server/... -short -v diff --git a/test/volume_server/README.md b/test/volume_server/README.md new file mode 100644 index 000000000..1fed73522 --- /dev/null +++ b/test/volume_server/README.md @@ -0,0 +1,27 @@ +# Volume Server Integration Tests + +This package contains integration tests for SeaweedFS volume server HTTP and gRPC APIs. + +## Run Tests + +Run tests from repo root: + +```bash +go test ./test/volume_server/... -v +``` + +If a `weed` binary is not found, the harness will build one automatically. + +## Optional environment variables + +- `WEED_BINARY`: explicit path to the `weed` executable (disables auto-build). +- `VOLUME_SERVER_IT_KEEP_LOGS=1`: keep temporary test directories and process logs. + +## Current scope (Phase 0) + +- Shared cluster/framework utilities +- Matrix profile definitions +- Initial HTTP admin endpoint checks +- Initial gRPC state/status checks + +More API coverage is tracked in `/Users/chris/dev/seaweedfs2/test/volume_server/DEV_PLAN.md`. diff --git a/test/volume_server/framework/cluster.go b/test/volume_server/framework/cluster.go new file mode 100644 index 000000000..4bb1b55d5 --- /dev/null +++ b/test/volume_server/framework/cluster.go @@ -0,0 +1,442 @@ +package framework + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +const ( + defaultWaitTimeout = 30 * time.Second + defaultWaitTick = 200 * time.Millisecond + testVolumeSizeLimitMB = 32 +) + +// Cluster is a lightweight SeaweedFS master + one volume server test harness. +type Cluster struct { + testingTB testing.TB + profile matrix.Profile + + weedBinary string + baseDir string + configDir string + logsDir string + keepLogs bool + + masterPort int + masterGrpcPort int + volumePort int + volumeGrpcPort int + volumePubPort int + + masterCmd *exec.Cmd + volumeCmd *exec.Cmd + + cleanupOnce sync.Once +} + +// StartSingleVolumeCluster boots one master and one volume server. +func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster { + t.Helper() + + weedBinary, err := FindOrBuildWeedBinary() + if err != nil { + t.Fatalf("resolve weed binary: %v", err) + } + + baseDir, keepLogs, err := newWorkDir() + if err != nil { + t.Fatalf("create temp test directory: %v", err) + } + + configDir := filepath.Join(baseDir, "config") + logsDir := filepath.Join(baseDir, "logs") + masterDataDir := filepath.Join(baseDir, "master") + volumeDataDir := filepath.Join(baseDir, "volume") + for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir} { + if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil { + t.Fatalf("create %s: %v", dir, mkErr) + } + } + + if err = writeSecurityConfig(configDir, profile); err != nil { + t.Fatalf("write security config: %v", err) + } + + masterPort, masterGrpcPort, err := allocateMasterPortPair() + if err != nil { + t.Fatalf("allocate master port pair: %v", err) + } + + ports, err := allocatePorts(3) + if err != nil { + t.Fatalf("allocate ports: %v", err) + } + + c := &Cluster{ + testingTB: t, + profile: profile, + weedBinary: weedBinary, + baseDir: baseDir, + configDir: configDir, + logsDir: logsDir, + keepLogs: keepLogs, + masterPort: masterPort, + masterGrpcPort: masterGrpcPort, + volumePort: ports[0], + volumeGrpcPort: ports[1], + volumePubPort: ports[0], + } + if profile.SplitPublicPort { + c.volumePubPort = ports[2] + } + + if err = c.startMaster(masterDataDir); err != nil { + c.Stop() + t.Fatalf("start master: %v", err) + } + if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil { + masterLog := c.tailLog("master.log") + c.Stop() + t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog) + } + + if err = c.startVolume(volumeDataDir); err != nil { + masterLog := c.tailLog("master.log") + c.Stop() + t.Fatalf("start volume: %v\nmaster log tail:\n%s", err, masterLog) + } + if err = c.waitForHTTP(c.VolumeAdminURL() + "/status"); err != nil { + volumeLog := c.tailLog("volume.log") + c.Stop() + t.Fatalf("wait for volume readiness: %v\nvolume log tail:\n%s", err, volumeLog) + } + if err = c.waitForTCP(c.VolumeGRPCAddress()); err != nil { + volumeLog := c.tailLog("volume.log") + c.Stop() + t.Fatalf("wait for volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog) + } + + t.Cleanup(func() { + c.Stop() + }) + + return c +} + +// Stop terminates all processes and cleans temporary files. +func (c *Cluster) Stop() { + if c == nil { + return + } + c.cleanupOnce.Do(func() { + stopProcess(c.volumeCmd) + stopProcess(c.masterCmd) + if !c.keepLogs && !c.testingTB.Failed() { + _ = os.RemoveAll(c.baseDir) + } else if c.baseDir != "" { + c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir) + } + }) +} + +func (c *Cluster) startMaster(dataDir string) error { + logFile, err := os.Create(filepath.Join(c.logsDir, "master.log")) + if err != nil { + return err + } + + args := []string{ + "-config_dir=" + c.configDir, + "master", + "-ip=127.0.0.1", + "-port=" + strconv.Itoa(c.masterPort), + "-port.grpc=" + strconv.Itoa(c.masterGrpcPort), + "-mdir=" + dataDir, + "-peers=none", + "-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB), + "-defaultReplication=000", + } + + c.masterCmd = exec.Command(c.weedBinary, args...) + c.masterCmd.Dir = c.baseDir + c.masterCmd.Stdout = logFile + c.masterCmd.Stderr = logFile + return c.masterCmd.Start() +} + +func (c *Cluster) startVolume(dataDir string) error { + logFile, err := os.Create(filepath.Join(c.logsDir, "volume.log")) + if err != nil { + return err + } + + args := []string{ + "-config_dir=" + c.configDir, + "volume", + "-ip=127.0.0.1", + "-port=" + strconv.Itoa(c.volumePort), + "-port.grpc=" + strconv.Itoa(c.volumeGrpcPort), + "-port.public=" + strconv.Itoa(c.volumePubPort), + "-dir=" + dataDir, + "-max=16", + "-master=127.0.0.1:" + strconv.Itoa(c.masterPort), + "-readMode=" + c.profile.ReadMode, + "-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB), + "-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB), + } + if c.profile.InflightUploadTimeout > 0 { + args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String()) + } + if c.profile.InflightDownloadTimeout > 0 { + args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String()) + } + + c.volumeCmd = exec.Command(c.weedBinary, args...) + c.volumeCmd.Dir = c.baseDir + c.volumeCmd.Stdout = logFile + c.volumeCmd.Stderr = logFile + return c.volumeCmd.Start() +} + +func (c *Cluster) waitForHTTP(url string) error { + client := &http.Client{Timeout: 1 * time.Second} + deadline := time.Now().Add(defaultWaitTimeout) + for time.Now().Before(deadline) { + resp, err := client.Get(url) + if err == nil { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.StatusCode < 500 { + return nil + } + } + time.Sleep(defaultWaitTick) + } + return fmt.Errorf("timed out waiting for %s", url) +} + +func (c *Cluster) waitForTCP(addr string) error { + deadline := time.Now().Add(defaultWaitTimeout) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", addr, time.Second) + if err == nil { + _ = conn.Close() + return nil + } + time.Sleep(defaultWaitTick) + } + return fmt.Errorf("timed out waiting for tcp %s", addr) +} + +func stopProcess(cmd *exec.Cmd) { + if cmd == nil || cmd.Process == nil { + return + } + + _ = cmd.Process.Signal(os.Interrupt) + done := make(chan error, 1) + go func() { + done <- cmd.Wait() + }() + + select { + case <-time.After(10 * time.Second): + _ = cmd.Process.Kill() + <-done + case <-done: + } +} + +func allocatePorts(count int) ([]int, error) { + listeners := make([]net.Listener, 0, count) + ports := make([]int, 0, count) + for i := 0; i < count; i++ { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + for _, ll := range listeners { + _ = ll.Close() + } + return nil, err + } + listeners = append(listeners, l) + ports = append(ports, l.Addr().(*net.TCPAddr).Port) + } + for _, l := range listeners { + _ = l.Close() + } + return ports, nil +} + +func allocateMasterPortPair() (int, int, error) { + for masterPort := 10000; masterPort <= 55535; masterPort++ { + masterGrpcPort := masterPort + 10000 + l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort))) + if err != nil { + continue + } + l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterGrpcPort))) + if err != nil { + _ = l1.Close() + continue + } + _ = l2.Close() + _ = l1.Close() + return masterPort, masterGrpcPort, nil + } + return 0, 0, errors.New("unable to find available master port pair") +} + +func newWorkDir() (dir string, keepLogs bool, err error) { + keepLogs = os.Getenv("VOLUME_SERVER_IT_KEEP_LOGS") == "1" + dir, err = os.MkdirTemp("", "seaweedfs_volume_server_it_") + return dir, keepLogs, err +} + +func writeSecurityConfig(configDir string, profile matrix.Profile) error { + var b strings.Builder + if profile.EnableJWT { + if profile.JWTSigningKey == "" || profile.JWTReadKey == "" { + return errors.New("jwt profile requires both write and read keys") + } + b.WriteString("[jwt.signing]\n") + b.WriteString("key = \"") + b.WriteString(profile.JWTSigningKey) + b.WriteString("\"\n") + b.WriteString("expires_after_seconds = 60\n\n") + + b.WriteString("[jwt.signing.read]\n") + b.WriteString("key = \"") + b.WriteString(profile.JWTReadKey) + b.WriteString("\"\n") + b.WriteString("expires_after_seconds = 60\n") + } + if b.Len() == 0 { + b.WriteString("# optional security config generated for integration tests\n") + } + return os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(b.String()), 0o644) +} + +// FindOrBuildWeedBinary returns an executable weed binary, building one when needed. +func FindOrBuildWeedBinary() (string, error) { + if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" { + if isExecutableFile(fromEnv) { + return fromEnv, nil + } + return "", fmt.Errorf("WEED_BINARY is set but not executable: %s", fromEnv) + } + + repoRoot := "" + if _, file, _, ok := runtime.Caller(0); ok { + repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..")) + candidate := filepath.Join(repoRoot, "weed", "weed") + if isExecutableFile(candidate) { + return candidate, nil + } + } + + if repoRoot == "" { + return "", errors.New("unable to detect repository root") + } + + binDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_bin") + if err := os.MkdirAll(binDir, 0o755); err != nil { + return "", fmt.Errorf("create binary directory %s: %w", binDir, err) + } + binPath := filepath.Join(binDir, "weed") + if isExecutableFile(binPath) { + return binPath, nil + } + + cmd := exec.Command("go", "build", "-o", binPath, ".") + cmd.Dir = filepath.Join(repoRoot, "weed") + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("build weed binary: %w\n%s", err, out.String()) + } + if !isExecutableFile(binPath) { + return "", fmt.Errorf("built weed binary is not executable: %s", binPath) + } + return binPath, nil +} + +func isExecutableFile(path string) bool { + info, err := os.Stat(path) + if err != nil || info.IsDir() { + return false + } + mode := info.Mode().Perm() + return mode&0o111 != 0 +} + +func (c *Cluster) tailLog(logName string) string { + f, err := os.Open(filepath.Join(c.logsDir, logName)) + if err != nil { + return "" + } + defer f.Close() + + scanner := bufio.NewScanner(f) + lines := make([]string, 0, 40) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + if len(lines) > 40 { + lines = lines[1:] + } + } + return strings.Join(lines, "\n") +} + +func (c *Cluster) MasterAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort)) +} + +func (c *Cluster) VolumeAdminAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort)) +} + +func (c *Cluster) VolumePublicAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort)) +} + +func (c *Cluster) VolumeGRPCAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort)) +} + +// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort +func (c *Cluster) VolumeServerAddress() string { + return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(), c.volumeGrpcPort) +} + +func (c *Cluster) MasterURL() string { + return "http://" + c.MasterAddress() +} + +func (c *Cluster) VolumeAdminURL() string { + return "http://" + c.VolumeAdminAddress() +} + +func (c *Cluster) VolumePublicURL() string { + return "http://" + c.VolumePublicAddress() +} + +func (c *Cluster) BaseDir() string { + return c.baseDir +} diff --git a/test/volume_server/framework/cluster_dual.go b/test/volume_server/framework/cluster_dual.go new file mode 100644 index 000000000..ffa28c75b --- /dev/null +++ b/test/volume_server/framework/cluster_dual.go @@ -0,0 +1,293 @@ +package framework + +import ( + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "sync" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +type DualVolumeCluster struct { + testingTB testing.TB + profile matrix.Profile + + weedBinary string + baseDir string + configDir string + logsDir string + keepLogs bool + + masterPort int + masterGrpcPort int + + volumePort0 int + volumeGrpcPort0 int + volumePubPort0 int + volumePort1 int + volumeGrpcPort1 int + volumePubPort1 int + + masterCmd *exec.Cmd + volumeCmd0 *exec.Cmd + volumeCmd1 *exec.Cmd + + cleanupOnce sync.Once +} + +func StartDualVolumeCluster(t testing.TB, profile matrix.Profile) *DualVolumeCluster { + t.Helper() + + weedBinary, err := FindOrBuildWeedBinary() + if err != nil { + t.Fatalf("resolve weed binary: %v", err) + } + + baseDir, keepLogs, err := newWorkDir() + if err != nil { + t.Fatalf("create temp test directory: %v", err) + } + + configDir := filepath.Join(baseDir, "config") + logsDir := filepath.Join(baseDir, "logs") + masterDataDir := filepath.Join(baseDir, "master") + volumeDataDir0 := filepath.Join(baseDir, "volume0") + volumeDataDir1 := filepath.Join(baseDir, "volume1") + for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir0, volumeDataDir1} { + if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil { + t.Fatalf("create %s: %v", dir, mkErr) + } + } + + if err = writeSecurityConfig(configDir, profile); err != nil { + t.Fatalf("write security config: %v", err) + } + + masterPort, masterGrpcPort, err := allocateMasterPortPair() + if err != nil { + t.Fatalf("allocate master port pair: %v", err) + } + + ports, err := allocatePorts(6) + if err != nil { + t.Fatalf("allocate volume ports: %v", err) + } + + c := &DualVolumeCluster{ + testingTB: t, + profile: profile, + weedBinary: weedBinary, + baseDir: baseDir, + configDir: configDir, + logsDir: logsDir, + keepLogs: keepLogs, + masterPort: masterPort, + masterGrpcPort: masterGrpcPort, + volumePort0: ports[0], + volumeGrpcPort0: ports[1], + volumePubPort0: ports[0], + volumePort1: ports[2], + volumeGrpcPort1: ports[3], + volumePubPort1: ports[2], + } + if profile.SplitPublicPort { + c.volumePubPort0 = ports[4] + c.volumePubPort1 = ports[5] + } + + if err = c.startMaster(masterDataDir); err != nil { + c.Stop() + t.Fatalf("start master: %v", err) + } + if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil { + masterLog := c.tailLog("master.log") + c.Stop() + t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog) + } + + if err = c.startVolume(0, volumeDataDir0); err != nil { + masterLog := c.tailLog("master.log") + c.Stop() + t.Fatalf("start first volume server: %v\nmaster log tail:\n%s", err, masterLog) + } + if err = c.waitForHTTP(c.VolumeAdminURL(0) + "/status"); err != nil { + volumeLog := c.tailLog("volume0.log") + c.Stop() + t.Fatalf("wait for first volume readiness: %v\nvolume log tail:\n%s", err, volumeLog) + } + if err = c.waitForTCP(c.VolumeGRPCAddress(0)); err != nil { + volumeLog := c.tailLog("volume0.log") + c.Stop() + t.Fatalf("wait for first volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog) + } + + if err = c.startVolume(1, volumeDataDir1); err != nil { + volumeLog := c.tailLog("volume0.log") + c.Stop() + t.Fatalf("start second volume server: %v\nfirst volume log tail:\n%s", err, volumeLog) + } + if err = c.waitForHTTP(c.VolumeAdminURL(1) + "/status"); err != nil { + volumeLog := c.tailLog("volume1.log") + c.Stop() + t.Fatalf("wait for second volume readiness: %v\nvolume log tail:\n%s", err, volumeLog) + } + if err = c.waitForTCP(c.VolumeGRPCAddress(1)); err != nil { + volumeLog := c.tailLog("volume1.log") + c.Stop() + t.Fatalf("wait for second volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog) + } + + t.Cleanup(func() { + c.Stop() + }) + + return c +} + +func (c *DualVolumeCluster) Stop() { + if c == nil { + return + } + c.cleanupOnce.Do(func() { + stopProcess(c.volumeCmd1) + stopProcess(c.volumeCmd0) + stopProcess(c.masterCmd) + if !c.keepLogs && !c.testingTB.Failed() { + _ = os.RemoveAll(c.baseDir) + } else if c.baseDir != "" { + c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir) + } + }) +} + +func (c *DualVolumeCluster) startMaster(dataDir string) error { + logFile, err := os.Create(filepath.Join(c.logsDir, "master.log")) + if err != nil { + return err + } + + args := []string{ + "-config_dir=" + c.configDir, + "master", + "-ip=127.0.0.1", + "-port=" + strconv.Itoa(c.masterPort), + "-port.grpc=" + strconv.Itoa(c.masterGrpcPort), + "-mdir=" + dataDir, + "-peers=none", + "-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB), + "-defaultReplication=000", + } + + c.masterCmd = exec.Command(c.weedBinary, args...) + c.masterCmd.Dir = c.baseDir + c.masterCmd.Stdout = logFile + c.masterCmd.Stderr = logFile + return c.masterCmd.Start() +} + +func (c *DualVolumeCluster) startVolume(index int, dataDir string) error { + logName := fmt.Sprintf("volume%d.log", index) + logFile, err := os.Create(filepath.Join(c.logsDir, logName)) + if err != nil { + return err + } + + volumePort := c.volumePort0 + volumeGrpcPort := c.volumeGrpcPort0 + volumePubPort := c.volumePubPort0 + if index == 1 { + volumePort = c.volumePort1 + volumeGrpcPort = c.volumeGrpcPort1 + volumePubPort = c.volumePubPort1 + } + + args := []string{ + "-config_dir=" + c.configDir, + "volume", + "-ip=127.0.0.1", + "-port=" + strconv.Itoa(volumePort), + "-port.grpc=" + strconv.Itoa(volumeGrpcPort), + "-port.public=" + strconv.Itoa(volumePubPort), + "-dir=" + dataDir, + "-max=16", + "-master=127.0.0.1:" + strconv.Itoa(c.masterPort), + "-readMode=" + c.profile.ReadMode, + "-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB), + "-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB), + } + if c.profile.InflightUploadTimeout > 0 { + args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String()) + } + if c.profile.InflightDownloadTimeout > 0 { + args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String()) + } + + cmd := exec.Command(c.weedBinary, args...) + cmd.Dir = c.baseDir + cmd.Stdout = logFile + cmd.Stderr = logFile + + if err = cmd.Start(); err != nil { + return err + } + if index == 1 { + c.volumeCmd1 = cmd + } else { + c.volumeCmd0 = cmd + } + return nil +} + +func (c *DualVolumeCluster) waitForHTTP(url string) error { + return (&Cluster{}).waitForHTTP(url) +} + +func (c *DualVolumeCluster) waitForTCP(addr string) error { + return (&Cluster{}).waitForTCP(addr) +} + +func (c *DualVolumeCluster) tailLog(logName string) string { + return (&Cluster{logsDir: c.logsDir}).tailLog(logName) +} + +func (c *DualVolumeCluster) MasterAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort)) +} + +func (c *DualVolumeCluster) MasterURL() string { + return "http://" + c.MasterAddress() +} + +func (c *DualVolumeCluster) VolumeAdminAddress(index int) string { + if index == 1 { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort1)) + } + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort0)) +} + +func (c *DualVolumeCluster) VolumePublicAddress(index int) string { + if index == 1 { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort1)) + } + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort0)) +} + +func (c *DualVolumeCluster) VolumeGRPCAddress(index int) string { + if index == 1 { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort1)) + } + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort0)) +} + +func (c *DualVolumeCluster) VolumeAdminURL(index int) string { + return "http://" + c.VolumeAdminAddress(index) +} + +func (c *DualVolumeCluster) VolumePublicURL(index int) string { + return "http://" + c.VolumePublicAddress(index) +} diff --git a/test/volume_server/framework/cluster_with_filer.go b/test/volume_server/framework/cluster_with_filer.go new file mode 100644 index 000000000..67f35fd26 --- /dev/null +++ b/test/volume_server/framework/cluster_with_filer.go @@ -0,0 +1,91 @@ +package framework + +import ( + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +type ClusterWithFiler struct { + *Cluster + + filerCmd *exec.Cmd + filerPort int + filerGrpcPort int +} + +func StartSingleVolumeClusterWithFiler(t testing.TB, profile matrix.Profile) *ClusterWithFiler { + t.Helper() + + baseCluster := StartSingleVolumeCluster(t, profile) + + ports, err := allocatePorts(2) + if err != nil { + t.Fatalf("allocate filer ports: %v", err) + } + + filerDataDir := filepath.Join(baseCluster.baseDir, "filer") + if mkErr := os.MkdirAll(filerDataDir, 0o755); mkErr != nil { + t.Fatalf("create filer data dir: %v", mkErr) + } + + logFile, err := os.Create(filepath.Join(baseCluster.logsDir, "filer.log")) + if err != nil { + t.Fatalf("create filer log file: %v", err) + } + + filerPort := ports[0] + filerGrpcPort := ports[1] + args := []string{ + "-config_dir=" + baseCluster.configDir, + "filer", + "-master=127.0.0.1:" + strconv.Itoa(baseCluster.masterPort), + "-ip=127.0.0.1", + "-port=" + strconv.Itoa(filerPort), + "-port.grpc=" + strconv.Itoa(filerGrpcPort), + "-defaultStoreDir=" + filerDataDir, + } + + filerCmd := exec.Command(baseCluster.weedBinary, args...) + filerCmd.Dir = baseCluster.baseDir + filerCmd.Stdout = logFile + filerCmd.Stderr = logFile + if err = filerCmd.Start(); err != nil { + t.Fatalf("start filer: %v", err) + } + + if err = baseCluster.waitForTCP(net.JoinHostPort("127.0.0.1", strconv.Itoa(filerGrpcPort))); err != nil { + filerLogTail := baseCluster.tailLog("filer.log") + stopProcess(filerCmd) + t.Fatalf("wait for filer grpc readiness: %v\nfiler log tail:\n%s", err, filerLogTail) + } + + t.Cleanup(func() { + stopProcess(filerCmd) + }) + + return &ClusterWithFiler{ + Cluster: baseCluster, + filerCmd: filerCmd, + filerPort: filerPort, + filerGrpcPort: filerGrpcPort, + } +} + +func (c *ClusterWithFiler) FilerAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerPort)) +} + +func (c *ClusterWithFiler) FilerGRPCAddress() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerGrpcPort)) +} + +func (c *ClusterWithFiler) FilerServerAddress() string { + return fmt.Sprintf("%s.%d", c.FilerAddress(), c.filerGrpcPort) +} diff --git a/test/volume_server/framework/fault_injection.go b/test/volume_server/framework/fault_injection.go new file mode 100644 index 000000000..4a711a340 --- /dev/null +++ b/test/volume_server/framework/fault_injection.go @@ -0,0 +1,8 @@ +package framework + +// Phase 0 placeholder for future fault injection utilities. +// +// Planned extensions: +// - restart/kill selected processes +// - temporary network isolation hooks +// - master or peer outage helpers for proxy/replication branch coverage diff --git a/test/volume_server/framework/grpc_client.go b/test/volume_server/framework/grpc_client.go new file mode 100644 index 000000000..bf4d95182 --- /dev/null +++ b/test/volume_server/framework/grpc_client.go @@ -0,0 +1,28 @@ +package framework + +import ( + "context" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func DialVolumeServer(t testing.TB, address string) (*grpc.ClientConn, volume_server_pb.VolumeServerClient) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + conn, err := grpc.DialContext(ctx, address, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithBlock(), + ) + if err != nil { + t.Fatalf("dial volume grpc %s: %v", address, err) + } + + return conn, volume_server_pb.NewVolumeServerClient(conn) +} diff --git a/test/volume_server/framework/http_client.go b/test/volume_server/framework/http_client.go new file mode 100644 index 000000000..816b64720 --- /dev/null +++ b/test/volume_server/framework/http_client.go @@ -0,0 +1,34 @@ +package framework + +import ( + "io" + "net/http" + "testing" + "time" +) + +func NewHTTPClient() *http.Client { + return &http.Client{Timeout: 10 * time.Second} +} + +func DoRequest(t testing.TB, client *http.Client, req *http.Request) *http.Response { + t.Helper() + resp, err := client.Do(req) + if err != nil { + t.Fatalf("http request %s %s: %v", req.Method, req.URL.String(), err) + } + return resp +} + +func ReadAllAndClose(t testing.TB, resp *http.Response) []byte { + t.Helper() + if resp == nil { + return nil + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read response body: %v", err) + } + return body +} diff --git a/test/volume_server/framework/volume_fixture.go b/test/volume_server/framework/volume_fixture.go new file mode 100644 index 000000000..f6229f3f8 --- /dev/null +++ b/test/volume_server/framework/volume_fixture.go @@ -0,0 +1,56 @@ +package framework + +import ( + "bytes" + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" +) + +func AllocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{ + VolumeId: volumeID, + Collection: collection, + Replication: "000", + Version: uint32(needle.GetCurrentVersion()), + }) + if err != nil { + t.Fatalf("allocate volume %d: %v", volumeID, err) + } +} + +func NewFileID(volumeID uint32, key uint64, cookie uint32) string { + return needle.NewFileId(needle.VolumeId(volumeID), key, cookie).String() +} + +func UploadBytes(t testing.TB, client *http.Client, volumeURL, fid string, data []byte) *http.Response { + t.Helper() + + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/%s", volumeURL, fid), bytes.NewReader(data)) + if err != nil { + t.Fatalf("build upload request: %v", err) + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Length", fmt.Sprintf("%d", len(data))) + return DoRequest(t, client, req) +} + +func ReadBytes(t testing.TB, client *http.Client, volumeURL, fid string) *http.Response { + t.Helper() + + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", volumeURL, fid), nil) + if err != nil { + t.Fatalf("build read request: %v", err) + } + return DoRequest(t, client, req) +} diff --git a/test/volume_server/grpc/admin_extra_test.go b/test/volume_server/grpc/admin_extra_test.go new file mode 100644 index 000000000..de62fcdb8 --- /dev/null +++ b/test/volume_server/grpc/admin_extra_test.go @@ -0,0 +1,445 @@ +package volume_server_grpc_test + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/cluster" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestVolumeNeedleStatusForUploadedFile(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(21) + const needleID = uint64(778899) + const cookie = uint32(0xA1B2C3D4) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + client := framework.NewHTTPClient() + payload := []byte("needle-status-payload") + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: volumeID, + NeedleId: needleID, + }) + if err != nil { + t.Fatalf("VolumeNeedleStatus failed: %v", err) + } + if statusResp.GetNeedleId() != needleID { + t.Fatalf("needle id mismatch: got %d want %d", statusResp.GetNeedleId(), needleID) + } + if statusResp.GetCookie() != cookie { + t.Fatalf("cookie mismatch: got %d want %d", statusResp.GetCookie(), cookie) + } + if statusResp.GetSize() == 0 { + t.Fatalf("expected non-zero needle size") + } +} + +func TestVolumeNeedleStatusViaEcShardsWhenNormalVolumeUnmounted(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(26) + const needleID = uint64(778900) + const cookie = uint32(0xA1B2C3D5) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, cookie) + payload := []byte("needle-status-ec-path-payload") + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount data shards failed: %v", err) + } + + _, err = grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{ + VolumeId: volumeID, + }) + if err != nil { + t.Fatalf("VolumeUnmount failed: %v", err) + } + + statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: volumeID, + NeedleId: needleID, + }) + if err != nil { + t.Fatalf("VolumeNeedleStatus via EC shards failed: %v", err) + } + if statusResp.GetNeedleId() != needleID { + t.Fatalf("needle id mismatch: got %d want %d", statusResp.GetNeedleId(), needleID) + } + if statusResp.GetCookie() != cookie { + t.Fatalf("cookie mismatch: got %d want %d", statusResp.GetCookie(), cookie) + } + if statusResp.GetSize() == 0 { + t.Fatalf("expected non-zero needle size from EC-backed needle status") + } + + _, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: volumeID, + NeedleId: needleID + 999999, + }) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "not found") { + t.Fatalf("VolumeNeedleStatus via EC shards missing-needle error mismatch: %v", err) + } +} + +func TestVolumeNeedleStatusMissingVolumeAndNeedle(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(25) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: 99925, + NeedleId: 1, + }) + if err == nil { + t.Fatalf("VolumeNeedleStatus should fail for missing volume") + } + if !strings.Contains(strings.ToLower(err.Error()), "volume not found") { + t.Fatalf("VolumeNeedleStatus missing-volume error mismatch: %v", err) + } + + _, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: volumeID, + NeedleId: 123456789, + }) + if err == nil { + t.Fatalf("VolumeNeedleStatus should fail for missing needle") + } + if !strings.Contains(strings.ToLower(err.Error()), "not found") { + t.Fatalf("VolumeNeedleStatus missing-needle error mismatch: %v", err) + } +} + +func mustNewRequest(t testing.TB, method, url string) *http.Request { + t.Helper() + req, err := http.NewRequest(method, url, nil) + if err != nil { + t.Fatalf("create request %s %s: %v", method, url, err) + } + return req +} + +func TestVolumeConfigureInvalidReplication(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(22) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{ + VolumeId: volumeID, + Replication: "bad-replication", + }) + if err != nil { + t.Fatalf("VolumeConfigure returned grpc error: %v", err) + } + if resp.GetError() == "" { + t.Fatalf("VolumeConfigure expected response error for invalid replication") + } + if !strings.Contains(strings.ToLower(resp.GetError()), "replication") { + t.Fatalf("VolumeConfigure error should mention replication, got: %q", resp.GetError()) + } +} + +func TestVolumeConfigureSuccessAndMissingRollbackPath(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(24) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + successResp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{ + VolumeId: volumeID, + Replication: "000", + }) + if err != nil { + t.Fatalf("VolumeConfigure success path returned grpc error: %v", err) + } + if successResp.GetError() != "" { + t.Fatalf("VolumeConfigure success path expected empty response error, got: %q", successResp.GetError()) + } + + statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeStatus after successful configure failed: %v", err) + } + if statusResp.GetIsReadOnly() { + t.Fatalf("VolumeStatus after configure expected writable volume") + } + + missingResp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{ + VolumeId: 99024, + Replication: "000", + }) + if err != nil { + t.Fatalf("VolumeConfigure missing-volume branch should return response error, got grpc error: %v", err) + } + if missingResp.GetError() == "" { + t.Fatalf("VolumeConfigure missing-volume expected non-empty response error") + } + lower := strings.ToLower(missingResp.GetError()) + if !strings.Contains(lower, "not found on disk") { + t.Fatalf("VolumeConfigure missing-volume error should mention not found on disk, got: %q", missingResp.GetError()) + } + if !strings.Contains(lower, "failed to restore mount") { + t.Fatalf("VolumeConfigure missing-volume error should include remount rollback failure, got: %q", missingResp.GetError()) + } +} + +func TestPingVolumeTargetAndLeaveAffectsHealthz(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + pingResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ + TargetType: cluster.VolumeServerType, + Target: clusterHarness.VolumeServerAddress(), + }) + if err != nil { + t.Fatalf("Ping target volume server failed: %v", err) + } + if pingResp.GetRemoteTimeNs() == 0 { + t.Fatalf("expected remote timestamp from ping target volume server") + } + + if _, err = grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil { + t.Fatalf("VolumeServerLeave failed: %v", err) + } + + client := framework.NewHTTPClient() + healthURL := clusterHarness.VolumeAdminURL() + "/healthz" + deadline := time.Now().Add(5 * time.Second) + for { + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, healthURL)) + _ = framework.ReadAllAndClose(t, resp) + if resp.StatusCode == http.StatusServiceUnavailable { + return + } + if time.Now().After(deadline) { + t.Fatalf("expected healthz to return 503 after leave, got %d", resp.StatusCode) + } + time.Sleep(100 * time.Millisecond) + } +} + +func TestVolumeServerLeaveIsIdempotent(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if _, err := grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil { + t.Fatalf("first VolumeServerLeave failed: %v", err) + } + if _, err := grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil { + t.Fatalf("second VolumeServerLeave should be idempotent success, got: %v", err) + } + + client := framework.NewHTTPClient() + healthURL := clusterHarness.VolumeAdminURL() + "/healthz" + deadline := time.Now().Add(5 * time.Second) + for { + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, healthURL)) + _ = framework.ReadAllAndClose(t, resp) + if resp.StatusCode == http.StatusServiceUnavailable { + return + } + if time.Now().After(deadline) { + t.Fatalf("expected healthz to stay 503 after repeated leave, got %d", resp.StatusCode) + } + time.Sleep(100 * time.Millisecond) + } +} + +func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + unknownResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ + TargetType: "unknown-type", + Target: "127.0.0.1:12345", + }) + if err != nil { + t.Fatalf("Ping unknown target type should not return grpc error, got: %v", err) + } + if unknownResp.GetRemoteTimeNs() != 0 { + t.Fatalf("Ping unknown target type expected remote_time_ns=0, got %d", unknownResp.GetRemoteTimeNs()) + } + if unknownResp.GetStopTimeNs() < unknownResp.GetStartTimeNs() { + t.Fatalf("Ping unknown target type expected stop_time_ns >= start_time_ns") + } + + _, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ + TargetType: cluster.MasterType, + Target: "127.0.0.1:1", + }) + if err == nil { + t.Fatalf("Ping master target should fail when target is unreachable") + } + if !strings.Contains(err.Error(), "ping master") { + t.Fatalf("Ping master unreachable error mismatch: %v", err) + } + + _, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ + TargetType: cluster.FilerType, + Target: "127.0.0.1:1", + }) + if err == nil { + t.Fatalf("Ping filer target should fail when target is unreachable") + } + if !strings.Contains(err.Error(), "ping filer") { + t.Fatalf("Ping filer unreachable error mismatch: %v", err) + } +} + +func TestPingMasterTargetSuccess(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ + TargetType: cluster.MasterType, + Target: clusterHarness.MasterAddress(), + }) + if err != nil { + t.Fatalf("Ping master target success path failed: %v", err) + } + if resp.GetRemoteTimeNs() == 0 { + t.Fatalf("Ping master target expected non-zero remote time") + } + if resp.GetStopTimeNs() < resp.GetStartTimeNs() { + t.Fatalf("Ping master target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs()) + } +} + +func TestPingFilerTargetSuccess(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeClusterWithFiler(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ + TargetType: cluster.FilerType, + Target: clusterHarness.FilerServerAddress(), + }) + if err != nil { + t.Fatalf("Ping filer target success path failed: %v", err) + } + if resp.GetRemoteTimeNs() == 0 { + t.Fatalf("Ping filer target expected non-zero remote time") + } + if resp.GetStopTimeNs() < resp.GetStartTimeNs() { + t.Fatalf("Ping filer target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs()) + } +} diff --git a/test/volume_server/grpc/admin_lifecycle_test.go b/test/volume_server/grpc/admin_lifecycle_test.go new file mode 100644 index 000000000..bdc4e5a45 --- /dev/null +++ b/test/volume_server/grpc/admin_lifecycle_test.go @@ -0,0 +1,215 @@ +package volume_server_grpc_test + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestVolumeAdminLifecycleRPCs(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + const volumeID = uint32(11) + framework.AllocateVolume(t, client, volumeID, "") + + statusResp, err := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeStatus failed: %v", err) + } + if statusResp.GetFileCount() != 0 { + t.Fatalf("new volume should be empty, got file_count=%d", statusResp.GetFileCount()) + } + + if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil { + t.Fatalf("VolumeUnmount failed: %v", err) + } + if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil { + t.Fatalf("VolumeMount failed: %v", err) + } + + if _, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}); err != nil { + t.Fatalf("VolumeDelete failed: %v", err) + } + + _, err = client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err == nil { + t.Fatalf("VolumeStatus should fail after delete") + } + if st, ok := status.FromError(err); !ok || st.Code() == codes.OK { + t.Fatalf("VolumeStatus error should be a non-OK grpc status, got: %v", err) + } +} + +func TestVolumeDeleteOnlyEmptyVariants(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(13) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 66001, 0x11223344) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("volume-delete-only-empty")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}) + if err == nil || !strings.Contains(err.Error(), "volume not empty") { + t.Fatalf("VolumeDelete only_empty=true expected volume-not-empty error, got: %v", err) + } + + _, err = grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: false}) + if err != nil { + t.Fatalf("VolumeDelete only_empty=false failed: %v", err) + } + + _, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err == nil { + t.Fatalf("VolumeStatus should fail after non-empty delete with only_empty=false") + } +} + +func TestMaintenanceModeRejectsAllocateVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{VolumeId: 12, Replication: "000"}) + if err == nil { + t.Fatalf("AllocateVolume should fail when maintenance mode is enabled") + } + if !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("expected maintenance mode error, got: %v", err) + } +} + +func TestAllocateDuplicateAndMountUnmountMissingVariants(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + const missingVolumeID = uint32(99331) + const volumeID = uint32(14) + + if _, err := client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: missingVolumeID}); err != nil { + t.Fatalf("VolumeUnmount missing volume should be idempotent success, got: %v", err) + } + + _, err := client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: missingVolumeID}) + if err == nil { + t.Fatalf("VolumeMount missing volume should fail") + } + if !strings.Contains(err.Error(), "not found on disk") { + t.Fatalf("VolumeMount missing volume error mismatch: %v", err) + } + + framework.AllocateVolume(t, client, volumeID, "") + + _, err = client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{ + VolumeId: volumeID, + Replication: "000", + }) + if err == nil { + t.Fatalf("AllocateVolume duplicate should fail") + } + if !strings.Contains(strings.ToLower(err.Error()), "already exists") { + t.Fatalf("AllocateVolume duplicate error mismatch: %v", err) + } + + if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil { + t.Fatalf("VolumeUnmount existing volume failed: %v", err) + } + if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil { + t.Fatalf("VolumeUnmount already-unmounted volume should be idempotent success, got: %v", err) + } + if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil { + t.Fatalf("VolumeMount remount failed: %v", err) + } +} + +func TestMaintenanceModeRejectsVolumeDelete(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(15) + framework.AllocateVolume(t, client, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}) + if err == nil { + t.Fatalf("VolumeDelete should fail when maintenance mode is enabled") + } + if !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("expected maintenance mode error, got: %v", err) + } +} diff --git a/test/volume_server/grpc/admin_readonly_collection_test.go b/test/volume_server/grpc/admin_readonly_collection_test.go new file mode 100644 index 000000000..621309729 --- /dev/null +++ b/test/volume_server/grpc/admin_readonly_collection_test.go @@ -0,0 +1,177 @@ +package volume_server_grpc_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestVolumeMarkReadonlyAndWritableLifecycle(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(72) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{ + VolumeId: volumeID, + Persist: false, + }) + if err != nil { + t.Fatalf("VolumeMarkReadonly failed: %v", err) + } + + readOnlyStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeStatus after readonly failed: %v", err) + } + if !readOnlyStatus.GetIsReadOnly() { + t.Fatalf("VolumeStatus expected readonly=true after VolumeMarkReadonly") + } + + _, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeMarkWritable failed: %v", err) + } + + writableStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeStatus after writable failed: %v", err) + } + if writableStatus.GetIsReadOnly() { + t.Fatalf("VolumeStatus expected readonly=false after VolumeMarkWritable") + } +} + +func TestVolumeMarkReadonlyPersistTrue(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(74) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{ + VolumeId: volumeID, + Persist: true, + }) + if err != nil { + t.Fatalf("VolumeMarkReadonly persist=true failed: %v", err) + } + + statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeStatus after persist readonly failed: %v", err) + } + if !statusResp.GetIsReadOnly() { + t.Fatalf("VolumeStatus expected readonly=true after persist readonly") + } +} + +func TestVolumeMarkReadonlyWritableErrorPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 98771, Persist: true}) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeMarkReadonly missing-volume error mismatch: %v", err) + } + + _, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 98772}) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeMarkWritable missing-volume error mismatch: %v", err) + } + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: true, + Version: stateResp.GetState().GetVersion(), + }, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 1, Persist: true}) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeMarkReadonly maintenance error mismatch: %v", err) + } + + _, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 1}) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeMarkWritable maintenance error mismatch: %v", err) + } +} + +func TestDeleteCollectionRemovesVolumeAndIsIdempotent(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(73) + const collection = "it-delete-collection" + + framework.AllocateVolume(t, grpcClient, volumeID, collection) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeStatus before DeleteCollection failed: %v", err) + } + + _, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection}) + if err != nil { + t.Fatalf("DeleteCollection existing collection failed: %v", err) + } + + _, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) + if err == nil { + t.Fatalf("VolumeStatus should fail after collection delete") + } + if !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("VolumeStatus after DeleteCollection error mismatch: %v", err) + } + + _, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection}) + if err != nil { + t.Fatalf("DeleteCollection idempotent retry failed: %v", err) + } +} diff --git a/test/volume_server/grpc/batch_delete_test.go b/test/volume_server/grpc/batch_delete_test.go new file mode 100644 index 000000000..b02d4ea27 --- /dev/null +++ b/test/volume_server/grpc/batch_delete_test.go @@ -0,0 +1,264 @@ +package volume_server_grpc_test + +import ( + "bytes" + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestBatchDeleteInvalidFidAndMaintenanceMode(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"bad-fid"}}) + if err != nil { + t.Fatalf("BatchDelete invalid fid should return response, got error: %v", err) + } + if len(resp.GetResults()) != 1 { + t.Fatalf("expected one batch delete result, got %d", len(resp.GetResults())) + } + if got := resp.GetResults()[0].GetStatus(); got != 400 { + t.Fatalf("invalid fid expected status 400, got %d", got) + } + + stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"1,1234567890ab"}}) + if err == nil { + t.Fatalf("BatchDelete should fail when maintenance mode is enabled") + } + if !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("expected maintenance mode error, got: %v", err) + } +} + +func TestBatchDeleteCookieMismatchAndSkipCheck(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(31) + const needleID = uint64(900001) + const correctCookie = uint32(0x1122AABB) + const wrongCookie = uint32(0x1122AABC) + framework.AllocateVolume(t, client, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, correctCookie) + uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, []byte("batch-delete-cookie-check")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + wrongCookieFid := framework.NewFileID(volumeID, needleID, wrongCookie) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + mismatchResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{ + FileIds: []string{wrongCookieFid}, + SkipCookieCheck: false, + }) + if err != nil { + t.Fatalf("BatchDelete with cookie check failed: %v", err) + } + if len(mismatchResp.GetResults()) != 1 { + t.Fatalf("BatchDelete cookie mismatch expected 1 result, got %d", len(mismatchResp.GetResults())) + } + if mismatchResp.GetResults()[0].GetStatus() != http.StatusBadRequest { + t.Fatalf("BatchDelete cookie mismatch expected status 400, got %d", mismatchResp.GetResults()[0].GetStatus()) + } + + skipCheckResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{ + FileIds: []string{wrongCookieFid}, + SkipCookieCheck: true, + }) + if err != nil { + t.Fatalf("BatchDelete skip cookie check failed: %v", err) + } + if len(skipCheckResp.GetResults()) != 1 { + t.Fatalf("BatchDelete skip check expected 1 result, got %d", len(skipCheckResp.GetResults())) + } + if skipCheckResp.GetResults()[0].GetStatus() != http.StatusAccepted { + t.Fatalf("BatchDelete skip check expected status 202, got %d", skipCheckResp.GetResults()[0].GetStatus()) + } + + readAfterDelete := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, readAfterDelete) + if readAfterDelete.StatusCode != http.StatusNotFound { + t.Fatalf("read after skip-check batch delete expected 404, got %d", readAfterDelete.StatusCode) + } +} + +func TestBatchDeleteMixedStatusesAndMismatchStopsProcessing(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(32) + framework.AllocateVolume(t, client, volumeID, "") + + const needleA = uint64(910001) + const needleB = uint64(910002) + const needleC = uint64(910003) + const cookieA = uint32(0x11111111) + const cookieB = uint32(0x22222222) + const cookieC = uint32(0x33333333) + + httpClient := framework.NewHTTPClient() + fidA := framework.NewFileID(volumeID, needleA, cookieA) + fidB := framework.NewFileID(volumeID, needleB, cookieB) + fidC := framework.NewFileID(volumeID, needleC, cookieC) + + for _, tc := range []struct { + fid string + body string + }{ + {fid: fidA, body: "batch-delete-mixed-a"}, + {fid: fidB, body: "batch-delete-mixed-b"}, + {fid: fidC, body: "batch-delete-mixed-c"}, + } { + uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, []byte(tc.body)) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload %s expected 201, got %d", tc.fid, uploadResp.StatusCode) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + missingFid := framework.NewFileID(volumeID, 919999, 0x44444444) + mixedResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{ + FileIds: []string{"bad-fid", fidA, missingFid}, + }) + if err != nil { + t.Fatalf("BatchDelete mixed status request failed: %v", err) + } + if len(mixedResp.GetResults()) != 3 { + t.Fatalf("BatchDelete mixed status expected 3 results, got %d", len(mixedResp.GetResults())) + } + if mixedResp.GetResults()[0].GetStatus() != http.StatusBadRequest { + t.Fatalf("BatchDelete mixed result[0] expected 400, got %d", mixedResp.GetResults()[0].GetStatus()) + } + if mixedResp.GetResults()[1].GetStatus() != http.StatusAccepted { + t.Fatalf("BatchDelete mixed result[1] expected 202, got %d", mixedResp.GetResults()[1].GetStatus()) + } + if mixedResp.GetResults()[2].GetStatus() != http.StatusNotFound { + t.Fatalf("BatchDelete mixed result[2] expected 404, got %d", mixedResp.GetResults()[2].GetStatus()) + } + + readDeletedA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA) + _ = framework.ReadAllAndClose(t, readDeletedA) + if readDeletedA.StatusCode != http.StatusNotFound { + t.Fatalf("fidA should be deleted after batch delete, got status %d", readDeletedA.StatusCode) + } + + wrongCookieB := framework.NewFileID(volumeID, needleB, cookieB+1) + stopResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{ + FileIds: []string{wrongCookieB, fidC}, + }) + if err != nil { + t.Fatalf("BatchDelete mismatch-stop request failed: %v", err) + } + if len(stopResp.GetResults()) != 1 { + t.Fatalf("BatchDelete mismatch-stop expected 1 result due early break, got %d", len(stopResp.GetResults())) + } + if stopResp.GetResults()[0].GetStatus() != http.StatusBadRequest { + t.Fatalf("BatchDelete mismatch-stop expected 400, got %d", stopResp.GetResults()[0].GetStatus()) + } + + readB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB) + _ = framework.ReadAllAndClose(t, readB) + if readB.StatusCode != http.StatusOK { + t.Fatalf("fidB should remain after cookie mismatch path, got %d", readB.StatusCode) + } + + readC := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidC) + _ = framework.ReadAllAndClose(t, readC) + if readC.StatusCode != http.StatusOK { + t.Fatalf("fidC should remain when batch processing stops on mismatch, got %d", readC.StatusCode) + } +} + +func TestBatchDeleteRejectsChunkManifestNeedles(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(33) + framework.AllocateVolume(t, client, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 920001, 0x5555AAAA) + req, err := http.NewRequest(http.MethodPost, cluster.VolumeAdminURL()+"/"+fid+"?cm=true", bytes.NewReader([]byte("manifest-placeholder-payload"))) + if err != nil { + t.Fatalf("create chunk manifest upload request: %v", err) + } + req.Header.Set("Content-Type", "application/octet-stream") + uploadResp := framework.DoRequest(t, httpClient, req) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("chunk manifest upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{fid}}) + if err != nil { + t.Fatalf("BatchDelete chunk manifest should return response, got grpc error: %v", err) + } + if len(resp.GetResults()) != 1 { + t.Fatalf("BatchDelete chunk manifest expected one result, got %d", len(resp.GetResults())) + } + if resp.GetResults()[0].GetStatus() != http.StatusNotAcceptable { + t.Fatalf("BatchDelete chunk manifest expected status 406, got %d", resp.GetResults()[0].GetStatus()) + } + if !strings.Contains(resp.GetResults()[0].GetError(), "ChunkManifest") { + t.Fatalf("BatchDelete chunk manifest expected error mentioning ChunkManifest, got %q", resp.GetResults()[0].GetError()) + } + + readResp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusOK { + t.Fatalf("chunk manifest should not be deleted by BatchDelete reject path, got %d", readResp.StatusCode) + } +} diff --git a/test/volume_server/grpc/copy_receive_variants_test.go b/test/volume_server/grpc/copy_receive_variants_test.go new file mode 100644 index 000000000..14d9cee72 --- /dev/null +++ b/test/volume_server/grpc/copy_receive_variants_test.go @@ -0,0 +1,431 @@ +package volume_server_grpc_test + +import ( + "context" + "io" + "math" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestVolumeIncrementalCopyDataAndNoDataPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(91) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 770001, 0x1122AABB) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("incremental-copy-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + dataStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{ + VolumeId: volumeID, + SinceNs: 0, + }) + if err != nil { + t.Fatalf("VolumeIncrementalCopy start failed: %v", err) + } + + totalBytes := 0 + for { + msg, recvErr := dataStream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("VolumeIncrementalCopy recv failed: %v", recvErr) + } + totalBytes += len(msg.GetFileContent()) + } + if totalBytes == 0 { + t.Fatalf("VolumeIncrementalCopy expected streamed bytes for since_ns=0") + } + + noDataStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{ + VolumeId: volumeID, + SinceNs: math.MaxUint64, + }) + if err != nil { + t.Fatalf("VolumeIncrementalCopy no-data start failed: %v", err) + } + _, err = noDataStream.Recv() + if err != io.EOF { + t.Fatalf("VolumeIncrementalCopy no-data expected EOF, got: %v", err) + } +} + +func TestCopyFileIgnoreNotFoundAndStopOffsetZeroPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(92) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + missingNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Ext: ".definitely-missing", + CompactionRevision: math.MaxUint32, + StopOffset: 1, + IgnoreSourceFileNotFound: false, + }) + if err == nil { + _, err = missingNoIgnore.Recv() + } + if err == nil { + t.Fatalf("CopyFile should fail for missing source file when ignore_source_file_not_found=false") + } + + missingIgnored, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Ext: ".definitely-missing", + CompactionRevision: math.MaxUint32, + StopOffset: 1, + IgnoreSourceFileNotFound: true, + }) + if err != nil { + t.Fatalf("CopyFile ignore-not-found start failed: %v", err) + } + _, err = missingIgnored.Recv() + if err != io.EOF { + t.Fatalf("CopyFile ignore-not-found expected EOF, got: %v", err) + } + + stopZeroStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Ext: ".definitely-missing", + CompactionRevision: math.MaxUint32, + StopOffset: 0, + IgnoreSourceFileNotFound: false, + }) + if err != nil { + t.Fatalf("CopyFile stop_offset=0 start failed: %v", err) + } + _, err = stopZeroStream.Recv() + if err != io.EOF { + t.Fatalf("CopyFile stop_offset=0 expected EOF, got: %v", err) + } +} + +func TestCopyFileCompactionRevisionMismatch(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(94) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Ext: ".idx", + CompactionRevision: 1, // fresh volume starts at revision 0 + StopOffset: 1, + }) + if err == nil { + _, err = stream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "is compacted") { + t.Fatalf("CopyFile compaction mismatch error mismatch: %v", err) + } +} + +func TestReceiveFileProtocolViolationResponses(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + contentFirstStream, err := grpcClient.ReceiveFile(ctx) + if err != nil { + t.Fatalf("ReceiveFile stream create failed: %v", err) + } + if err = contentFirstStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_FileContent{ + FileContent: []byte("content-before-info"), + }, + }); err != nil { + t.Fatalf("ReceiveFile send content-first failed: %v", err) + } + contentFirstResp, err := contentFirstStream.CloseAndRecv() + if err != nil { + t.Fatalf("ReceiveFile content-first close failed: %v", err) + } + if !strings.Contains(contentFirstResp.GetError(), "file info must be sent first") { + t.Fatalf("ReceiveFile content-first response mismatch: %+v", contentFirstResp) + } + + unknownTypeStream, err := grpcClient.ReceiveFile(ctx) + if err != nil { + t.Fatalf("ReceiveFile stream create for unknown-type failed: %v", err) + } + if err = unknownTypeStream.Send(&volume_server_pb.ReceiveFileRequest{}); err != nil { + t.Fatalf("ReceiveFile send unknown-type request failed: %v", err) + } + unknownTypeResp, err := unknownTypeStream.CloseAndRecv() + if err != nil { + t.Fatalf("ReceiveFile unknown-type close failed: %v", err) + } + if !strings.Contains(unknownTypeResp.GetError(), "unknown message type") { + t.Fatalf("ReceiveFile unknown-type response mismatch: %+v", unknownTypeResp) + } +} + +func TestReceiveFileSuccessForRegularVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(95) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + payloadA := []byte("receive-file-chunk-a:") + payloadB := []byte("receive-file-chunk-b") + expected := append(append([]byte{}, payloadA...), payloadB...) + + receiveStream, err := grpcClient.ReceiveFile(ctx) + if err != nil { + t.Fatalf("ReceiveFile stream create failed: %v", err) + } + + if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_Info{ + Info: &volume_server_pb.ReceiveFileInfo{ + VolumeId: volumeID, + Ext: ".tmprecv", + Collection: "", + IsEcVolume: false, + FileSize: uint64(len(expected)), + }, + }, + }); err != nil { + t.Fatalf("ReceiveFile send info failed: %v", err) + } + if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadA}, + }); err != nil { + t.Fatalf("ReceiveFile send payloadA failed: %v", err) + } + if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadB}, + }); err != nil { + t.Fatalf("ReceiveFile send payloadB failed: %v", err) + } + + resp, err := receiveStream.CloseAndRecv() + if err != nil { + t.Fatalf("ReceiveFile close failed: %v", err) + } + if resp.GetError() != "" { + t.Fatalf("ReceiveFile unexpected error response: %+v", resp) + } + if resp.GetBytesWritten() != uint64(len(expected)) { + t.Fatalf("ReceiveFile bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(expected)) + } + + copyStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Ext: ".tmprecv", + CompactionRevision: math.MaxUint32, + StopOffset: uint64(len(expected)), + }) + if err != nil { + t.Fatalf("CopyFile for received data start failed: %v", err) + } + + var copied []byte + for { + msg, recvErr := copyStream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("CopyFile for received data recv failed: %v", recvErr) + } + copied = append(copied, msg.GetFileContent()...) + } + + if string(copied) != string(expected) { + t.Fatalf("received file data mismatch: got %q want %q", string(copied), string(expected)) + } +} + +func TestReceiveFileSuccessForEcVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + const volumeID = uint32(96) + const collection = "ec-receive-success" + const ext = ".ec00" + + payloadA := []byte("receive-ec-file-chunk-a:") + payloadB := []byte("receive-ec-file-chunk-b") + expected := append(append([]byte{}, payloadA...), payloadB...) + + receiveStream, err := grpcClient.ReceiveFile(ctx) + if err != nil { + t.Fatalf("ReceiveFile stream create failed: %v", err) + } + + if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_Info{ + Info: &volume_server_pb.ReceiveFileInfo{ + VolumeId: volumeID, + Ext: ext, + Collection: collection, + IsEcVolume: true, + ShardId: 0, + FileSize: uint64(len(expected)), + }, + }, + }); err != nil { + t.Fatalf("ReceiveFile send EC info failed: %v", err) + } + if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadA}, + }); err != nil { + t.Fatalf("ReceiveFile send EC payloadA failed: %v", err) + } + if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadB}, + }); err != nil { + t.Fatalf("ReceiveFile send EC payloadB failed: %v", err) + } + + resp, err := receiveStream.CloseAndRecv() + if err != nil { + t.Fatalf("ReceiveFile EC close failed: %v", err) + } + if resp.GetError() != "" { + t.Fatalf("ReceiveFile EC unexpected error response: %+v", resp) + } + if resp.GetBytesWritten() != uint64(len(expected)) { + t.Fatalf("ReceiveFile EC bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(expected)) + } + + copyStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Collection: collection, + IsEcVolume: true, + Ext: ext, + CompactionRevision: math.MaxUint32, + StopOffset: uint64(len(expected)), + }) + if err != nil { + t.Fatalf("CopyFile for received EC data start failed: %v", err) + } + + var copied []byte + for { + msg, recvErr := copyStream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("CopyFile for received EC data recv failed: %v", recvErr) + } + copied = append(copied, msg.GetFileContent()...) + } + + if string(copied) != string(expected) { + t.Fatalf("received EC file data mismatch: got %q want %q", string(copied), string(expected)) + } +} + +func TestCopyFileEcVolumeIgnoreMissingSourcePaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + streamNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: 99601, + Collection: "ec-copy-missing", + IsEcVolume: true, + Ext: ".ec00", + CompactionRevision: math.MaxUint32, + StopOffset: 1, + IgnoreSourceFileNotFound: false, + }) + if err == nil { + _, err = streamNoIgnore.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found ec volume id") { + t.Fatalf("CopyFile EC missing source error mismatch: %v", err) + } + + streamIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: 99602, + Collection: "ec-copy-missing", + IsEcVolume: true, + Ext: ".ec00", + CompactionRevision: math.MaxUint32, + StopOffset: 1, + IgnoreSourceFileNotFound: true, + }) + if err != nil { + t.Fatalf("CopyFile EC ignore-missing start failed: %v", err) + } + _, err = streamIgnore.Recv() + if err != io.EOF { + t.Fatalf("CopyFile EC ignore-missing expected EOF, got: %v", err) + } +} diff --git a/test/volume_server/grpc/copy_sync_test.go b/test/volume_server/grpc/copy_sync_test.go new file mode 100644 index 000000000..3c2916fd0 --- /dev/null +++ b/test/volume_server/grpc/copy_sync_test.go @@ -0,0 +1,284 @@ +package volume_server_grpc_test + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestVolumeSyncStatusAndReadVolumeFileStatus(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(41) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + syncResp, err := grpcClient.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VolumeSyncStatus failed: %v", err) + } + if syncResp.GetVolumeId() != volumeID { + t.Fatalf("VolumeSyncStatus volume id mismatch: got %d want %d", syncResp.GetVolumeId(), volumeID) + } + + statusResp, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("ReadVolumeFileStatus failed: %v", err) + } + if statusResp.GetVolumeId() != volumeID { + t.Fatalf("ReadVolumeFileStatus volume id mismatch: got %d want %d", statusResp.GetVolumeId(), volumeID) + } + if statusResp.GetVersion() == 0 { + t.Fatalf("ReadVolumeFileStatus expected non-zero version") + } +} + +func TestCopyAndStreamMethodsMissingVolumePaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{VolumeId: 98761}) + if err == nil { + t.Fatalf("VolumeSyncStatus should fail for missing volume") + } + + incrementalStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{VolumeId: 98762, SinceNs: 0}) + if err == nil { + _, err = incrementalStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("VolumeIncrementalCopy missing-volume error mismatch: %v", err) + } + + readAllStream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{98763}}) + if err == nil { + _, err = readAllStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", err) + } + + copyFileStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{VolumeId: 98764, Ext: ".dat", StopOffset: 1}) + if err == nil { + _, err = copyFileStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("CopyFile missing-volume error mismatch: %v", err) + } + + _, err = grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: 98765}) + if err == nil || !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("ReadVolumeFileStatus missing-volume error mismatch: %v", err) + } +} + +func TestVolumeCopyAndReceiveFileMaintenanceRejection(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + copyStream, err := grpcClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{VolumeId: 1, SourceDataNode: "127.0.0.1:1234"}) + if err == nil { + _, err = copyStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeCopy maintenance error mismatch: %v", err) + } + + receiveClient, err := grpcClient.ReceiveFile(ctx) + if err != nil { + t.Fatalf("ReceiveFile client creation failed: %v", err) + } + _ = receiveClient.Send(&volume_server_pb.ReceiveFileRequest{ + Data: &volume_server_pb.ReceiveFileRequest_Info{ + Info: &volume_server_pb.ReceiveFileInfo{VolumeId: 1, Ext: ".dat"}, + }, + }) + _, err = receiveClient.CloseAndRecv() + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("ReceiveFile maintenance error mismatch: %v", err) + } +} + +func TestVolumeCopySuccessFromPeerAndMountsDestination(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1()) + sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer sourceConn.Close() + destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) + defer destConn.Close() + + const volumeID = uint32(42) + framework.AllocateVolume(t, sourceClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 880001, 0x12345678) + payload := []byte("volume-copy-success-payload") + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload to source expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{ + VolumeId: volumeID, + Collection: "", + SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1], + }) + if err != nil { + t.Fatalf("VolumeCopy start failed: %v", err) + } + + sawFinalAppendTimestamp := false + for { + msg, recvErr := copyStream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("VolumeCopy recv failed: %v", recvErr) + } + if msg.GetLastAppendAtNs() > 0 { + sawFinalAppendTimestamp = true + } + } + if !sawFinalAppendTimestamp { + t.Fatalf("VolumeCopy expected final response with last_append_at_ns") + } + + destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid) + destReadBody := framework.ReadAllAndClose(t, destReadResp) + if destReadResp.StatusCode != http.StatusOK { + t.Fatalf("read from copied destination expected 200, got %d", destReadResp.StatusCode) + } + if string(destReadBody) != string(payload) { + t.Fatalf("destination copied payload mismatch: got %q want %q", string(destReadBody), string(payload)) + } +} + +func TestVolumeCopyOverwritesExistingDestinationVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1()) + sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer sourceConn.Close() + destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) + defer destConn.Close() + + const volumeID = uint32(43) + framework.AllocateVolume(t, sourceClient, volumeID, "") + framework.AllocateVolume(t, destClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 880002, 0x23456789) + sourcePayload := []byte("volume-copy-overwrite-source") + destPayload := []byte("volume-copy-overwrite-destination-old") + + sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, sourcePayload) + _ = framework.ReadAllAndClose(t, sourceUploadResp) + if sourceUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload to source expected 201, got %d", sourceUploadResp.StatusCode) + } + + destUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid, destPayload) + _ = framework.ReadAllAndClose(t, destUploadResp) + if destUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload to destination expected 201, got %d", destUploadResp.StatusCode) + } + + destReadBeforeResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid) + destReadBeforeBody := framework.ReadAllAndClose(t, destReadBeforeResp) + if destReadBeforeResp.StatusCode != http.StatusOK { + t.Fatalf("destination pre-copy read expected 200, got %d", destReadBeforeResp.StatusCode) + } + if string(destReadBeforeBody) != string(destPayload) { + t.Fatalf("destination pre-copy payload mismatch: got %q want %q", string(destReadBeforeBody), string(destPayload)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{ + VolumeId: volumeID, + Collection: "", + SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1], + }) + if err != nil { + t.Fatalf("VolumeCopy overwrite start failed: %v", err) + } + + sawFinalAppendTimestamp := false + for { + msg, recvErr := copyStream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("VolumeCopy overwrite recv failed: %v", recvErr) + } + if msg.GetLastAppendAtNs() > 0 { + sawFinalAppendTimestamp = true + } + } + if !sawFinalAppendTimestamp { + t.Fatalf("VolumeCopy overwrite expected final response with last_append_at_ns") + } + + destReadAfterResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid) + destReadAfterBody := framework.ReadAllAndClose(t, destReadAfterResp) + if destReadAfterResp.StatusCode != http.StatusOK { + t.Fatalf("destination post-copy read expected 200, got %d", destReadAfterResp.StatusCode) + } + if string(destReadAfterBody) != string(sourcePayload) { + t.Fatalf("destination post-copy payload mismatch: got %q want %q", string(destReadAfterBody), string(sourcePayload)) + } +} diff --git a/test/volume_server/grpc/data_rw_test.go b/test/volume_server/grpc/data_rw_test.go new file mode 100644 index 000000000..43969532d --- /dev/null +++ b/test/volume_server/grpc/data_rw_test.go @@ -0,0 +1,146 @@ +package volume_server_grpc_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestReadNeedleBlobAndMetaMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{ + VolumeId: 99111, + Offset: 0, + Size: 16, + }) + if err == nil { + t.Fatalf("ReadNeedleBlob should fail for missing volume") + } + if !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("ReadNeedleBlob missing volume error mismatch: %v", err) + } + + _, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{ + VolumeId: 99112, + NeedleId: 1, + Offset: 0, + Size: 16, + }) + if err == nil { + t.Fatalf("ReadNeedleMeta should fail for missing volume") + } + if !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("ReadNeedleMeta missing volume error mismatch: %v", err) + } +} + +func TestWriteNeedleBlobMaintenanceAndMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{ + VolumeId: 99113, + NeedleId: 1, + NeedleBlob: []byte("abc"), + Size: 3, + }) + if err == nil { + t.Fatalf("WriteNeedleBlob should fail for missing volume") + } + if !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("WriteNeedleBlob missing volume error mismatch: %v", err) + } + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{ + VolumeId: 1, + NeedleId: 2, + NeedleBlob: []byte("def"), + Size: 3, + }) + if err == nil { + t.Fatalf("WriteNeedleBlob should fail in maintenance mode") + } + if !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("WriteNeedleBlob maintenance mode error mismatch: %v", err) + } +} + +func TestReadNeedleBlobAndMetaInvalidOffsets(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(92) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 880001, 0xCCDD1122) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("invalid-offset-check")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{ + VolumeId: volumeID, + Offset: 1 << 40, + Size: 64, + }) + if err == nil { + t.Fatalf("ReadNeedleBlob should fail for invalid offset") + } + if !strings.Contains(strings.ToLower(err.Error()), "read needle blob") { + t.Fatalf("ReadNeedleBlob invalid offset error mismatch: %v", err) + } + + _, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{ + VolumeId: volumeID, + NeedleId: 880001, + Offset: 1 << 40, + Size: 64, + }) + if err == nil { + t.Fatalf("ReadNeedleMeta should fail for invalid offset") + } +} diff --git a/test/volume_server/grpc/data_stream_success_test.go b/test/volume_server/grpc/data_stream_success_test.go new file mode 100644 index 000000000..90f2a8248 --- /dev/null +++ b/test/volume_server/grpc/data_stream_success_test.go @@ -0,0 +1,273 @@ +package volume_server_grpc_test + +import ( + "context" + "io" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/idx" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(83) + const sourceNeedleID = uint64(333333) + const sourceCookie = uint32(0xABCD0102) + const clonedNeedleID = uint64(333334) + + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + payload := []byte("blob-roundtrip-content") + fid := framework.NewFileID(volumeID, sourceNeedleID, sourceCookie) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + fileStatus, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("ReadVolumeFileStatus failed: %v", err) + } + if fileStatus.GetIdxFileSize() == 0 { + t.Fatalf("expected non-zero idx file size after upload") + } + + idxBytes := copyFileBytes(t, grpcClient, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Ext: ".idx", + CompactionRevision: fileStatus.GetCompactionRevision(), + StopOffset: fileStatus.GetIdxFileSize(), + }) + offset, size := findNeedleOffsetAndSize(t, idxBytes, sourceNeedleID) + + blobResp, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{ + VolumeId: volumeID, + Offset: offset, + Size: size, + }) + if err != nil { + t.Fatalf("ReadNeedleBlob failed: %v", err) + } + if len(blobResp.GetNeedleBlob()) == 0 { + t.Fatalf("ReadNeedleBlob returned empty blob") + } + + metaResp, err := grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{ + VolumeId: volumeID, + NeedleId: sourceNeedleID, + Offset: offset, + Size: size, + }) + if err != nil { + t.Fatalf("ReadNeedleMeta failed: %v", err) + } + if metaResp.GetCookie() != sourceCookie { + t.Fatalf("ReadNeedleMeta cookie mismatch: got %d want %d", metaResp.GetCookie(), sourceCookie) + } + + _, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{ + VolumeId: volumeID, + NeedleId: clonedNeedleID, + Size: size, + NeedleBlob: blobResp.GetNeedleBlob(), + }) + if err != nil { + t.Fatalf("WriteNeedleBlob failed: %v", err) + } + + clonedStatus, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: volumeID, + NeedleId: clonedNeedleID, + }) + if err != nil { + t.Fatalf("VolumeNeedleStatus for cloned needle failed: %v", err) + } + if clonedStatus.GetNeedleId() != sourceNeedleID { + t.Fatalf("cloned needle status id mismatch: got %d want %d", clonedStatus.GetNeedleId(), sourceNeedleID) + } + if clonedStatus.GetCookie() != sourceCookie { + t.Fatalf("cloned needle cookie mismatch: got %d want %d", clonedStatus.GetCookie(), sourceCookie) + } + + clonedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, clonedNeedleID, sourceCookie)) + clonedReadBody := framework.ReadAllAndClose(t, clonedReadResp) + if clonedReadResp.StatusCode != 200 { + t.Fatalf("cloned needle GET expected 200, got %d", clonedReadResp.StatusCode) + } + if string(clonedReadBody) != string(payload) { + t.Fatalf("cloned needle body mismatch: got %q want %q", string(clonedReadBody), string(payload)) + } +} + +func TestReadAllNeedlesStreamsUploadedRecords(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(84) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + expected := map[uint64]string{ + 444441: "read-all-needle-one", + 444442: "read-all-needle-two", + } + for key, body := range expected { + resp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, key, 0xA0B0C0D0), []byte(body)) + _ = framework.ReadAllAndClose(t, resp) + if resp.StatusCode != 201 { + t.Fatalf("upload for key %d expected 201, got %d", key, resp.StatusCode) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{volumeID}}) + if err != nil { + t.Fatalf("ReadAllNeedles start failed: %v", err) + } + + seen := map[uint64]string{} + for { + msg, recvErr := stream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("ReadAllNeedles recv failed: %v", recvErr) + } + if _, wanted := expected[msg.GetNeedleId()]; wanted { + seen[msg.GetNeedleId()] = string(msg.GetNeedleBlob()) + } + } + + for key, body := range expected { + got, found := seen[key] + if !found { + t.Fatalf("ReadAllNeedles missing key %d in stream", key) + } + if got != body { + t.Fatalf("ReadAllNeedles body mismatch for key %d: got %q want %q", key, got, body) + } + } +} + +func TestReadAllNeedlesExistingThenMissingVolumeAbortsStream(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const existingVolumeID = uint32(85) + const missingVolumeID = uint32(98585) + const needleID = uint64(445551) + framework.AllocateVolume(t, grpcClient, existingVolumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(existingVolumeID, needleID, 0xAA11BB22) + payload := "read-all-existing-then-missing" + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte(payload)) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{ + VolumeIds: []uint32{existingVolumeID, missingVolumeID}, + }) + if err != nil { + t.Fatalf("ReadAllNeedles start failed: %v", err) + } + + seenUploadedNeedle := false + for { + msg, recvErr := stream.Recv() + if recvErr == io.EOF { + t.Fatalf("ReadAllNeedles expected stream error for missing volume, got EOF") + } + if recvErr != nil { + if !strings.Contains(recvErr.Error(), "not found volume id") { + t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", recvErr) + } + break + } + if msg.GetNeedleId() == needleID && string(msg.GetNeedleBlob()) == payload { + seenUploadedNeedle = true + } + } + + if !seenUploadedNeedle { + t.Fatalf("ReadAllNeedles should stream entries from existing volume before missing-volume abort") + } +} + +func copyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.CopyFile(ctx, req) + if err != nil { + t.Fatalf("CopyFile start failed: %v", err) + } + + var out []byte + for { + msg, recvErr := stream.Recv() + if recvErr == io.EOF { + return out + } + if recvErr != nil { + t.Fatalf("CopyFile recv failed: %v", recvErr) + } + out = append(out, msg.GetFileContent()...) + } +} + +func findNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) { + t.Helper() + + for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize { + key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize]) + if uint64(key) != needleID { + continue + } + if entryOffset.IsZero() || entrySize <= 0 { + continue + } + return entryOffset.ToActualOffset(), int32(entrySize) + } + + t.Fatalf("needle id %d not found in idx entries", needleID) + return 0, 0 +} diff --git a/test/volume_server/grpc/erasure_coding_test.go b/test/volume_server/grpc/erasure_coding_test.go new file mode 100644 index 000000000..8a0d8f75f --- /dev/null +++ b/test/volume_server/grpc/erasure_coding_test.go @@ -0,0 +1,777 @@ +package volume_server_grpc_test + +import ( + "context" + "io" + "math" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestEcMaintenanceModeRejections(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: true, + Version: stateResp.GetState().GetVersion(), + }, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: 1, Collection: ""}) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeEcShardsGenerate maintenance error mismatch: %v", err) + } + + _, err = grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: 1, + Collection: "", + SourceDataNode: "127.0.0.1:1", + ShardIds: []uint32{0}, + }) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeEcShardsCopy maintenance error mismatch: %v", err) + } + + _, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: 1, + Collection: "", + ShardIds: []uint32{0}, + }) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeEcShardsDelete maintenance error mismatch: %v", err) + } + + _, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{ + VolumeId: 1, + Collection: "", + FileKey: 1, + Version: 3, + }) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeEcBlobDelete maintenance error mismatch: %v", err) + } + + _, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ + VolumeId: 1, + Collection: "", + }) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeEcShardsToVolume maintenance error mismatch: %v", err) + } +} + +func TestEcMissingInvalidAndNoopPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: 98791, + Collection: "", + }) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeEcShardsGenerate missing-volume error mismatch: %v", err) + } + + rebuildResp, err := grpcClient.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{ + VolumeId: 98792, + Collection: "ec-rebuild", + }) + if err != nil { + t.Fatalf("VolumeEcShardsRebuild missing-volume should return empty success, got: %v", err) + } + if len(rebuildResp.GetRebuiltShardIds()) != 0 { + t.Fatalf("VolumeEcShardsRebuild expected no rebuilt shards for missing volume, got %v", rebuildResp.GetRebuiltShardIds()) + } + + _, err = grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: 98793, + Collection: "ec-copy", + SourceDataNode: "127.0.0.1:1", + ShardIds: []uint32{0}, + DiskId: 99, + }) + if err == nil || !strings.Contains(err.Error(), "invalid disk_id") { + t.Fatalf("VolumeEcShardsCopy invalid-disk error mismatch: %v", err) + } + + _, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: 98794, + Collection: "ec-delete", + ShardIds: []uint32{0, 1}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsDelete missing-volume should be no-op success, got: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: 98795, + Collection: "ec-mount", + ShardIds: []uint32{0}, + }) + if err == nil { + t.Fatalf("VolumeEcShardsMount should fail for missing EC shards") + } + + _, err = grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{ + VolumeId: 98796, + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsUnmount missing shards should be no-op success, got: %v", err) + } + + readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{ + VolumeId: 98797, + ShardId: 0, + Offset: 0, + Size: 1, + }) + if err == nil { + _, err = readStream.Recv() + } + if err == nil || err == io.EOF { + t.Fatalf("VolumeEcShardRead should fail for missing EC volume") + } + + _, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{ + VolumeId: 98798, + Collection: "ec-blob", + FileKey: 1, + Version: 3, + }) + if err != nil { + t.Fatalf("VolumeEcBlobDelete missing local EC volume should be no-op success, got: %v", err) + } + + _, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ + VolumeId: 98799, + Collection: "ec-to-volume", + }) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeEcShardsToVolume missing-volume error mismatch: %v", err) + } + + _, err = grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{ + VolumeId: 98800, + }) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeEcShardsInfo missing-volume error mismatch: %v", err) + } +} + +func TestEcGenerateMountInfoUnmountLifecycle(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(115) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 990001, 0x1234ABCD) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-generate-lifecycle-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate success path failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount success path failed: %v", err) + } + + infoResp, err := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{ + VolumeId: volumeID, + }) + if err != nil { + t.Fatalf("VolumeEcShardsInfo after mount failed: %v", err) + } + if len(infoResp.GetEcShardInfos()) == 0 { + t.Fatalf("VolumeEcShardsInfo expected non-empty shard infos after mount") + } + if infoResp.GetVolumeSize() == 0 { + t.Fatalf("VolumeEcShardsInfo expected non-zero volume size after mount") + } + + _, err = grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{ + VolumeId: volumeID, + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsUnmount success path failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{ + VolumeId: volumeID, + }) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeEcShardsInfo after unmount expected not-found error, got: %v", err) + } +} + +func TestEcShardReadAndBlobDeleteLifecycle(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(116) + const fileKey = uint64(990002) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, fileKey, 0x2233CCDD) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-shard-read-delete-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount failed: %v", err) + } + + readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{ + VolumeId: volumeID, + ShardId: 0, + Offset: 0, + Size: 1, + }) + if err != nil { + t.Fatalf("VolumeEcShardRead start failed: %v", err) + } + firstChunk, err := readStream.Recv() + if err != nil { + t.Fatalf("VolumeEcShardRead recv failed: %v", err) + } + if len(firstChunk.GetData()) == 0 { + t.Fatalf("VolumeEcShardRead expected non-empty data chunk before deletion") + } + + _, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{ + VolumeId: volumeID, + Collection: "", + FileKey: fileKey, + Version: uint32(needle.GetCurrentVersion()), + }) + if err != nil { + t.Fatalf("VolumeEcBlobDelete first delete failed: %v", err) + } + + _, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{ + VolumeId: volumeID, + Collection: "", + FileKey: fileKey, + Version: uint32(needle.GetCurrentVersion()), + }) + if err != nil { + t.Fatalf("VolumeEcBlobDelete second delete should be idempotent success, got: %v", err) + } + + deletedStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{ + VolumeId: volumeID, + ShardId: 0, + FileKey: fileKey, + Offset: 0, + Size: 1, + }) + if err != nil { + t.Fatalf("VolumeEcShardRead deleted-check start failed: %v", err) + } + deletedMsg, err := deletedStream.Recv() + if err != nil { + t.Fatalf("VolumeEcShardRead deleted-check recv failed: %v", err) + } + if !deletedMsg.GetIsDeleted() { + t.Fatalf("VolumeEcShardRead expected IsDeleted=true after blob delete") + } + _, err = deletedStream.Recv() + if err != io.EOF { + t.Fatalf("VolumeEcShardRead deleted-check expected EOF after deleted marker, got: %v", err) + } +} + +func TestEcRebuildMissingShardLifecycle(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(117) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 990003, 0x3344DDEE) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-rebuild-shard-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsDelete shard 0 failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0}, + }) + if err == nil { + t.Fatalf("VolumeEcShardsMount should fail when shard 0 has been deleted") + } + + rebuildResp, err := grpcClient.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsRebuild failed: %v", err) + } + if len(rebuildResp.GetRebuiltShardIds()) == 0 { + t.Fatalf("VolumeEcShardsRebuild expected rebuilt shard ids") + } + foundShard0 := false + for _, shardID := range rebuildResp.GetRebuiltShardIds() { + if shardID == 0 { + foundShard0 = true + break + } + } + if !foundShard0 { + t.Fatalf("VolumeEcShardsRebuild expected shard 0 to be rebuilt, got %v", rebuildResp.GetRebuiltShardIds()) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount shard 0 after rebuild failed: %v", err) + } +} + +func TestEcShardsToVolumeMissingShardAndNoLiveEntries(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + httpClient := framework.NewHTTPClient() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + t.Run("missing shard returns error", func(t *testing.T) { + const volumeID = uint32(118) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 990004, 0x4455EEFF) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-to-volume-missing-shard-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsDelete shard 0 failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{1}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount shard 1 failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err == nil || !strings.Contains(err.Error(), "missing shard 0") { + t.Fatalf("VolumeEcShardsToVolume missing-shard error mismatch: %v", err) + } + }) + + t.Run("no live entries returns failed precondition", func(t *testing.T) { + const volumeID = uint32(119) + const needleID = uint64(990005) + const cookie = uint32(0x5566FF11) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-no-live-entries-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + deleteResp := framework.DoRequest(t, httpClient, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid)) + _ = framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode) + } + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount data shards failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err == nil { + t.Fatalf("VolumeEcShardsToVolume expected failed-precondition error when no live entries") + } + if status.Code(err) != codes.FailedPrecondition { + t.Fatalf("VolumeEcShardsToVolume no-live-entries expected FailedPrecondition, got %v (%v)", status.Code(err), err) + } + if !strings.Contains(err.Error(), erasure_coding.EcNoLiveEntriesSubstring) { + t.Fatalf("VolumeEcShardsToVolume no-live-entries error should mention %q, got %v", erasure_coding.EcNoLiveEntriesSubstring, err) + } + }) +} + +func TestEcShardsToVolumeSuccessRoundTrip(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(120) + const needleID = uint64(990006) + const cookie = uint32(0x66771122) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, cookie) + payload := []byte("ec-shards-to-volume-success-roundtrip-content") + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsMount data shards failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsToVolume success path failed: %v", err) + } + + readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid) + readBody := framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusOK { + t.Fatalf("post-conversion read expected 200, got %d", readResp.StatusCode) + } + if string(readBody) != string(payload) { + t.Fatalf("post-conversion payload mismatch: got %q want %q", string(readBody), string(payload)) + } +} + +func TestEcShardsDeleteLastShardRemovesEcx(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(121) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 990007, 0x77882233) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-delete-all-shards-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("VolumeEcShardsGenerate failed: %v", err) + } + + // Verify .ecx is present before deleting all shards. + ecxBeforeDelete, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Collection: "", + IsEcVolume: true, + Ext: ".ecx", + CompactionRevision: math.MaxUint32, + StopOffset: 1, + }) + if err != nil { + t.Fatalf("CopyFile .ecx before shard deletion start failed: %v", err) + } + if _, err = ecxBeforeDelete.Recv(); err != nil { + t.Fatalf("CopyFile .ecx before shard deletion recv failed: %v", err) + } + + _, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: volumeID, + Collection: "", + ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, + }) + if err != nil { + t.Fatalf("VolumeEcShardsDelete all shards failed: %v", err) + } + + ecxAfterDelete, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Collection: "", + IsEcVolume: true, + Ext: ".ecx", + CompactionRevision: math.MaxUint32, + StopOffset: 1, + }) + if err == nil { + _, err = ecxAfterDelete.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found ec volume id") { + t.Fatalf("CopyFile .ecx after deleting all shards should fail not-found, got: %v", err) + } +} + +func TestEcShardsCopyFromPeerSuccess(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1()) + sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer sourceConn.Close() + destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) + defer destConn.Close() + + const volumeID = uint32(122) + framework.AllocateVolume(t, sourceClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 990008, 0x88993344) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, []byte("ec-copy-from-peer-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("source upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := sourceClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + Collection: "", + }) + if err != nil { + t.Fatalf("source VolumeEcShardsGenerate failed: %v", err) + } + + sourceDataNode := clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1] + _, err = destClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: volumeID, + Collection: "", + SourceDataNode: sourceDataNode, + ShardIds: []uint32{0}, + CopyEcxFile: true, + CopyVifFile: true, + }) + if err != nil { + t.Fatalf("destination VolumeEcShardsCopy success path failed: %v", err) + } + + for _, ext := range []string{".ec00", ".ecx", ".vif"} { + copyStream, copyErr := destClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ + VolumeId: volumeID, + Collection: "", + IsEcVolume: true, + Ext: ext, + CompactionRevision: math.MaxUint32, + StopOffset: 1, + }) + if copyErr != nil { + t.Fatalf("destination CopyFile %s start failed: %v", ext, copyErr) + } + if _, copyErr = copyStream.Recv(); copyErr != nil { + t.Fatalf("destination CopyFile %s recv failed: %v", ext, copyErr) + } + } +} + +func TestEcShardsCopyFailsWhenSourceUnavailable(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: 12345, + Collection: "", + SourceDataNode: "127.0.0.1:1.1", + ShardIds: []uint32{0}, + CopyEcxFile: true, + }) + if err == nil || !strings.Contains(err.Error(), "VolumeEcShardsCopy volume") { + t.Fatalf("VolumeEcShardsCopy source-unavailable error mismatch: %v", err) + } +} diff --git a/test/volume_server/grpc/health_state_test.go b/test/volume_server/grpc/health_state_test.go new file mode 100644 index 000000000..cac40731b --- /dev/null +++ b/test/volume_server/grpc/health_state_test.go @@ -0,0 +1,139 @@ +package volume_server_grpc_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestStateAndStatusRPCs(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + initialState, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + if initialState.GetState() == nil { + t.Fatalf("GetState returned nil state") + } + + setResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: true, + Version: initialState.GetState().GetVersion(), + }, + }) + if err != nil { + t.Fatalf("SetState(maintenance=true) failed: %v", err) + } + if !setResp.GetState().GetMaintenance() { + t.Fatalf("expected maintenance=true after SetState") + } + + setResp, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: false, + Version: setResp.GetState().GetVersion(), + }, + }) + if err != nil { + t.Fatalf("SetState(maintenance=false) failed: %v", err) + } + if setResp.GetState().GetMaintenance() { + t.Fatalf("expected maintenance=false after SetState") + } + + statusResp, err := client.VolumeServerStatus(ctx, &volume_server_pb.VolumeServerStatusRequest{}) + if err != nil { + t.Fatalf("VolumeServerStatus failed: %v", err) + } + if statusResp.GetVersion() == "" { + t.Fatalf("VolumeServerStatus returned empty version") + } + if len(statusResp.GetDiskStatuses()) == 0 { + t.Fatalf("VolumeServerStatus returned no disk statuses") + } + if statusResp.GetState() == nil { + t.Fatalf("VolumeServerStatus returned nil state") + } + if statusResp.GetMemoryStatus() == nil { + t.Fatalf("VolumeServerStatus returned nil memory status") + } + if statusResp.GetMemoryStatus().GetGoroutines() <= 0 { + t.Fatalf("VolumeServerStatus memory status should report goroutines, got %d", statusResp.GetMemoryStatus().GetGoroutines()) + } + + pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{}) + if err != nil { + t.Fatalf("Ping failed: %v", err) + } + if pingResp.GetStartTimeNs() == 0 || pingResp.GetStopTimeNs() == 0 { + t.Fatalf("Ping timestamps should be non-zero: %+v", pingResp) + } + if pingResp.GetStopTimeNs() < pingResp.GetStartTimeNs() { + t.Fatalf("Ping stop time should be >= start time: %+v", pingResp) + } +} + +func TestSetStateVersionMismatchAndNilStateNoop(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + initialState, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + initialVersion := initialState.GetState().GetVersion() + + staleResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: true, + Version: initialVersion + 1, + }, + }) + if err == nil { + t.Fatalf("SetState with stale version should fail") + } + if !strings.Contains(err.Error(), "version mismatch") { + t.Fatalf("SetState stale version error mismatch: %v", err) + } + if staleResp.GetState().GetVersion() != initialVersion { + t.Fatalf("SetState stale version should not mutate server version: got %d want %d", staleResp.GetState().GetVersion(), initialVersion) + } + if staleResp.GetState().GetMaintenance() != initialState.GetState().GetMaintenance() { + t.Fatalf("SetState stale version should not mutate maintenance flag") + } + + nilResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{}) + if err != nil { + t.Fatalf("SetState nil-state request should be no-op success: %v", err) + } + if nilResp.GetState().GetVersion() != initialVersion { + t.Fatalf("SetState nil-state should keep version unchanged: got %d want %d", nilResp.GetState().GetVersion(), initialVersion) + } + if nilResp.GetState().GetMaintenance() != initialState.GetState().GetMaintenance() { + t.Fatalf("SetState nil-state should keep maintenance unchanged") + } +} diff --git a/test/volume_server/grpc/scrub_query_test.go b/test/volume_server/grpc/scrub_query_test.go new file mode 100644 index 000000000..66766f79d --- /dev/null +++ b/test/volume_server/grpc/scrub_query_test.go @@ -0,0 +1,385 @@ +package volume_server_grpc_test + +import ( + "context" + "io" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestScrubVolumeIndexAndUnsupportedMode(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(61) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + indexResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ + VolumeIds: []uint32{volumeID}, + Mode: volume_server_pb.VolumeScrubMode_INDEX, + }) + if err != nil { + t.Fatalf("ScrubVolume index mode failed: %v", err) + } + if indexResp.GetTotalVolumes() != 1 { + t.Fatalf("ScrubVolume expected total_volumes=1, got %d", indexResp.GetTotalVolumes()) + } + + _, err = grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ + VolumeIds: []uint32{volumeID}, + Mode: volume_server_pb.VolumeScrubMode(99), + }) + if err == nil { + t.Fatalf("ScrubVolume should fail for unsupported mode") + } + if !strings.Contains(err.Error(), "unsupported volume scrub mode") { + t.Fatalf("ScrubVolume unsupported mode error mismatch: %v", err) + } +} + +func TestScrubEcVolumeMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{ + VolumeIds: []uint32{98765}, + Mode: volume_server_pb.VolumeScrubMode_INDEX, + }) + if err == nil { + t.Fatalf("ScrubEcVolume should fail for missing EC volume") + } + if !strings.Contains(err.Error(), "EC volume id") { + t.Fatalf("ScrubEcVolume missing-volume error mismatch: %v", err) + } +} + +func TestScrubEcVolumeAutoSelectNoEcVolumes(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{ + Mode: volume_server_pb.VolumeScrubMode_INDEX, + }) + if err != nil { + t.Fatalf("ScrubEcVolume auto-select failed: %v", err) + } + if resp.GetTotalVolumes() != 0 { + t.Fatalf("ScrubEcVolume auto-select expected total_volumes=0 without EC data, got %d", resp.GetTotalVolumes()) + } + if len(resp.GetBrokenVolumeIds()) != 0 { + t.Fatalf("ScrubEcVolume auto-select expected no broken volumes, got %v", resp.GetBrokenVolumeIds()) + } +} + +func TestQueryInvalidAndMissingFileIDPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + invalidStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ + FromFileIds: []string{"bad-fid"}, + Selections: []string{"name"}, + Filter: &volume_server_pb.QueryRequest_Filter{}, + InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ + JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{}, + }, + }) + if err == nil { + _, err = invalidStream.Recv() + } + if err == nil { + t.Fatalf("Query should fail for invalid file id") + } + + missingFid := framework.NewFileID(98766, 1, 1) + missingStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ + FromFileIds: []string{missingFid}, + Selections: []string{"name"}, + Filter: &volume_server_pb.QueryRequest_Filter{}, + InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ + JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{}, + }, + }) + if err == nil { + _, err = missingStream.Recv() + } + if err == nil { + t.Fatalf("Query should fail for missing file id volume") + } +} + +func TestScrubVolumeAutoSelectAndNotImplementedModes(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeIDA = uint32(62) + const volumeIDB = uint32(63) + framework.AllocateVolume(t, grpcClient, volumeIDA, "") + framework.AllocateVolume(t, grpcClient, volumeIDB, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + autoResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ + Mode: volume_server_pb.VolumeScrubMode_INDEX, + }) + if err != nil { + t.Fatalf("ScrubVolume auto-select failed: %v", err) + } + if autoResp.GetTotalVolumes() < 2 { + t.Fatalf("ScrubVolume auto-select expected at least 2 volumes, got %d", autoResp.GetTotalVolumes()) + } + + localResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ + VolumeIds: []uint32{volumeIDA}, + Mode: volume_server_pb.VolumeScrubMode_LOCAL, + }) + if err != nil { + t.Fatalf("ScrubVolume local mode failed: %v", err) + } + if localResp.GetTotalVolumes() != 1 { + t.Fatalf("ScrubVolume local mode expected total_volumes=1, got %d", localResp.GetTotalVolumes()) + } + if len(localResp.GetBrokenVolumeIds()) != 1 || localResp.GetBrokenVolumeIds()[0] != volumeIDA { + t.Fatalf("ScrubVolume local mode expected broken volume %d, got %v", volumeIDA, localResp.GetBrokenVolumeIds()) + } + if len(localResp.GetDetails()) == 0 || !strings.Contains(strings.Join(localResp.GetDetails(), " "), "not implemented") { + t.Fatalf("ScrubVolume local mode expected not-implemented details, got %v", localResp.GetDetails()) + } + + fullResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ + VolumeIds: []uint32{volumeIDA}, + Mode: volume_server_pb.VolumeScrubMode_FULL, + }) + if err != nil { + t.Fatalf("ScrubVolume full mode failed: %v", err) + } + if fullResp.GetTotalVolumes() != 1 { + t.Fatalf("ScrubVolume full mode expected total_volumes=1, got %d", fullResp.GetTotalVolumes()) + } + if len(fullResp.GetDetails()) == 0 || !strings.Contains(strings.Join(fullResp.GetDetails(), " "), "not implemented") { + t.Fatalf("ScrubVolume full mode expected not-implemented details, got %v", fullResp.GetDetails()) + } +} + +func TestQueryJsonSuccessAndCsvNoOutput(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(64) + const needleID = uint64(777001) + const cookie = uint32(0xAABBCCDD) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + jsonLines := []byte("{\"score\":3}\n{\"score\":12}\n{\"score\":18}\n") + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, cookie) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + queryStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ + FromFileIds: []string{fid}, + Selections: []string{"score"}, + Filter: &volume_server_pb.QueryRequest_Filter{ + Field: "score", + Operand: ">", + Value: "10", + }, + InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ + JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"}, + }, + }) + if err != nil { + t.Fatalf("Query json start failed: %v", err) + } + + firstStripe, err := queryStream.Recv() + if err != nil { + t.Fatalf("Query json recv failed: %v", err) + } + records := string(firstStripe.GetRecords()) + if !strings.Contains(records, "score:12") || !strings.Contains(records, "score:18") { + t.Fatalf("Query json records missing expected filtered scores: %q", records) + } + if strings.Contains(records, "score:3") { + t.Fatalf("Query json records should not include filtered-out score: %q", records) + } + _, err = queryStream.Recv() + if err != io.EOF { + t.Fatalf("Query json expected EOF after first stripe, got: %v", err) + } + + csvStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ + FromFileIds: []string{fid}, + Selections: []string{"score"}, + Filter: &volume_server_pb.QueryRequest_Filter{}, + InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ + CsvInput: &volume_server_pb.QueryRequest_InputSerialization_CSVInput{}, + }, + }) + if err != nil { + t.Fatalf("Query csv start failed: %v", err) + } + _, err = csvStream.Recv() + if err != io.EOF { + t.Fatalf("Query csv expected EOF with no rows, got: %v", err) + } +} + +func TestQueryJsonNoMatchReturnsEmptyStripe(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(65) + const needleID = uint64(777002) + const cookie = uint32(0xABABCDCD) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + jsonLines := []byte("{\"score\":1}\n{\"score\":2}\n") + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, cookie) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + queryStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ + FromFileIds: []string{fid}, + Selections: []string{"score"}, + Filter: &volume_server_pb.QueryRequest_Filter{ + Field: "score", + Operand: ">", + Value: "100", + }, + InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ + JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"}, + }, + }) + if err != nil { + t.Fatalf("Query json no-match start failed: %v", err) + } + + firstStripe, err := queryStream.Recv() + if err != nil { + t.Fatalf("Query json no-match recv failed: %v", err) + } + if len(firstStripe.GetRecords()) != 0 { + t.Fatalf("Query json no-match expected empty records stripe, got: %q", string(firstStripe.GetRecords())) + } + + _, err = queryStream.Recv() + if err != io.EOF { + t.Fatalf("Query json no-match expected EOF after first empty stripe, got: %v", err) + } +} + +func TestQueryCookieMismatchReturnsEOFNoResults(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(66) + const needleID = uint64(777003) + const cookie = uint32(0xCDCDABAB) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + jsonLines := []byte("{\"score\":7}\n{\"score\":8}\n") + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, cookie) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != 201 { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + wrongCookieFid := framework.NewFileID(volumeID, needleID, cookie+1) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ + FromFileIds: []string{wrongCookieFid}, + Selections: []string{"score"}, + Filter: &volume_server_pb.QueryRequest_Filter{ + Field: "score", + Operand: ">", + Value: "0", + }, + InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ + JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"}, + }, + }) + if err != nil { + t.Fatalf("Query start for cookie mismatch should not fail immediately, got: %v", err) + } + + _, err = stream.Recv() + if err != io.EOF { + t.Fatalf("Query cookie mismatch expected EOF with no streamed records, got: %v", err) + } +} diff --git a/test/volume_server/grpc/tail_test.go b/test/volume_server/grpc/tail_test.go new file mode 100644 index 000000000..09657edb5 --- /dev/null +++ b/test/volume_server/grpc/tail_test.go @@ -0,0 +1,206 @@ +package volume_server_grpc_test + +import ( + "bytes" + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestVolumeTailSenderMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{VolumeId: 77777, SinceNs: 0, IdleTimeoutSeconds: 1}) + if err == nil { + _, err = stream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found volume") { + t.Fatalf("VolumeTailSender missing-volume error mismatch: %v", err) + } +} + +func TestVolumeTailSenderHeartbeatThenEOF(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(71) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{ + VolumeId: volumeID, + SinceNs: 0, + IdleTimeoutSeconds: 1, + }) + if err != nil { + t.Fatalf("VolumeTailSender start failed: %v", err) + } + + msg, err := stream.Recv() + if err != nil { + t.Fatalf("VolumeTailSender first recv failed: %v", err) + } + if !msg.GetIsLastChunk() { + t.Fatalf("expected first tail message to be heartbeat IsLastChunk=true") + } + + _, err = stream.Recv() + if err != io.EOF { + t.Fatalf("expected EOF after idle timeout drain, got: %v", err) + } +} + +func TestVolumeTailReceiverMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{VolumeId: 88888, SourceVolumeServer: clusterHarness.VolumeServerAddress(), SinceNs: 0, IdleTimeoutSeconds: 1}) + if err == nil || !strings.Contains(err.Error(), "receiver not found volume") { + t.Fatalf("VolumeTailReceiver missing-volume error mismatch: %v", err) + } +} + +func TestVolumeTailReceiverReplicatesSourceUpdates(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1()) + sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer sourceConn.Close() + destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) + defer destConn.Close() + + const volumeID = uint32(72) + framework.AllocateVolume(t, sourceClient, volumeID, "") + framework.AllocateVolume(t, destClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 880003, 0x3456789A) + payload := []byte("tail-receiver-replicates-source-updates") + + sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, sourceUploadResp) + if sourceUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("source upload expected 201, got %d", sourceUploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := destClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{ + VolumeId: volumeID, + SourceVolumeServer: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1], + SinceNs: 0, + IdleTimeoutSeconds: 1, + }) + if err != nil { + t.Fatalf("VolumeTailReceiver success path failed: %v", err) + } + + destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid) + destReadBody := framework.ReadAllAndClose(t, destReadResp) + if destReadResp.StatusCode != http.StatusOK { + t.Fatalf("destination read after tail receive expected 200, got %d", destReadResp.StatusCode) + } + if string(destReadBody) != string(payload) { + t.Fatalf("destination tail-received payload mismatch: got %q want %q", string(destReadBody), string(payload)) + } +} + +func TestVolumeTailSenderLargeNeedleChunking(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(73) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 880004, 0x456789AB) + largePayload := bytes.Repeat([]byte("L"), 2*1024*1024+128*1024) + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{ + VolumeId: volumeID, + SinceNs: 0, + IdleTimeoutSeconds: 1, + }) + if err != nil { + t.Fatalf("VolumeTailSender start failed: %v", err) + } + + dataChunkCount := 0 + sawNonLastDataChunk := false + sawLastDataChunk := false + for { + msg, recvErr := stream.Recv() + if recvErr == io.EOF { + break + } + if recvErr != nil { + t.Fatalf("VolumeTailSender recv failed: %v", recvErr) + } + if len(msg.GetNeedleBody()) == 0 { + continue + } + dataChunkCount++ + if msg.GetIsLastChunk() { + sawLastDataChunk = true + } else { + sawNonLastDataChunk = true + } + } + + if dataChunkCount < 2 { + t.Fatalf("VolumeTailSender expected multiple chunks for large needle, got %d", dataChunkCount) + } + if !sawNonLastDataChunk { + t.Fatalf("VolumeTailSender expected at least one non-last data chunk") + } + if !sawLastDataChunk { + t.Fatalf("VolumeTailSender expected a final data chunk marked IsLastChunk=true") + } +} diff --git a/test/volume_server/grpc/tiering_remote_test.go b/test/volume_server/grpc/tiering_remote_test.go new file mode 100644 index 000000000..db36e7cfd --- /dev/null +++ b/test/volume_server/grpc/tiering_remote_test.go @@ -0,0 +1,236 @@ +package volume_server_grpc_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestFetchAndWriteNeedleMaintenanceAndMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{ + VolumeId: 98781, + NeedleId: 1, + }) + if err == nil || !strings.Contains(err.Error(), "not found volume id") { + t.Fatalf("FetchAndWriteNeedle missing-volume error mismatch: %v", err) + } + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: true, + Version: stateResp.GetState().GetVersion(), + }, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + _, err = grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{ + VolumeId: 1, + NeedleId: 1, + }) + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("FetchAndWriteNeedle maintenance error mismatch: %v", err) + } +} + +func TestFetchAndWriteNeedleInvalidRemoteConfig(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(88) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{ + VolumeId: volumeID, + NeedleId: 1, + Cookie: 1, + Size: 1, + RemoteConf: &remote_pb.RemoteConf{ + Name: "it-invalid-remote", + Type: "does-not-exist", + }, + RemoteLocation: &remote_pb.RemoteStorageLocation{ + Name: "it-invalid-remote", + Path: "/test", + }, + }) + if err == nil || !strings.Contains(err.Error(), "get remote client") { + t.Fatalf("FetchAndWriteNeedle invalid-remote error mismatch: %v", err) + } +} + +func TestVolumeTierMoveDatToRemoteErrorPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(85) + const collection = "tier-collection" + framework.AllocateVolume(t, grpcClient, volumeID, collection) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + missingStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ + VolumeId: 98782, + Collection: collection, + DestinationBackendName: "dummy", + }) + if err == nil { + _, err = missingStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeTierMoveDatToRemote missing-volume error mismatch: %v", err) + } + + mismatchStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ + VolumeId: volumeID, + Collection: "wrong-collection", + DestinationBackendName: "dummy", + }) + if err == nil { + _, err = mismatchStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "unexpected input") { + t.Fatalf("VolumeTierMoveDatToRemote collection mismatch error mismatch: %v", err) + } + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{ + Maintenance: true, + Version: stateResp.GetState().GetVersion(), + }, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + maintenanceStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ + VolumeId: volumeID, + Collection: collection, + DestinationBackendName: "dummy", + }) + if err == nil { + _, err = maintenanceStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("VolumeTierMoveDatToRemote maintenance error mismatch: %v", err) + } +} + +func TestVolumeTierMoveDatToRemoteMissingBackend(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(89) + const collection = "tier-missing-backend" + framework.AllocateVolume(t, grpcClient, volumeID, collection) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ + VolumeId: volumeID, + Collection: collection, + DestinationBackendName: "definitely-missing-backend", + }) + if err == nil { + _, err = stream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "destination definitely-missing-backend not found") { + t.Fatalf("VolumeTierMoveDatToRemote missing-backend error mismatch: %v", err) + } +} + +func TestVolumeTierMoveDatFromRemoteErrorPaths(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(86) + const collection = "tier-download-collection" + framework.AllocateVolume(t, grpcClient, volumeID, collection) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + missingStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{ + VolumeId: 98783, + Collection: collection, + }) + if err == nil { + _, err = missingStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("VolumeTierMoveDatFromRemote missing-volume error mismatch: %v", err) + } + + mismatchStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{ + VolumeId: volumeID, + Collection: "wrong-collection", + }) + if err == nil { + _, err = mismatchStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "unexpected input") { + t.Fatalf("VolumeTierMoveDatFromRemote collection mismatch error mismatch: %v", err) + } + + localDiskStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{ + VolumeId: volumeID, + Collection: collection, + }) + if err == nil { + _, err = localDiskStream.Recv() + } + if err == nil || !strings.Contains(err.Error(), "already on local disk") { + t.Fatalf("VolumeTierMoveDatFromRemote local-disk error mismatch: %v", err) + } +} diff --git a/test/volume_server/grpc/vacuum_test.go b/test/volume_server/grpc/vacuum_test.go new file mode 100644 index 000000000..ea986fed2 --- /dev/null +++ b/test/volume_server/grpc/vacuum_test.go @@ -0,0 +1,87 @@ +package volume_server_grpc_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func TestVacuumVolumeCheckSuccessAndMissingVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(31) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := grpcClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{VolumeId: volumeID}) + if err != nil { + t.Fatalf("VacuumVolumeCheck existing volume failed: %v", err) + } + if resp.GetGarbageRatio() < 0 || resp.GetGarbageRatio() > 1 { + t.Fatalf("unexpected garbage ratio: %f", resp.GetGarbageRatio()) + } + + _, err = grpcClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{VolumeId: 99999}) + if err == nil { + t.Fatalf("VacuumVolumeCheck should fail for missing volume") + } +} + +func TestVacuumMaintenanceModeRejections(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) + if err != nil { + t.Fatalf("GetState failed: %v", err) + } + _, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ + State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, + }) + if err != nil { + t.Fatalf("SetState maintenance=true failed: %v", err) + } + + assertMaintenanceErr := func(name string, err error) { + t.Helper() + if err == nil { + t.Fatalf("%s should fail in maintenance mode", name) + } + if !strings.Contains(err.Error(), "maintenance mode") { + t.Fatalf("%s expected maintenance mode error, got: %v", name, err) + } + } + + compactStream, err := grpcClient.VacuumVolumeCompact(ctx, &volume_server_pb.VacuumVolumeCompactRequest{VolumeId: 31}) + if err == nil { + _, err = compactStream.Recv() + } + assertMaintenanceErr("VacuumVolumeCompact", err) + + _, err = grpcClient.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: 31}) + assertMaintenanceErr("VacuumVolumeCommit", err) + + _, err = grpcClient.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{VolumeId: 31}) + assertMaintenanceErr("VacuumVolumeCleanup", err) +} diff --git a/test/volume_server/http/admin_test.go b/test/volume_server/http/admin_test.go new file mode 100644 index 000000000..be4445ebc --- /dev/null +++ b/test/volume_server/http/admin_test.go @@ -0,0 +1,174 @@ +package volume_server_http_test + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/util/request_id" +) + +func TestAdminStatusAndHealthz(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + client := framework.NewHTTPClient() + + statusReq, err := http.NewRequest(http.MethodGet, cluster.VolumeAdminURL()+"/status", nil) + if err != nil { + t.Fatalf("create status request: %v", err) + } + statusReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-1") + + statusResp := framework.DoRequest(t, client, statusReq) + statusBody := framework.ReadAllAndClose(t, statusResp) + + if statusResp.StatusCode != http.StatusOK { + t.Fatalf("expected /status code 200, got %d, body: %s", statusResp.StatusCode, string(statusBody)) + } + if got := statusResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") { + t.Fatalf("expected /status Server header to contain SeaweedFS Volume, got %q", got) + } + if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-1" { + t.Fatalf("expected echoed request id, got %q", got) + } + + var payload map[string]interface{} + if err := json.Unmarshal(statusBody, &payload); err != nil { + t.Fatalf("decode status response: %v", err) + } + for _, field := range []string{"Version", "DiskStatuses", "Volumes"} { + if _, found := payload[field]; !found { + t.Fatalf("status payload missing field %q", field) + } + } + + healthReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz") + healthReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-2") + healthResp := framework.DoRequest(t, client, healthReq) + _ = framework.ReadAllAndClose(t, healthResp) + if healthResp.StatusCode != http.StatusOK { + t.Fatalf("expected /healthz code 200, got %d", healthResp.StatusCode) + } + if got := healthResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") { + t.Fatalf("expected /healthz Server header to contain SeaweedFS Volume, got %q", got) + } + if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-2" { + t.Fatalf("expected /healthz echoed request id, got %q", got) + } + + uiResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html")) + uiBody := framework.ReadAllAndClose(t, uiResp) + if uiResp.StatusCode != http.StatusOK { + t.Fatalf("expected /ui/index.html code 200, got %d, body: %s", uiResp.StatusCode, string(uiBody)) + } + if !strings.Contains(strings.ToLower(string(uiBody)), "volume") { + t.Fatalf("ui page does not look like volume status page") + } +} + +func TestOptionsMethodsByPort(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P2()) + client := framework.NewHTTPClient() + + adminResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/")) + _ = framework.ReadAllAndClose(t, adminResp) + if adminResp.StatusCode != http.StatusOK { + t.Fatalf("admin OPTIONS expected 200, got %d", adminResp.StatusCode) + } + adminAllowed := adminResp.Header.Get("Access-Control-Allow-Methods") + for _, expected := range []string{"PUT", "POST", "GET", "DELETE", "OPTIONS"} { + if !strings.Contains(adminAllowed, expected) { + t.Fatalf("admin allow methods missing %q, got %q", expected, adminAllowed) + } + } + if adminResp.Header.Get("Access-Control-Allow-Headers") != "*" { + t.Fatalf("admin allow headers expected '*', got %q", adminResp.Header.Get("Access-Control-Allow-Headers")) + } + + publicResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodOptions, cluster.VolumePublicURL()+"/")) + _ = framework.ReadAllAndClose(t, publicResp) + if publicResp.StatusCode != http.StatusOK { + t.Fatalf("public OPTIONS expected 200, got %d", publicResp.StatusCode) + } + publicAllowed := publicResp.Header.Get("Access-Control-Allow-Methods") + if !strings.Contains(publicAllowed, "GET") || !strings.Contains(publicAllowed, "OPTIONS") { + t.Fatalf("public allow methods expected GET and OPTIONS, got %q", publicAllowed) + } + if strings.Contains(publicAllowed, "POST") { + t.Fatalf("public allow methods should not include POST, got %q", publicAllowed) + } + if publicResp.Header.Get("Access-Control-Allow-Headers") != "*" { + t.Fatalf("public allow headers expected '*', got %q", publicResp.Header.Get("Access-Control-Allow-Headers")) + } +} + +func TestOptionsWithOriginIncludesCorsHeaders(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P2()) + client := framework.NewHTTPClient() + + adminReq := mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/") + adminReq.Header.Set("Origin", "https://example.com") + adminResp := framework.DoRequest(t, client, adminReq) + _ = framework.ReadAllAndClose(t, adminResp) + if adminResp.StatusCode != http.StatusOK { + t.Fatalf("admin OPTIONS expected 200, got %d", adminResp.StatusCode) + } + if adminResp.Header.Get("Access-Control-Allow-Origin") != "*" { + t.Fatalf("admin OPTIONS expected Access-Control-Allow-Origin=*, got %q", adminResp.Header.Get("Access-Control-Allow-Origin")) + } + if adminResp.Header.Get("Access-Control-Allow-Credentials") != "true" { + t.Fatalf("admin OPTIONS expected Access-Control-Allow-Credentials=true, got %q", adminResp.Header.Get("Access-Control-Allow-Credentials")) + } + + publicReq := mustNewRequest(t, http.MethodOptions, cluster.VolumePublicURL()+"/") + publicReq.Header.Set("Origin", "https://example.com") + publicResp := framework.DoRequest(t, client, publicReq) + _ = framework.ReadAllAndClose(t, publicResp) + if publicResp.StatusCode != http.StatusOK { + t.Fatalf("public OPTIONS expected 200, got %d", publicResp.StatusCode) + } + if publicResp.Header.Get("Access-Control-Allow-Origin") != "*" { + t.Fatalf("public OPTIONS expected Access-Control-Allow-Origin=*, got %q", publicResp.Header.Get("Access-Control-Allow-Origin")) + } + if publicResp.Header.Get("Access-Control-Allow-Credentials") != "true" { + t.Fatalf("public OPTIONS expected Access-Control-Allow-Credentials=true, got %q", publicResp.Header.Get("Access-Control-Allow-Credentials")) + } +} + +func TestUiIndexNotExposedWhenJwtSigningEnabled(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P3()) + client := framework.NewHTTPClient() + + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html")) + body := framework.ReadAllAndClose(t, resp) + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected /ui/index.html to be gated by auth under JWT profile (401), got %d body=%s", resp.StatusCode, string(body)) + } +} + +func mustNewRequest(t testing.TB, method, url string) *http.Request { + t.Helper() + req, err := http.NewRequest(method, url, nil) + if err != nil { + t.Fatalf("create request %s %s: %v", method, url, err) + } + return req +} diff --git a/test/volume_server/http/auth_test.go b/test/volume_server/http/auth_test.go new file mode 100644 index 000000000..5b093bba1 --- /dev/null +++ b/test/volume_server/http/auth_test.go @@ -0,0 +1,419 @@ +package volume_server_http_test + +import ( + "bytes" + "net/http" + "testing" + "time" + + jwt "github.com/golang-jwt/jwt/v5" + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/security" +) + +func TestJWTAuthForWriteAndRead(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(51) + const needleID = uint64(123456) + const cookie = uint32(0xABCDEF12) + + framework.AllocateVolume(t, grpcClient, volumeID, "") + fid := framework.NewFileID(volumeID, needleID, cookie) + payload := []byte("jwt-protected-content") + client := framework.NewHTTPClient() + + unauthWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + unauthWriteResp := framework.DoRequest(t, client, unauthWrite) + _ = framework.ReadAllAndClose(t, unauthWriteResp) + if unauthWriteResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("unauthorized write expected 401, got %d", unauthWriteResp.StatusCode) + } + + invalidWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + invalidWrite.Header.Set("Authorization", "Bearer invalid") + invalidWriteResp := framework.DoRequest(t, client, invalidWrite) + _ = framework.ReadAllAndClose(t, invalidWriteResp) + if invalidWriteResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("invalid write token expected 401, got %d", invalidWriteResp.StatusCode) + } + + writeToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + authWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + authWrite.Header.Set("Authorization", "Bearer "+string(writeToken)) + authWriteResp := framework.DoRequest(t, client, authWrite) + _ = framework.ReadAllAndClose(t, authWriteResp) + if authWriteResp.StatusCode != http.StatusCreated { + t.Fatalf("authorized write expected 201, got %d", authWriteResp.StatusCode) + } + + unauthReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + unauthReadResp := framework.DoRequest(t, client, unauthReadReq) + _ = framework.ReadAllAndClose(t, unauthReadResp) + if unauthReadResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("unauthorized read expected 401, got %d", unauthReadResp.StatusCode) + } + + readToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) + authReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + authReadReq.Header.Set("Authorization", "Bearer "+string(readToken)) + authReadResp := framework.DoRequest(t, client, authReadReq) + authReadBody := framework.ReadAllAndClose(t, authReadResp) + if authReadResp.StatusCode != http.StatusOK { + t.Fatalf("authorized read expected 200, got %d", authReadResp.StatusCode) + } + if string(authReadBody) != string(payload) { + t.Fatalf("authorized read content mismatch: got %q want %q", string(authReadBody), string(payload)) + } +} + +func TestJWTAuthRejectsFidMismatch(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(52) + const needleID = uint64(223344) + const cookie = uint32(0x10203040) + const otherNeedleID = uint64(223345) + const otherCookie = uint32(0x50607080) + const wrongCookie = uint32(0x10203041) + + framework.AllocateVolume(t, grpcClient, volumeID, "") + fid := framework.NewFileID(volumeID, needleID, cookie) + otherFid := framework.NewFileID(volumeID, otherNeedleID, otherCookie) + payload := []byte("jwt-fid-mismatch-content") + client := framework.NewHTTPClient() + + writeTokenForOtherFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFid) + mismatchedWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + mismatchedWrite.Header.Set("Authorization", "Bearer "+string(writeTokenForOtherFid)) + mismatchedWriteResp := framework.DoRequest(t, client, mismatchedWrite) + _ = framework.ReadAllAndClose(t, mismatchedWriteResp) + if mismatchedWriteResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("write with mismatched fid token expected 401, got %d", mismatchedWriteResp.StatusCode) + } + + wrongCookieFid := framework.NewFileID(volumeID, needleID, wrongCookie) + writeTokenWrongCookie := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, wrongCookieFid) + wrongCookieWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + wrongCookieWrite.Header.Set("Authorization", "Bearer "+string(writeTokenWrongCookie)) + wrongCookieWriteResp := framework.DoRequest(t, client, wrongCookieWrite) + _ = framework.ReadAllAndClose(t, wrongCookieWriteResp) + if wrongCookieWriteResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("write with wrong-cookie fid token expected 401, got %d", wrongCookieWriteResp.StatusCode) + } + + writeTokenForFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + validWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + validWrite.Header.Set("Authorization", "Bearer "+string(writeTokenForFid)) + validWriteResp := framework.DoRequest(t, client, validWrite) + _ = framework.ReadAllAndClose(t, validWriteResp) + if validWriteResp.StatusCode != http.StatusCreated { + t.Fatalf("authorized write expected 201, got %d", validWriteResp.StatusCode) + } + + readTokenForOtherFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFid) + mismatchedReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + mismatchedReadReq.Header.Set("Authorization", "Bearer "+string(readTokenForOtherFid)) + mismatchedReadResp := framework.DoRequest(t, client, mismatchedReadReq) + _ = framework.ReadAllAndClose(t, mismatchedReadResp) + if mismatchedReadResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("read with mismatched fid token expected 401, got %d", mismatchedReadResp.StatusCode) + } + + readTokenWrongCookie := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, wrongCookieFid) + wrongCookieReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + wrongCookieReadReq.Header.Set("Authorization", "Bearer "+string(readTokenWrongCookie)) + wrongCookieReadResp := framework.DoRequest(t, client, wrongCookieReadReq) + _ = framework.ReadAllAndClose(t, wrongCookieReadResp) + if wrongCookieReadResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("read with wrong-cookie fid token expected 401, got %d", wrongCookieReadResp.StatusCode) + } +} + +func newUploadRequest(t testing.TB, url string, payload []byte) *http.Request { + t.Helper() + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(payload)) + if err != nil { + t.Fatalf("create upload request %s: %v", url, err) + } + req.Header.Set("Content-Type", "application/octet-stream") + return req +} + +func TestJWTAuthRejectsExpiredTokens(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(53) + const needleID = uint64(334455) + const cookie = uint32(0x22334455) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + payload := []byte("expired-token-content") + client := framework.NewHTTPClient() + + expiredWriteToken := mustGenExpiredToken(t, []byte(profile.JWTSigningKey), fid) + writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + writeReq.Header.Set("Authorization", "Bearer "+expiredWriteToken) + writeResp := framework.DoRequest(t, client, writeReq) + _ = framework.ReadAllAndClose(t, writeResp) + if writeResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expired write token expected 401, got %d", writeResp.StatusCode) + } + + // Seed data with a valid token so read auth path can be exercised against existing content. + validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + validWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + validWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) + validWriteResp := framework.DoRequest(t, client, validWriteReq) + _ = framework.ReadAllAndClose(t, validWriteResp) + if validWriteResp.StatusCode != http.StatusCreated { + t.Fatalf("valid write expected 201, got %d", validWriteResp.StatusCode) + } + + expiredReadToken := mustGenExpiredToken(t, []byte(profile.JWTReadKey), fid) + readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + readReq.Header.Set("Authorization", "Bearer "+expiredReadToken) + readResp := framework.DoRequest(t, client, readReq) + _ = framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expired read token expected 401, got %d", readResp.StatusCode) + } +} + +func TestJWTAuthViaQueryParamAndCookie(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(54) + const needleID = uint64(445566) + const cookie = uint32(0x31415926) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + payload := []byte("jwt-query-cookie-content") + client := framework.NewHTTPClient() + + writeToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(writeToken), payload) + writeResp := framework.DoRequest(t, client, writeReq) + _ = framework.ReadAllAndClose(t, writeResp) + if writeResp.StatusCode != http.StatusCreated { + t.Fatalf("query-jwt write expected 201, got %d", writeResp.StatusCode) + } + + readToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) + readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(readToken)}) + readResp := framework.DoRequest(t, client, readReq) + readBody := framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusOK { + t.Fatalf("cookie-jwt read expected 200, got %d", readResp.StatusCode) + } + if string(readBody) != string(payload) { + t.Fatalf("cookie-jwt read body mismatch: got %q want %q", string(readBody), string(payload)) + } +} + +func TestJWTTokenSourcePrecedenceQueryOverHeader(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(55) + const needleID = uint64(556677) + const cookie = uint32(0x99887766) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1) + payload := []byte("jwt-precedence-content") + client := framework.NewHTTPClient() + + validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + invalidWriteQueryToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID) + writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidWriteQueryToken), payload) + writeReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) + writeResp := framework.DoRequest(t, client, writeReq) + _ = framework.ReadAllAndClose(t, writeResp) + if writeResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("query token should take precedence over header token for write, expected 401 got %d", writeResp.StatusCode) + } + + // Seed data with valid write token, then exercise read precedence. + seedWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + seedWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) + seedWriteResp := framework.DoRequest(t, client, seedWriteReq) + _ = framework.ReadAllAndClose(t, seedWriteResp) + if seedWriteResp.StatusCode != http.StatusCreated { + t.Fatalf("seed write expected 201, got %d", seedWriteResp.StatusCode) + } + + validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) + invalidReadQueryToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID) + readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidReadQueryToken)) + readReq.Header.Set("Authorization", "Bearer "+string(validReadToken)) + readResp := framework.DoRequest(t, client, readReq) + _ = framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("query token should take precedence over header token for read, expected 401 got %d", readResp.StatusCode) + } +} + +func TestJWTTokenSourcePrecedenceHeaderOverCookie(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(56) + const needleID = uint64(667788) + const cookie = uint32(0x11229988) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1) + payload := []byte("jwt-precedence-header-cookie") + client := framework.NewHTTPClient() + + validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + invalidCookieWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID) + writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + writeReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) + writeReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieWriteToken)}) + writeResp := framework.DoRequest(t, client, writeReq) + _ = framework.ReadAllAndClose(t, writeResp) + if writeResp.StatusCode != http.StatusCreated { + t.Fatalf("header token should take precedence over cookie token for write, expected 201 got %d", writeResp.StatusCode) + } + + validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) + invalidCookieReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID) + readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + readReq.Header.Set("Authorization", "Bearer "+string(validReadToken)) + readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieReadToken)}) + readResp := framework.DoRequest(t, client, readReq) + readBody := framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusOK { + t.Fatalf("header token should take precedence over cookie token for read, expected 200 got %d", readResp.StatusCode) + } + if string(readBody) != string(payload) { + t.Fatalf("header-over-cookie read body mismatch: got %q want %q", string(readBody), string(payload)) + } +} + +func TestJWTTokenSourcePrecedenceQueryOverCookie(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P3() + clusterHarness := framework.StartSingleVolumeCluster(t, profile) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(57) + const needleID = uint64(778899) + const cookie = uint32(0x88776655) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, needleID, cookie) + otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1) + payload := []byte("jwt-precedence-query-cookie") + client := framework.NewHTTPClient() + + validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) + invalidQueryWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID) + writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidQueryWriteToken), payload) + writeReq.AddCookie(&http.Cookie{Name: "AT", Value: string(validWriteToken)}) + writeResp := framework.DoRequest(t, client, writeReq) + _ = framework.ReadAllAndClose(t, writeResp) + if writeResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("query token should take precedence over cookie token for write, expected 401 got %d", writeResp.StatusCode) + } + + // Seed data with valid write token so read precedence can be exercised. + seedWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + seedWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) + seedWriteResp := framework.DoRequest(t, client, seedWriteReq) + _ = framework.ReadAllAndClose(t, seedWriteResp) + if seedWriteResp.StatusCode != http.StatusCreated { + t.Fatalf("seed write expected 201, got %d", seedWriteResp.StatusCode) + } + + validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) + invalidQueryReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID) + readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidQueryReadToken)) + readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(validReadToken)}) + readResp := framework.DoRequest(t, client, readReq) + _ = framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusUnauthorized { + t.Fatalf("query token should take precedence over cookie token for read, expected 401 got %d", readResp.StatusCode) + } + + // Validate positive path: valid query token should succeed even if cookie token is invalid. + validQueryReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(validReadToken)) + invalidCookieReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID) + validQueryReadReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieReadToken)}) + validQueryReadResp := framework.DoRequest(t, client, validQueryReadReq) + validQueryReadBody := framework.ReadAllAndClose(t, validQueryReadResp) + if validQueryReadResp.StatusCode != http.StatusOK { + t.Fatalf("valid query token should succeed over invalid cookie token, expected 200 got %d", validQueryReadResp.StatusCode) + } + if string(validQueryReadBody) != string(payload) { + t.Fatalf("query-over-cookie read body mismatch: got %q want %q", string(validQueryReadBody), string(payload)) + } +} + +func mustGenExpiredToken(t testing.TB, key []byte, fid string) string { + t.Helper() + claims := security.SeaweedFileIdClaims{ + Fid: fid, + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(-1 * time.Minute)), + }, + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + signed, err := token.SignedString(key) + if err != nil { + t.Fatalf("sign expired token: %v", err) + } + return signed +} diff --git a/test/volume_server/http/chunk_manifest_test.go b/test/volume_server/http/chunk_manifest_test.go new file mode 100644 index 000000000..d3806d7f4 --- /dev/null +++ b/test/volume_server/http/chunk_manifest_test.go @@ -0,0 +1,232 @@ +package volume_server_http_test + +import ( + "bytes" + "encoding/json" + "net/http" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/operation" +) + +func TestChunkManifestExpansionAndBypass(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(102) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + + chunkFID := framework.NewFileID(volumeID, 772005, 0x5E6F7081) + chunkPayload := []byte("chunk-manifest-expanded-content") + chunkUploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID, chunkPayload) + _ = framework.ReadAllAndClose(t, chunkUploadResp) + if chunkUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("chunk upload expected 201, got %d", chunkUploadResp.StatusCode) + } + + manifest := &operation.ChunkManifest{ + Name: "manifest.bin", + Mime: "application/octet-stream", + Size: int64(len(chunkPayload)), + Chunks: []*operation.ChunkInfo{ + { + Fid: chunkFID, + Offset: 0, + Size: int64(len(chunkPayload)), + }, + }, + } + manifestBytes, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal chunk manifest: %v", err) + } + + manifestFID := framework.NewFileID(volumeID, 772006, 0x6F708192) + manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes)) + if err != nil { + t.Fatalf("create manifest upload request: %v", err) + } + manifestUploadReq.Header.Set("Content-Type", "application/json") + manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq) + _ = framework.ReadAllAndClose(t, manifestUploadResp) + if manifestUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode) + } + + expandedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), manifestFID) + expandedReadBody := framework.ReadAllAndClose(t, expandedReadResp) + if expandedReadResp.StatusCode != http.StatusOK { + t.Fatalf("manifest expanded read expected 200, got %d", expandedReadResp.StatusCode) + } + if string(expandedReadBody) != string(chunkPayload) { + t.Fatalf("manifest expanded read mismatch: got %q want %q", string(expandedReadBody), string(chunkPayload)) + } + if expandedReadResp.Header.Get("X-File-Store") != "chunked" { + t.Fatalf("manifest expanded read expected X-File-Store=chunked, got %q", expandedReadResp.Header.Get("X-File-Store")) + } + + bypassReadResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=false")) + bypassReadBody := framework.ReadAllAndClose(t, bypassReadResp) + if bypassReadResp.StatusCode != http.StatusOK { + t.Fatalf("manifest bypass read expected 200, got %d", bypassReadResp.StatusCode) + } + if bypassReadResp.Header.Get("X-File-Store") != "" { + t.Fatalf("manifest bypass read expected empty X-File-Store header, got %q", bypassReadResp.Header.Get("X-File-Store")) + } + + var gotManifest operation.ChunkManifest + if err = json.Unmarshal(bypassReadBody, &gotManifest); err != nil { + t.Fatalf("manifest bypass read expected JSON payload, got decode error: %v body=%q", err, string(bypassReadBody)) + } + if len(gotManifest.Chunks) != 1 || gotManifest.Chunks[0].Fid != chunkFID { + t.Fatalf("manifest bypass read payload mismatch: %+v", gotManifest) + } +} + +func TestChunkManifestDeleteRemovesChildChunks(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(104) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + + chunkFID := framework.NewFileID(volumeID, 772008, 0x8192A3B4) + chunkPayload := []byte("chunk-manifest-delete-content") + chunkUploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID, chunkPayload) + _ = framework.ReadAllAndClose(t, chunkUploadResp) + if chunkUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("chunk upload expected 201, got %d", chunkUploadResp.StatusCode) + } + + manifest := &operation.ChunkManifest{ + Name: "manifest-delete.bin", + Mime: "application/octet-stream", + Size: int64(len(chunkPayload)), + Chunks: []*operation.ChunkInfo{ + { + Fid: chunkFID, + Offset: 0, + Size: int64(len(chunkPayload)), + }, + }, + } + manifestBytes, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal chunk manifest: %v", err) + } + + manifestFID := framework.NewFileID(volumeID, 772009, 0x92A3B4C5) + manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes)) + if err != nil { + t.Fatalf("create manifest upload request: %v", err) + } + manifestUploadReq.Header.Set("Content-Type", "application/json") + manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq) + _ = framework.ReadAllAndClose(t, manifestUploadResp) + if manifestUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode) + } + + deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+manifestFID)) + deleteBody := framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("manifest delete expected 202, got %d", deleteResp.StatusCode) + } + var deleteResult map[string]int64 + if err = json.Unmarshal(deleteBody, &deleteResult); err != nil { + t.Fatalf("decode manifest delete response: %v body=%q", err, string(deleteBody)) + } + if deleteResult["size"] != int64(len(chunkPayload)) { + t.Fatalf("manifest delete expected size=%d, got %d", len(chunkPayload), deleteResult["size"]) + } + + manifestReadAfterDelete := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), manifestFID) + _ = framework.ReadAllAndClose(t, manifestReadAfterDelete) + if manifestReadAfterDelete.StatusCode != http.StatusNotFound { + t.Fatalf("manifest read after delete expected 404, got %d", manifestReadAfterDelete.StatusCode) + } + + chunkReadAfterDelete := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID) + _ = framework.ReadAllAndClose(t, chunkReadAfterDelete) + if chunkReadAfterDelete.StatusCode != http.StatusNotFound { + t.Fatalf("chunk read after manifest delete expected 404, got %d", chunkReadAfterDelete.StatusCode) + } +} + +func TestChunkManifestDeleteFailsWhenChildDeletionFails(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(105) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + manifest := &operation.ChunkManifest{ + Name: "manifest-delete-failure.bin", + Mime: "application/octet-stream", + Size: 1, + Chunks: []*operation.ChunkInfo{ + { + Fid: "not-a-valid-fid", + Offset: 0, + Size: 1, + }, + }, + } + manifestBytes, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal chunk manifest: %v", err) + } + + manifestFID := framework.NewFileID(volumeID, 772010, 0xA3B4C5D6) + manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes)) + if err != nil { + t.Fatalf("create manifest upload request: %v", err) + } + manifestUploadReq.Header.Set("Content-Type", "application/json") + manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq) + _ = framework.ReadAllAndClose(t, manifestUploadResp) + if manifestUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode) + } + + deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+manifestFID)) + deleteBody := framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusInternalServerError { + t.Fatalf("manifest delete with invalid child fid expected 500, got %d body=%q", deleteResp.StatusCode, string(deleteBody)) + } + + manifestBypassRead := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=false")) + manifestBypassBody := framework.ReadAllAndClose(t, manifestBypassRead) + if manifestBypassRead.StatusCode != http.StatusOK { + t.Fatalf("manifest bypass read after failed delete expected 200, got %d", manifestBypassRead.StatusCode) + } + var gotManifest operation.ChunkManifest + if err = json.Unmarshal(manifestBypassBody, &gotManifest); err != nil { + t.Fatalf("manifest bypass read expected JSON payload, got decode error: %v body=%q", err, string(manifestBypassBody)) + } + if len(gotManifest.Chunks) != 1 || gotManifest.Chunks[0].Fid != "not-a-valid-fid" { + t.Fatalf("manifest payload mismatch after failed delete: %+v", gotManifest) + } +} diff --git a/test/volume_server/http/compressed_read_test.go b/test/volume_server/http/compressed_read_test.go new file mode 100644 index 000000000..8a9ac5c41 --- /dev/null +++ b/test/volume_server/http/compressed_read_test.go @@ -0,0 +1,97 @@ +package volume_server_http_test + +import ( + "bytes" + "compress/gzip" + "io" + "net/http" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func gzipData(t testing.TB, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write(data); err != nil { + t.Fatalf("gzip write: %v", err) + } + if err := zw.Close(); err != nil { + t.Fatalf("gzip close: %v", err) + } + return buf.Bytes() +} + +func gunzipData(t testing.TB, data []byte) []byte { + t.Helper() + zr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("gunzip new reader: %v", err) + } + defer zr.Close() + out, err := io.ReadAll(zr) + if err != nil { + t.Fatalf("gunzip read: %v", err) + } + return out +} + +func TestCompressedReadAcceptEncodingMatrix(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(103) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 772007, 0x708192A3) + plainPayload := []byte("compressed-read-accept-encoding-matrix-content-compressed-read-accept-encoding-matrix-content") + compressedPayload := gzipData(t, plainPayload) + + uploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+fid, bytes.NewReader(compressedPayload)) + if err != nil { + t.Fatalf("create compressed upload request: %v", err) + } + uploadReq.Header.Set("Content-Type", "text/plain") + uploadReq.Header.Set("Content-Encoding", "gzip") + uploadResp := framework.DoRequest(t, client, uploadReq) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("compressed upload expected 201, got %d", uploadResp.StatusCode) + } + + gzipReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + gzipReadReq.Header.Set("Accept-Encoding", "gzip") + gzipReadResp := framework.DoRequest(t, client, gzipReadReq) + gzipReadBody := framework.ReadAllAndClose(t, gzipReadResp) + if gzipReadResp.StatusCode != http.StatusOK { + t.Fatalf("gzip-accepted read expected 200, got %d", gzipReadResp.StatusCode) + } + if gzipReadResp.Header.Get("Content-Encoding") != "gzip" { + t.Fatalf("gzip-accepted read expected Content-Encoding=gzip, got %q", gzipReadResp.Header.Get("Content-Encoding")) + } + if string(gunzipData(t, gzipReadBody)) != string(plainPayload) { + t.Fatalf("gzip-accepted read body mismatch after gunzip") + } + + identityReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + identityReadReq.Header.Set("Accept-Encoding", "identity") + identityReadResp := framework.DoRequest(t, client, identityReadReq) + identityReadBody := framework.ReadAllAndClose(t, identityReadResp) + if identityReadResp.StatusCode != http.StatusOK { + t.Fatalf("identity read expected 200, got %d", identityReadResp.StatusCode) + } + if identityReadResp.Header.Get("Content-Encoding") != "" { + t.Fatalf("identity read expected no Content-Encoding header, got %q", identityReadResp.Header.Get("Content-Encoding")) + } + if string(identityReadBody) != string(plainPayload) { + t.Fatalf("identity read body mismatch: got %q want %q", string(identityReadBody), string(plainPayload)) + } +} diff --git a/test/volume_server/http/headers_static_test.go b/test/volume_server/http/headers_static_test.go new file mode 100644 index 000000000..5b4a2fd93 --- /dev/null +++ b/test/volume_server/http/headers_static_test.go @@ -0,0 +1,102 @@ +package volume_server_http_test + +import ( + "fmt" + "net/http" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestReadPassthroughHeadersAndDownloadDisposition(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(96) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fullFileID := framework.NewFileID(volumeID, 661122, 0x55667788) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fullFileID, []byte("passthrough-header-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + parts := strings.SplitN(fullFileID, ",", 2) + if len(parts) != 2 { + t.Fatalf("unexpected file id format: %q", fullFileID) + } + fidOnly := parts[1] + + url := fmt.Sprintf("%s/%d/%s/%s?response-content-type=text/plain&response-cache-control=no-store&dl=true", + clusterHarness.VolumeAdminURL(), + volumeID, + fidOnly, + "report.txt", + ) + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, url)) + _ = framework.ReadAllAndClose(t, resp) + if resp.StatusCode != http.StatusOK { + t.Fatalf("passthrough read expected 200, got %d", resp.StatusCode) + } + if resp.Header.Get("Content-Type") != "text/plain" { + t.Fatalf("response-content-type override mismatch: %q", resp.Header.Get("Content-Type")) + } + if resp.Header.Get("Cache-Control") != "no-store" { + t.Fatalf("response-cache-control override mismatch: %q", resp.Header.Get("Cache-Control")) + } + contentDisposition := resp.Header.Get("Content-Disposition") + if !strings.Contains(contentDisposition, "attachment") || !strings.Contains(contentDisposition, "report.txt") { + t.Fatalf("download disposition header mismatch: %q", contentDisposition) + } +} + +func TestStaticAssetEndpoints(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + client := framework.NewHTTPClient() + + faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/favicon.ico")) + _ = framework.ReadAllAndClose(t, faviconResp) + if faviconResp.StatusCode != http.StatusOK { + t.Fatalf("/favicon.ico expected 200, got %d", faviconResp.StatusCode) + } + + staticResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/seaweedfsstatic/seaweed50x50.png")) + _ = framework.ReadAllAndClose(t, staticResp) + if staticResp.StatusCode != http.StatusOK { + t.Fatalf("/seaweedfsstatic/seaweed50x50.png expected 200, got %d", staticResp.StatusCode) + } +} + +func TestStaticAssetEndpointsOnPublicPort(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + client := framework.NewHTTPClient() + + faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/favicon.ico")) + _ = framework.ReadAllAndClose(t, faviconResp) + if faviconResp.StatusCode != http.StatusOK { + t.Fatalf("public /favicon.ico expected 200, got %d", faviconResp.StatusCode) + } + + staticResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/seaweedfsstatic/seaweed50x50.png")) + _ = framework.ReadAllAndClose(t, staticResp) + if staticResp.StatusCode != http.StatusOK { + t.Fatalf("public /seaweedfsstatic/seaweed50x50.png expected 200, got %d", staticResp.StatusCode) + } +} diff --git a/test/volume_server/http/image_transform_test.go b/test/volume_server/http/image_transform_test.go new file mode 100644 index 000000000..222fc951f --- /dev/null +++ b/test/volume_server/http/image_transform_test.go @@ -0,0 +1,92 @@ +package volume_server_http_test + +import ( + "bytes" + "fmt" + "image" + "image/color" + "image/png" + "net/http" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func makePNGFixture(t testing.TB, width, height int) []byte { + t.Helper() + + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + img.Set(x, y, color.RGBA{R: uint8(x * 20), G: uint8(y * 20), B: 200, A: 255}) + } + } + + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatalf("encode png fixture: %v", err) + } + return buf.Bytes() +} + +func decodeImageConfig(t testing.TB, data []byte) image.Config { + t.Helper() + cfg, _, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil { + t.Fatalf("decode image config: %v", err) + } + return cfg +} + +func TestImageResizeAndCropReadVariants(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(101) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fullFileID := framework.NewFileID(volumeID, 772004, 0x4D5E6F70) + uploadReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fullFileID, makePNGFixture(t, 6, 4)) + uploadReq.Header.Set("Content-Type", "image/png") + uploadResp := framework.DoRequest(t, client, uploadReq) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("image upload expected 201, got %d", uploadResp.StatusCode) + } + + parts := strings.SplitN(fullFileID, ",", 2) + if len(parts) != 2 { + t.Fatalf("unexpected file id format: %q", fullFileID) + } + fidOnly := parts[1] + + resizeURL := fmt.Sprintf("%s/%d/%s/%s?width=2&height=1", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "fixture.png") + resizeResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, resizeURL)) + resizeBody := framework.ReadAllAndClose(t, resizeResp) + if resizeResp.StatusCode != http.StatusOK { + t.Fatalf("image resize read expected 200, got %d", resizeResp.StatusCode) + } + resizeCfg := decodeImageConfig(t, resizeBody) + if resizeCfg.Width > 2 || resizeCfg.Height > 1 { + t.Fatalf("image resize expected dimensions <= 2x1, got %dx%d", resizeCfg.Width, resizeCfg.Height) + } + + cropURL := fmt.Sprintf("%s/%d/%s/%s?crop_x1=1&crop_y1=1&crop_x2=4&crop_y2=3", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "fixture.png") + cropResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cropURL)) + cropBody := framework.ReadAllAndClose(t, cropResp) + if cropResp.StatusCode != http.StatusOK { + t.Fatalf("image crop read expected 200, got %d", cropResp.StatusCode) + } + cropCfg := decodeImageConfig(t, cropBody) + if cropCfg.Width != 3 || cropCfg.Height != 2 { + t.Fatalf("image crop expected 3x2, got %dx%d", cropCfg.Width, cropCfg.Height) + } +} diff --git a/test/volume_server/http/public_cors_methods_test.go b/test/volume_server/http/public_cors_methods_test.go new file mode 100644 index 000000000..5328b9a8b --- /dev/null +++ b/test/volume_server/http/public_cors_methods_test.go @@ -0,0 +1,287 @@ +package volume_server_http_test + +import ( + "bytes" + "net/http" + "strconv" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestPublicPortReadOnlyMethodBehavior(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(81) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 123321, 0x01020304) + originalData := []byte("public-port-original") + replacementData := []byte("public-port-replacement") + client := framework.NewHTTPClient() + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, originalData) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("admin upload expected 201, got %d", uploadResp.StatusCode) + } + + publicReadResp := framework.ReadBytes(t, client, clusterHarness.VolumePublicURL(), fid) + publicReadBody := framework.ReadAllAndClose(t, publicReadResp) + if publicReadResp.StatusCode != http.StatusOK { + t.Fatalf("public GET expected 200, got %d", publicReadResp.StatusCode) + } + if string(publicReadBody) != string(originalData) { + t.Fatalf("public GET body mismatch: got %q want %q", string(publicReadBody), string(originalData)) + } + + publicPostReq := newUploadRequest(t, clusterHarness.VolumePublicURL()+"/"+fid, replacementData) + publicPostResp := framework.DoRequest(t, client, publicPostReq) + _ = framework.ReadAllAndClose(t, publicPostResp) + if publicPostResp.StatusCode != http.StatusOK { + t.Fatalf("public POST expected passthrough 200, got %d", publicPostResp.StatusCode) + } + + publicDeleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumePublicURL()+"/"+fid)) + _ = framework.ReadAllAndClose(t, publicDeleteResp) + if publicDeleteResp.StatusCode != http.StatusOK { + t.Fatalf("public DELETE expected passthrough 200, got %d", publicDeleteResp.StatusCode) + } + + adminReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) + adminReadBody := framework.ReadAllAndClose(t, adminReadResp) + if adminReadResp.StatusCode != http.StatusOK { + t.Fatalf("admin GET after public POST/DELETE expected 200, got %d", adminReadResp.StatusCode) + } + if string(adminReadBody) != string(originalData) { + t.Fatalf("public port should not mutate data: got %q want %q", string(adminReadBody), string(originalData)) + } +} + +func TestCorsAndUnsupportedMethodBehavior(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(82) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 789789, 0x0A0B0C0D) + client := framework.NewHTTPClient() + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("cors-check")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("admin upload expected 201, got %d", uploadResp.StatusCode) + } + + adminOriginReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + adminOriginReq.Header.Set("Origin", "https://example.com") + adminOriginResp := framework.DoRequest(t, client, adminOriginReq) + _ = framework.ReadAllAndClose(t, adminOriginResp) + if adminOriginResp.Header.Get("Access-Control-Allow-Origin") != "*" { + t.Fatalf("admin GET origin header mismatch: %q", adminOriginResp.Header.Get("Access-Control-Allow-Origin")) + } + if adminOriginResp.Header.Get("Access-Control-Allow-Credentials") != "true" { + t.Fatalf("admin GET credentials header mismatch: %q", adminOriginResp.Header.Get("Access-Control-Allow-Credentials")) + } + + publicOriginReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/"+fid) + publicOriginReq.Header.Set("Origin", "https://example.com") + publicOriginResp := framework.DoRequest(t, client, publicOriginReq) + _ = framework.ReadAllAndClose(t, publicOriginResp) + if publicOriginResp.Header.Get("Access-Control-Allow-Origin") != "*" { + t.Fatalf("public GET origin header mismatch: %q", publicOriginResp.Header.Get("Access-Control-Allow-Origin")) + } + if publicOriginResp.Header.Get("Access-Control-Allow-Credentials") != "true" { + t.Fatalf("public GET credentials header mismatch: %q", publicOriginResp.Header.Get("Access-Control-Allow-Credentials")) + } + + adminPatchReq, err := http.NewRequest(http.MethodPatch, clusterHarness.VolumeAdminURL()+"/"+fid, bytes.NewReader([]byte("patch"))) + if err != nil { + t.Fatalf("create admin PATCH request: %v", err) + } + adminPatchResp := framework.DoRequest(t, client, adminPatchReq) + _ = framework.ReadAllAndClose(t, adminPatchResp) + if adminPatchResp.StatusCode != http.StatusBadRequest { + t.Fatalf("admin PATCH expected 400, got %d", adminPatchResp.StatusCode) + } + + publicPatchReq, err := http.NewRequest(http.MethodPatch, clusterHarness.VolumePublicURL()+"/"+fid, bytes.NewReader([]byte("patch"))) + if err != nil { + t.Fatalf("create public PATCH request: %v", err) + } + publicPatchResp := framework.DoRequest(t, client, publicPatchReq) + _ = framework.ReadAllAndClose(t, publicPatchResp) + if publicPatchResp.StatusCode != http.StatusOK { + t.Fatalf("public PATCH expected passthrough 200, got %d", publicPatchResp.StatusCode) + } +} + +func TestUnsupportedMethodTraceParity(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(83) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 123999, 0x01010101) + client := framework.NewHTTPClient() + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("trace-method-check")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + adminTraceReq := mustNewRequest(t, http.MethodTrace, clusterHarness.VolumeAdminURL()+"/"+fid) + adminTraceResp := framework.DoRequest(t, client, adminTraceReq) + _ = framework.ReadAllAndClose(t, adminTraceResp) + if adminTraceResp.StatusCode != http.StatusBadRequest { + t.Fatalf("admin TRACE expected 400, got %d", adminTraceResp.StatusCode) + } + + publicTraceReq := mustNewRequest(t, http.MethodTrace, clusterHarness.VolumePublicURL()+"/"+fid) + publicTraceResp := framework.DoRequest(t, client, publicTraceReq) + _ = framework.ReadAllAndClose(t, publicTraceResp) + if publicTraceResp.StatusCode != http.StatusOK { + t.Fatalf("public TRACE expected passthrough 200, got %d", publicTraceResp.StatusCode) + } +} + +func TestUnsupportedMethodPropfindParity(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(84) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 124000, 0x02020202) + client := framework.NewHTTPClient() + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("propfind-method-check")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + adminReq := mustNewRequest(t, "PROPFIND", clusterHarness.VolumeAdminURL()+"/"+fid) + adminResp := framework.DoRequest(t, client, adminReq) + _ = framework.ReadAllAndClose(t, adminResp) + if adminResp.StatusCode != http.StatusBadRequest { + t.Fatalf("admin PROPFIND expected 400, got %d", adminResp.StatusCode) + } + + publicReq := mustNewRequest(t, "PROPFIND", clusterHarness.VolumePublicURL()+"/"+fid) + publicResp := framework.DoRequest(t, client, publicReq) + _ = framework.ReadAllAndClose(t, publicResp) + if publicResp.StatusCode != http.StatusOK { + t.Fatalf("public PROPFIND expected passthrough 200, got %d", publicResp.StatusCode) + } + + verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) + verifyBody := framework.ReadAllAndClose(t, verifyResp) + if verifyResp.StatusCode != http.StatusOK { + t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode) + } + if string(verifyBody) != "propfind-method-check" { + t.Fatalf("PROPFIND should not mutate data, got %q", string(verifyBody)) + } +} + +func TestUnsupportedMethodConnectParity(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(85) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 124001, 0x03030303) + client := framework.NewHTTPClient() + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("connect-method-check")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + adminReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumeAdminURL()+"/"+fid) + adminResp := framework.DoRequest(t, client, adminReq) + _ = framework.ReadAllAndClose(t, adminResp) + if adminResp.StatusCode != http.StatusBadRequest { + t.Fatalf("admin CONNECT expected 400, got %d", adminResp.StatusCode) + } + + publicReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumePublicURL()+"/"+fid) + publicResp := framework.DoRequest(t, client, publicReq) + _ = framework.ReadAllAndClose(t, publicResp) + if publicResp.StatusCode != http.StatusOK { + t.Fatalf("public CONNECT expected passthrough 200, got %d", publicResp.StatusCode) + } + + verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) + verifyBody := framework.ReadAllAndClose(t, verifyResp) + if verifyResp.StatusCode != http.StatusOK { + t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode) + } + if string(verifyBody) != "connect-method-check" { + t.Fatalf("CONNECT should not mutate data, got %q", string(verifyBody)) + } +} + +func TestPublicPortHeadReadParity(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(86) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 124002, 0x04040404) + payload := []byte("public-head-parity-content") + client := framework.NewHTTPClient() + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumePublicURL()+"/"+fid)) + headBody := framework.ReadAllAndClose(t, headResp) + if headResp.StatusCode != http.StatusOK { + t.Fatalf("public HEAD expected 200, got %d", headResp.StatusCode) + } + if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) { + t.Fatalf("public HEAD content-length mismatch: got %q want %d", got, len(payload)) + } + if len(headBody) != 0 { + t.Fatalf("public HEAD body should be empty, got %d bytes", len(headBody)) + } +} diff --git a/test/volume_server/http/range_variants_test.go b/test/volume_server/http/range_variants_test.go new file mode 100644 index 000000000..2e1f5e286 --- /dev/null +++ b/test/volume_server/http/range_variants_test.go @@ -0,0 +1,82 @@ +package volume_server_http_test + +import ( + "net/http" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestMultiRangeReadReturnsMultipartPayload(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(97) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 771999, 0x0A1B2C3D) + payload := []byte("0123456789abcdef") + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + multiRangeReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + multiRangeReq.Header.Set("Range", "bytes=0-1,4-5") + multiRangeResp := framework.DoRequest(t, client, multiRangeReq) + multiRangeBody := framework.ReadAllAndClose(t, multiRangeResp) + if multiRangeResp.StatusCode != http.StatusPartialContent { + t.Fatalf("multi-range expected 206, got %d", multiRangeResp.StatusCode) + } + if !strings.Contains(multiRangeResp.Header.Get("Content-Type"), "multipart/byteranges") { + t.Fatalf("multi-range content-type mismatch: %q", multiRangeResp.Header.Get("Content-Type")) + } + + bodyText := string(multiRangeBody) + if !strings.Contains(bodyText, "01") || !strings.Contains(bodyText, "45") { + t.Fatalf("multi-range body missing expected segments: %q", bodyText) + } +} + +func TestOversizedCombinedRangesAreIgnored(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(100) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 772003, 0x3C4D5E6F) + payload := []byte("0123456789abcdef") + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + // Range bytes sum is 22 (> payload size 16), which exercises the oversized-range guard path. + oversizedRangeReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + oversizedRangeReq.Header.Set("Range", "bytes=0-10,5-15") + oversizedRangeResp := framework.DoRequest(t, client, oversizedRangeReq) + oversizedRangeBody := framework.ReadAllAndClose(t, oversizedRangeResp) + if oversizedRangeResp.StatusCode != http.StatusOK { + t.Fatalf("oversized combined range expected 200, got %d", oversizedRangeResp.StatusCode) + } + if len(oversizedRangeBody) != 0 { + t.Fatalf("oversized combined range expected empty body, got %d bytes", len(oversizedRangeBody)) + } +} diff --git a/test/volume_server/http/read_deleted_test.go b/test/volume_server/http/read_deleted_test.go new file mode 100644 index 000000000..23d400e23 --- /dev/null +++ b/test/volume_server/http/read_deleted_test.go @@ -0,0 +1,54 @@ +package volume_server_http_test + +import ( + "net/http" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestReadDeletedQueryReturnsDeletedNeedleData(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(94) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 551234, 0xCAFE1234) + payload := []byte("read-deleted-needle-payload") + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid)) + _ = framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode) + } + + normalRead := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, normalRead) + if normalRead.StatusCode != http.StatusNotFound { + t.Fatalf("normal read after delete expected 404, got %d", normalRead.StatusCode) + } + + readDeletedReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true") + readDeletedResp := framework.DoRequest(t, client, readDeletedReq) + readDeletedBody := framework.ReadAllAndClose(t, readDeletedResp) + if readDeletedResp.StatusCode != http.StatusOK { + t.Fatalf("read with readDeleted=true expected 200, got %d", readDeletedResp.StatusCode) + } + if string(readDeletedBody) != string(payload) { + t.Fatalf("readDeleted body mismatch: got %q want %q", string(readDeletedBody), string(payload)) + } +} diff --git a/test/volume_server/http/read_mode_proxy_redirect_test.go b/test/volume_server/http/read_mode_proxy_redirect_test.go new file mode 100644 index 000000000..f82438043 --- /dev/null +++ b/test/volume_server/http/read_mode_proxy_redirect_test.go @@ -0,0 +1,319 @@ +package volume_server_http_test + +import ( + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestReadModeProxyMissingLocalVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P1() + profile.ReadMode = "proxy" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + + const volumeID = uint32(101) + framework.AllocateVolume(t, grpc0, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 120001, 0x0102ABCD) + payload := []byte("proxy-read-mode-forwarded-content") + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + var finalBody []byte + if !waitForHTTPStatus(t, client, readURL, http.StatusOK, 10*time.Second, func(resp *http.Response) { + finalBody = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("proxy read mode did not return 200 from non-owning volume server within deadline") + } + if string(finalBody) != string(payload) { + t.Fatalf("proxy read mode body mismatch: got %q want %q", string(finalBody), string(payload)) + } +} + +func TestReadModeRedirectMissingLocalVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P1() + profile.ReadMode = "redirect" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + + const volumeID = uint32(102) + framework.AllocateVolume(t, grpc0, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 120002, 0x0102DCBA) + payload := []byte("redirect-read-mode-content") + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + noRedirectClient := &http.Client{ + Timeout: 10 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + var redirectLocation string + if !waitForHTTPStatus(t, noRedirectClient, readURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) { + redirectLocation = resp.Header.Get("Location") + _ = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("redirect read mode did not return 301 from non-owning volume server within deadline") + } + if redirectLocation == "" { + t.Fatalf("redirect response missing Location header") + } + if !strings.Contains(redirectLocation, "proxied=true") { + t.Fatalf("redirect Location should include proxied=true, got %q", redirectLocation) + } + + followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, redirectLocation)) + followBody := framework.ReadAllAndClose(t, followResp) + if followResp.StatusCode != http.StatusOK { + t.Fatalf("following redirect expected 200, got %d", followResp.StatusCode) + } + if string(followBody) != string(payload) { + t.Fatalf("redirect-follow body mismatch: got %q want %q", string(followBody), string(payload)) + } +} + +func TestReadModeLocalMissingLocalVolumeReturnsNotFound(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P1() + profile.ReadMode = "local" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + + const volumeID = uint32(103) + framework.AllocateVolume(t, grpc0, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 120003, 0x0102BEEF) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, []byte("local-read-mode-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + readResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(1), fid) + _ = framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusNotFound { + t.Fatalf("local read mode expected 404 on non-owning server, got %d", readResp.StatusCode) + } +} + +func TestReadDeletedProxyModeOnMissingLocalVolume(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P1() + profile.ReadMode = "proxy" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + + const volumeID = uint32(104) + framework.AllocateVolume(t, grpc0, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 120004, 0x0102CAFE) + payload := []byte("proxy-readDeleted-missing-local-content") + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL(0)+"/"+fid)) + _ = framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode) + } + + readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?readDeleted=true" + var proxiedBody []byte + if !waitForHTTPStatus(t, client, readURL, http.StatusOK, 10*time.Second, func(resp *http.Response) { + proxiedBody = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("proxy readDeleted path did not return 200 from non-owning volume server within deadline") + } + if string(proxiedBody) != string(payload) { + t.Fatalf("proxy readDeleted body mismatch: got %q want %q", string(proxiedBody), string(payload)) + } +} + +func TestReadDeletedRedirectModeDropsQueryParameterParity(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P1() + profile.ReadMode = "redirect" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + + const volumeID = uint32(105) + framework.AllocateVolume(t, grpc0, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 120005, 0x0102FACE) + payload := []byte("redirect-readDeleted-query-drop-parity") + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL(0)+"/"+fid)) + _ = framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode) + } + + noRedirectClient := &http.Client{ + Timeout: 10 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + redirectURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?readDeleted=true" + var location string + if !waitForHTTPStatus(t, noRedirectClient, redirectURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) { + location = resp.Header.Get("Location") + _ = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("redirect readDeleted path did not return 301 from non-owning volume server within deadline") + } + if location == "" { + t.Fatalf("redirect readDeleted response missing Location header") + } + if !strings.Contains(location, "proxied=true") { + t.Fatalf("redirect readDeleted Location should include proxied=true, got %q", location) + } + if strings.Contains(location, "readDeleted=true") { + t.Fatalf("redirect readDeleted Location should reflect current query-drop behavior, got %q", location) + } + + followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, location)) + _ = framework.ReadAllAndClose(t, followResp) + if followResp.StatusCode != http.StatusNotFound { + t.Fatalf("redirect-follow without readDeleted query expected 404 for deleted needle, got %d", followResp.StatusCode) + } +} + +func TestReadModeRedirectPreservesCollectionQuery(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P1() + profile.ReadMode = "redirect" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + + const volumeID = uint32(109) + const collection = "redirect-collection" + framework.AllocateVolume(t, grpc0, volumeID, collection) + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 120006, 0x0102F00D) + payload := []byte("redirect-collection-preserve-content") + + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + noRedirectClient := &http.Client{ + Timeout: 10 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + redirectURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?collection=" + collection + var location string + if !waitForHTTPStatus(t, noRedirectClient, redirectURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) { + location = resp.Header.Get("Location") + _ = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("redirect collection path did not return 301 from non-owning volume server within deadline") + } + if location == "" { + t.Fatalf("redirect collection response missing Location header") + } + if !strings.Contains(location, "proxied=true") { + t.Fatalf("redirect collection Location should include proxied=true, got %q", location) + } + if !strings.Contains(location, "collection="+collection) { + t.Fatalf("redirect collection Location should preserve collection query, got %q", location) + } + + followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, location)) + followBody := framework.ReadAllAndClose(t, followResp) + if followResp.StatusCode != http.StatusOK { + t.Fatalf("redirect-follow expected 200, got %d", followResp.StatusCode) + } + if string(followBody) != string(payload) { + t.Fatalf("redirect-follow body mismatch: got %q want %q", string(followBody), string(payload)) + } +} + +func waitForHTTPStatus(t testing.TB, client *http.Client, url string, expectedStatus int, timeout time.Duration, onMatch func(resp *http.Response)) bool { + t.Helper() + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, url)) + if resp.StatusCode == expectedStatus { + onMatch(resp) + return true + } + _ = framework.ReadAllAndClose(t, resp) + time.Sleep(200 * time.Millisecond) + } + + return false +} diff --git a/test/volume_server/http/read_path_variants_test.go b/test/volume_server/http/read_path_variants_test.go new file mode 100644 index 000000000..97a7ac628 --- /dev/null +++ b/test/volume_server/http/read_path_variants_test.go @@ -0,0 +1,191 @@ +package volume_server_http_test + +import ( + "fmt" + "net/http" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestReadPathShapesAndIfModifiedSince(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(93) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fullFileID := framework.NewFileID(volumeID, 771234, 0xBEEFCACE) + uploadPayload := []byte("read-path-shape-content") + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fullFileID, uploadPayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + parts := strings.SplitN(fullFileID, ",", 2) + if len(parts) != 2 { + t.Fatalf("unexpected file id format: %q", fullFileID) + } + fidOnly := parts[1] + + readByVidFid := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, fmt.Sprintf("%s/%d/%s", clusterHarness.VolumeAdminURL(), volumeID, fidOnly))) + readByVidFidBody := framework.ReadAllAndClose(t, readByVidFid) + if readByVidFid.StatusCode != http.StatusOK { + t.Fatalf("GET /{vid}/{fid} expected 200, got %d", readByVidFid.StatusCode) + } + if string(readByVidFidBody) != string(uploadPayload) { + t.Fatalf("GET /{vid}/{fid} body mismatch: got %q want %q", string(readByVidFidBody), string(uploadPayload)) + } + + readWithFilename := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, fmt.Sprintf("%s/%d/%s/%s", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "named.bin"))) + readWithFilenameBody := framework.ReadAllAndClose(t, readWithFilename) + if readWithFilename.StatusCode != http.StatusOK { + t.Fatalf("GET /{vid}/{fid}/{filename} expected 200, got %d", readWithFilename.StatusCode) + } + if string(readWithFilenameBody) != string(uploadPayload) { + t.Fatalf("GET /{vid}/{fid}/{filename} body mismatch: got %q want %q", string(readWithFilenameBody), string(uploadPayload)) + } + + lastModified := readWithFilename.Header.Get("Last-Modified") + if lastModified == "" { + t.Fatalf("expected Last-Modified header on read response") + } + + ifModifiedSinceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fullFileID) + ifModifiedSinceReq.Header.Set("If-Modified-Since", lastModified) + ifModifiedSinceResp := framework.DoRequest(t, client, ifModifiedSinceReq) + _ = framework.ReadAllAndClose(t, ifModifiedSinceResp) + if ifModifiedSinceResp.StatusCode != http.StatusNotModified { + t.Fatalf("If-Modified-Since expected 304, got %d", ifModifiedSinceResp.StatusCode) + } + + headIfModifiedSinceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fullFileID) + headIfModifiedSinceReq.Header.Set("If-Modified-Since", lastModified) + headIfModifiedSinceResp := framework.DoRequest(t, client, headIfModifiedSinceReq) + headIfModifiedSinceBody := framework.ReadAllAndClose(t, headIfModifiedSinceResp) + if headIfModifiedSinceResp.StatusCode != http.StatusNotModified { + t.Fatalf("HEAD If-Modified-Since expected 304, got %d", headIfModifiedSinceResp.StatusCode) + } + if len(headIfModifiedSinceBody) != 0 { + t.Fatalf("HEAD If-Modified-Since expected empty body, got %d bytes", len(headIfModifiedSinceBody)) + } +} + +func TestMalformedVidFidPathReturnsBadRequest(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + client := framework.NewHTTPClient() + + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/not-a-vid/not-a-fid")) + _ = framework.ReadAllAndClose(t, resp) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("malformed /{vid}/{fid} expected 400, got %d", resp.StatusCode) + } +} + +func TestReadWrongCookieReturnsNotFound(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(95) + const needleID = uint64(771235) + const cookie = uint32(0xBEEFCACF) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, needleID, cookie) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("read-cookie-mismatch-content")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + wrongCookieFid := framework.NewFileID(volumeID, needleID, cookie+1) + getResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), wrongCookieFid) + _ = framework.ReadAllAndClose(t, getResp) + if getResp.StatusCode != http.StatusNotFound { + t.Fatalf("GET with wrong cookie expected 404, got %d", getResp.StatusCode) + } + + headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+wrongCookieFid)) + headBody := framework.ReadAllAndClose(t, headResp) + if headResp.StatusCode != http.StatusNotFound { + t.Fatalf("HEAD with wrong cookie expected 404, got %d", headResp.StatusCode) + } + if len(headBody) != 0 { + t.Fatalf("HEAD wrong-cookie response body should be empty, got %d bytes", len(headBody)) + } +} + +func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(99) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 772002, 0x2B3C4D5E) + payload := []byte("conditional-precedence-content") + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + baselineResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, baselineResp) + if baselineResp.StatusCode != http.StatusOK { + t.Fatalf("baseline read expected 200, got %d", baselineResp.StatusCode) + } + lastModified := baselineResp.Header.Get("Last-Modified") + if lastModified == "" { + t.Fatalf("baseline read expected Last-Modified header") + } + + precedenceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + precedenceReq.Header.Set("If-Modified-Since", lastModified) + precedenceReq.Header.Set("If-None-Match", "\"definitely-different-etag\"") + precedenceResp := framework.DoRequest(t, client, precedenceReq) + precedenceBody := framework.ReadAllAndClose(t, precedenceResp) + if precedenceResp.StatusCode != http.StatusNotModified { + t.Fatalf("conditional precedence expected 304, got %d", precedenceResp.StatusCode) + } + if len(precedenceBody) != 0 { + t.Fatalf("conditional precedence expected empty body, got %d bytes", len(precedenceBody)) + } + + invalidIMSReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) + invalidIMSReq.Header.Set("If-Modified-Since", "not-a-valid-http-date") + invalidIMSReq.Header.Set("If-None-Match", "\"definitely-different-etag\"") + invalidIMSResp := framework.DoRequest(t, client, invalidIMSReq) + invalidIMSBody := framework.ReadAllAndClose(t, invalidIMSResp) + if invalidIMSResp.StatusCode != http.StatusOK { + t.Fatalf("invalid If-Modified-Since with mismatched etag expected 200, got %d", invalidIMSResp.StatusCode) + } + if string(invalidIMSBody) != string(payload) { + t.Fatalf("invalid If-Modified-Since fallback body mismatch: got %q want %q", string(invalidIMSBody), string(payload)) + } +} diff --git a/test/volume_server/http/read_write_delete_test.go b/test/volume_server/http/read_write_delete_test.go new file mode 100644 index 000000000..b122d697c --- /dev/null +++ b/test/volume_server/http/read_write_delete_test.go @@ -0,0 +1,123 @@ +package volume_server_http_test + +import ( + "net/http" + "strconv" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestUploadReadRangeHeadDeleteRoundTrip(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(7) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + fid := framework.NewFileID(volumeID, 123456, 0xA1B2C3D4) + data := []byte("hello-volume-server-integration") + client := framework.NewHTTPClient() + + uploadResp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(), fid, data) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode) + } + + getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid) + getBody := framework.ReadAllAndClose(t, getResp) + if getResp.StatusCode != http.StatusOK { + t.Fatalf("get status: expected 200, got %d", getResp.StatusCode) + } + if string(getBody) != string(data) { + t.Fatalf("get body mismatch: got %q want %q", string(getBody), string(data)) + } + etag := getResp.Header.Get("ETag") + if etag == "" { + t.Fatalf("expected ETag header from GET response") + } + + notModifiedReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid) + notModifiedReq.Header.Set("If-None-Match", etag) + notModifiedResp := framework.DoRequest(t, client, notModifiedReq) + _ = framework.ReadAllAndClose(t, notModifiedResp) + if notModifiedResp.StatusCode != http.StatusNotModified { + t.Fatalf("if-none-match expected 304, got %d", notModifiedResp.StatusCode) + } + + rangeReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid) + rangeReq.Header.Set("Range", "bytes=0-4") + rangeResp := framework.DoRequest(t, client, rangeReq) + rangeBody := framework.ReadAllAndClose(t, rangeResp) + if rangeResp.StatusCode != http.StatusPartialContent { + t.Fatalf("range status: expected 206, got %d", rangeResp.StatusCode) + } + if got, want := string(rangeBody), "hello"; got != want { + t.Fatalf("range body mismatch: got %q want %q", got, want) + } + + invalidRangeReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid) + invalidRangeReq.Header.Set("Range", "bytes=9999-10000") + invalidRangeResp := framework.DoRequest(t, client, invalidRangeReq) + _ = framework.ReadAllAndClose(t, invalidRangeResp) + if invalidRangeResp.StatusCode != http.StatusRequestedRangeNotSatisfiable { + t.Fatalf("invalid range expected 416, got %d", invalidRangeResp.StatusCode) + } + + headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, cluster.VolumeAdminURL()+"/"+fid)) + headBody := framework.ReadAllAndClose(t, headResp) + if headResp.StatusCode != http.StatusOK { + t.Fatalf("head status: expected 200, got %d", headResp.StatusCode) + } + if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(data)) { + t.Fatalf("head content-length mismatch: got %q want %d", got, len(data)) + } + if len(headBody) != 0 { + t.Fatalf("head body should be empty, got %d bytes", len(headBody)) + } + + headNotModifiedReq := mustNewRequest(t, http.MethodHead, cluster.VolumeAdminURL()+"/"+fid) + headNotModifiedReq.Header.Set("If-None-Match", etag) + headNotModifiedResp := framework.DoRequest(t, client, headNotModifiedReq) + headNotModifiedBody := framework.ReadAllAndClose(t, headNotModifiedResp) + if headNotModifiedResp.StatusCode != http.StatusNotModified { + t.Fatalf("head if-none-match expected 304, got %d", headNotModifiedResp.StatusCode) + } + if len(headNotModifiedBody) != 0 { + t.Fatalf("head if-none-match body should be empty, got %d bytes", len(headNotModifiedBody)) + } + + deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, cluster.VolumeAdminURL()+"/"+fid)) + _ = framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("delete status: expected 202, got %d", deleteResp.StatusCode) + } + + notFoundResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, notFoundResp) + if notFoundResp.StatusCode != http.StatusNotFound { + t.Fatalf("read after delete: expected 404, got %d", notFoundResp.StatusCode) + } +} + +func TestInvalidReadPathReturnsBadRequest(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) + client := framework.NewHTTPClient() + + resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/invalid,needle")) + _ = framework.ReadAllAndClose(t, resp) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("invalid read expected 400, got %d", resp.StatusCode) + } +} diff --git a/test/volume_server/http/throttling_test.go b/test/volume_server/http/throttling_test.go new file mode 100644 index 000000000..7a66e9ebb --- /dev/null +++ b/test/volume_server/http/throttling_test.go @@ -0,0 +1,730 @@ +package volume_server_http_test + +import ( + "bytes" + "context" + "io" + "net/http" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" +) + +type pausableReader struct { + remaining int64 + pauseAfter int64 + paused bool + unblock <-chan struct{} +} + +func (r *pausableReader) Read(p []byte) (int, error) { + if r.remaining <= 0 { + return 0, io.EOF + } + if !r.paused && r.pauseAfter > 0 { + n := int64(len(p)) + if n > r.pauseAfter { + n = r.pauseAfter + } + for i := int64(0); i < n; i++ { + p[i] = 'a' + } + r.remaining -= n + r.pauseAfter -= n + if r.pauseAfter == 0 { + r.paused = true + } + return int(n), nil + } + if r.paused { + <-r.unblock + r.paused = false + } + n := int64(len(p)) + if n > r.remaining { + n = r.remaining + } + for i := int64(0); i < n; i++ { + p[i] = 'b' + } + r.remaining -= n + return int(n), nil +} + +func TestUploadLimitTimeoutAndReplicateBypass(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(98) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + const blockedUploadSize = 2 * 1024 * 1024 // over 1MB P8 upload limit + + unblockFirstUpload := make(chan struct{}) + firstUploadDone := make(chan error, 1) + firstFID := framework.NewFileID(volumeID, 880001, 0x1A2B3C4D) + go func() { + req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{ + remaining: blockedUploadSize, + pauseAfter: 1, + unblock: unblockFirstUpload, + }) + if err != nil { + firstUploadDone <- err + return + } + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = blockedUploadSize + + resp, err := (&http.Client{}).Do(req) + if resp != nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } + firstUploadDone <- err + }() + + // Give the first upload time to pass limit checks and block in body processing. + time.Sleep(300 * time.Millisecond) + + replicateFID := framework.NewFileID(volumeID, 880002, 0x5E6F7A8B) + replicateReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+replicateFID+"?type=replicate", bytes.NewReader([]byte("replicate"))) + if err != nil { + t.Fatalf("create replicate request: %v", err) + } + replicateReq.Header.Set("Content-Type", "application/octet-stream") + replicateReq.ContentLength = int64(len("replicate")) + replicateResp, err := framework.NewHTTPClient().Do(replicateReq) + if err != nil { + t.Fatalf("replicate request failed: %v", err) + } + _ = framework.ReadAllAndClose(t, replicateResp) + if replicateResp.StatusCode != http.StatusCreated { + t.Fatalf("replicate request expected 201 bypassing limit, got %d", replicateResp.StatusCode) + } + + normalFID := framework.NewFileID(volumeID, 880003, 0x9C0D1E2F) + normalReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+normalFID, bytes.NewReader([]byte("normal"))) + if err != nil { + t.Fatalf("create normal request: %v", err) + } + normalReq.Header.Set("Content-Type", "application/octet-stream") + normalReq.ContentLength = int64(len("normal")) + + timeoutClient := &http.Client{Timeout: 10 * time.Second} + normalResp, err := timeoutClient.Do(normalReq) + if err != nil { + t.Fatalf("normal upload request failed: %v", err) + } + _ = framework.ReadAllAndClose(t, normalResp) + if normalResp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("normal upload expected 429 while limit blocked, got %d", normalResp.StatusCode) + } + + close(unblockFirstUpload) + select { + case <-firstUploadDone: + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for blocked upload to finish") + } +} + +func TestUploadLimitWaitThenProceed(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(111) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + const blockedUploadSize = 2 * 1024 * 1024 + + unblockFirstUpload := make(chan struct{}) + firstUploadDone := make(chan error, 1) + firstFID := framework.NewFileID(volumeID, 880601, 0x6A2B3C4D) + go func() { + req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{ + remaining: blockedUploadSize, + pauseAfter: 1, + unblock: unblockFirstUpload, + }) + if err != nil { + firstUploadDone <- err + return + } + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = blockedUploadSize + + resp, err := (&http.Client{}).Do(req) + if resp != nil { + _ = framework.ReadAllAndClose(t, resp) + } + firstUploadDone <- err + }() + + time.Sleep(300 * time.Millisecond) + + type uploadResult struct { + resp *http.Response + err error + } + secondUploadDone := make(chan uploadResult, 1) + secondFID := framework.NewFileID(volumeID, 880602, 0x6A2B3C4E) + go func() { + req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+secondFID, bytes.NewReader([]byte("wait-then-proceed"))) + if err != nil { + secondUploadDone <- uploadResult{err: err} + return + } + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = int64(len("wait-then-proceed")) + resp, err := (&http.Client{Timeout: 10 * time.Second}).Do(req) + secondUploadDone <- uploadResult{resp: resp, err: err} + }() + + time.Sleep(500 * time.Millisecond) + close(unblockFirstUpload) + + select { + case firstErr := <-firstUploadDone: + if firstErr != nil { + t.Fatalf("first blocked upload failed: %v", firstErr) + } + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for first upload completion") + } + + select { + case result := <-secondUploadDone: + if result.err != nil { + t.Fatalf("second upload failed: %v", result.err) + } + _ = framework.ReadAllAndClose(t, result.resp) + if result.resp.StatusCode != http.StatusCreated { + t.Fatalf("second upload expected 201 after waiting for slot, got %d", result.resp.StatusCode) + } + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for second upload completion") + } +} + +func TestUploadLimitTimeoutThenRecovery(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(113) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + const blockedUploadSize = 2 * 1024 * 1024 + + unblockFirstUpload := make(chan struct{}) + firstUploadDone := make(chan error, 1) + firstFID := framework.NewFileID(volumeID, 880801, 0x7A2B3C4D) + go func() { + req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{ + remaining: blockedUploadSize, + pauseAfter: 1, + unblock: unblockFirstUpload, + }) + if err != nil { + firstUploadDone <- err + return + } + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = blockedUploadSize + resp, err := (&http.Client{}).Do(req) + if resp != nil { + _ = framework.ReadAllAndClose(t, resp) + } + firstUploadDone <- err + }() + + time.Sleep(300 * time.Millisecond) + + timeoutFID := framework.NewFileID(volumeID, 880802, 0x7A2B3C4E) + timeoutResp := framework.UploadBytes(t, &http.Client{Timeout: 10 * time.Second}, clusterHarness.VolumeAdminURL(), timeoutFID, []byte("should-timeout")) + _ = framework.ReadAllAndClose(t, timeoutResp) + if timeoutResp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("second upload under blocked pressure expected 429, got %d", timeoutResp.StatusCode) + } + + close(unblockFirstUpload) + select { + case firstErr := <-firstUploadDone: + if firstErr != nil { + t.Fatalf("first blocked upload failed: %v", firstErr) + } + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for first upload completion") + } + + recoveryFID := framework.NewFileID(volumeID, 880803, 0x7A2B3C4F) + recoveryResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), recoveryFID, []byte("recovered-upload")) + _ = framework.ReadAllAndClose(t, recoveryResp) + if recoveryResp.StatusCode != http.StatusCreated { + t.Fatalf("recovery upload expected 201, got %d", recoveryResp.StatusCode) + } +} + +func TestDownloadLimitTimeoutReturnsTooManyRequests(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(99) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + largePayload := make([]byte, 12*1024*1024) // over 1MB P8 download limit + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + downloadFID := framework.NewFileID(volumeID, 880101, 0x10203040) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), downloadFID, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+downloadFID)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + defer firstResp.Body.Close() + + // Keep first response body unread so server write path stays in-flight. + time.Sleep(300 * time.Millisecond) + + secondClient := &http.Client{Timeout: 10 * time.Second} + secondResp, err := secondClient.Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+downloadFID)) + if err != nil { + t.Fatalf("second GET failed: %v", err) + } + _ = framework.ReadAllAndClose(t, secondResp) + if secondResp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("second GET expected 429 while first download holds limit, got %d", secondResp.StatusCode) + } +} + +func TestDownloadLimitWaitThenProceedWithoutReplica(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(112) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + largePayload := make([]byte, 12*1024*1024) + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + fid := framework.NewFileID(volumeID, 880701, 0x60708090) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + + type readResult struct { + resp *http.Response + err error + } + secondReadDone := make(chan readResult, 1) + go func() { + resp, readErr := (&http.Client{Timeout: 10 * time.Second}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) + secondReadDone <- readResult{resp: resp, err: readErr} + }() + + time.Sleep(500 * time.Millisecond) + _ = firstResp.Body.Close() + + select { + case result := <-secondReadDone: + if result.err != nil { + t.Fatalf("second GET failed: %v", result.err) + } + secondBody := framework.ReadAllAndClose(t, result.resp) + if result.resp.StatusCode != http.StatusOK { + t.Fatalf("second GET expected 200 after waiting for slot, got %d", result.resp.StatusCode) + } + if len(secondBody) != len(largePayload) { + t.Fatalf("second GET body size mismatch: got %d want %d", len(secondBody), len(largePayload)) + } + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for second GET completion") + } +} + +func TestDownloadLimitTimeoutThenRecovery(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(114) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + largePayload := make([]byte, 12*1024*1024) + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + fid := framework.NewFileID(volumeID, 880901, 0x708090A0) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + + time.Sleep(300 * time.Millisecond) + + timeoutResp := framework.ReadBytes(t, &http.Client{Timeout: 10 * time.Second}, clusterHarness.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, timeoutResp) + if timeoutResp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("second GET under blocked pressure expected 429, got %d", timeoutResp.StatusCode) + } + + _ = firstResp.Body.Close() + + recoveryResp := framework.ReadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid) + recoveryBody := framework.ReadAllAndClose(t, recoveryResp) + if recoveryResp.StatusCode != http.StatusOK { + t.Fatalf("recovery GET expected 200, got %d", recoveryResp.StatusCode) + } + if len(recoveryBody) != len(largePayload) { + t.Fatalf("recovery GET body size mismatch: got %d want %d", len(recoveryBody), len(largePayload)) + } +} + +func TestDownloadLimitOverageProxiesToReplica(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P8() + profile.ReadMode = "proxy" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + conn1, grpc1 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) + defer conn1.Close() + + const volumeID = uint32(100) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + req := &volume_server_pb.AllocateVolumeRequest{ + VolumeId: volumeID, + Replication: "001", + Version: uint32(needle.GetCurrentVersion()), + } + if _, err := grpc0.AllocateVolume(ctx, req); err != nil { + t.Fatalf("allocate replicated volume on node0: %v", err) + } + if _, err := grpc1.AllocateVolume(ctx, req); err != nil { + t.Fatalf("allocate replicated volume on node1: %v", err) + } + + largePayload := make([]byte, 12*1024*1024) + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + fid := framework.NewFileID(volumeID, 880201, 0x0A0B0C0D) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(0), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("replicated large upload expected 201, got %d", uploadResp.StatusCode) + } + + replicaReadURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + if !waitForHTTPStatus(t, framework.NewHTTPClient(), replicaReadURL, http.StatusOK, 10*time.Second, func(resp *http.Response) { + _ = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("replica did not become readable within deadline") + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + defer firstResp.Body.Close() + + time.Sleep(300 * time.Millisecond) + + secondResp, err := framework.NewHTTPClient().Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid)) + if err != nil { + t.Fatalf("second GET failed: %v", err) + } + secondBody := framework.ReadAllAndClose(t, secondResp) + if secondResp.StatusCode != http.StatusOK { + t.Fatalf("second GET expected 200 via replica proxy fallback, got %d", secondResp.StatusCode) + } + if len(secondBody) != len(largePayload) { + t.Fatalf("second GET proxied body size mismatch: got %d want %d", len(secondBody), len(largePayload)) + } +} + +func TestDownloadLimitProxiedRequestSkipsReplicaFallbackAndTimesOut(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + profile := matrix.P8() + profile.ReadMode = "proxy" + clusterHarness := framework.StartDualVolumeCluster(t, profile) + + conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) + defer conn0.Close() + conn1, grpc1 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) + defer conn1.Close() + + const volumeID = uint32(106) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + req := &volume_server_pb.AllocateVolumeRequest{ + VolumeId: volumeID, + Replication: "001", + Version: uint32(needle.GetCurrentVersion()), + } + if _, err := grpc0.AllocateVolume(ctx, req); err != nil { + t.Fatalf("allocate replicated volume on node0: %v", err) + } + if _, err := grpc1.AllocateVolume(ctx, req); err != nil { + t.Fatalf("allocate replicated volume on node1: %v", err) + } + + largePayload := make([]byte, 12*1024*1024) + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + fid := framework.NewFileID(volumeID, 880202, 0x0A0B0D0E) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(0), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("replicated large upload expected 201, got %d", uploadResp.StatusCode) + } + + // Ensure replica path is actually available, so a non-proxied request would proxy. + replicaReadURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + if !waitForHTTPStatus(t, framework.NewHTTPClient(), replicaReadURL, http.StatusOK, 10*time.Second, func(resp *http.Response) { + _ = framework.ReadAllAndClose(t, resp) + }) { + t.Fatalf("replica did not become readable within deadline") + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + defer firstResp.Body.Close() + + time.Sleep(300 * time.Millisecond) + + // proxied=true should bypass replica fallback and hit wait/timeout branch. + secondResp, err := framework.NewHTTPClient().Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid+"?proxied=true")) + if err != nil { + t.Fatalf("second GET failed: %v", err) + } + _ = framework.ReadAllAndClose(t, secondResp) + if secondResp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("second GET with proxied=true expected 429 timeout path, got %d", secondResp.StatusCode) + } +} + +func TestUploadLimitDisabledAllowsConcurrentUploads(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(107) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + const blockedUploadSize = 2 * 1024 * 1024 + unblockFirstUpload := make(chan struct{}) + firstUploadDone := make(chan error, 1) + firstFID := framework.NewFileID(volumeID, 880301, 0x1A2B3C5D) + go func() { + req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{ + remaining: blockedUploadSize, + pauseAfter: 1, + unblock: unblockFirstUpload, + }) + if err != nil { + firstUploadDone <- err + return + } + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = blockedUploadSize + + resp, err := (&http.Client{}).Do(req) + if resp != nil { + _ = framework.ReadAllAndClose(t, resp) + } + firstUploadDone <- err + }() + + time.Sleep(300 * time.Millisecond) + + secondFID := framework.NewFileID(volumeID, 880302, 0x1A2B3C5E) + secondResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), secondFID, []byte("no-limit-second-upload")) + _ = framework.ReadAllAndClose(t, secondResp) + if secondResp.StatusCode != http.StatusCreated { + t.Fatalf("second upload with disabled limit expected 201, got %d", secondResp.StatusCode) + } + + close(unblockFirstUpload) + select { + case <-firstUploadDone: + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for first upload completion") + } +} + +func TestDownloadLimitDisabledAllowsConcurrentDownloads(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(108) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + largePayload := make([]byte, 12*1024*1024) + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + fid := framework.NewFileID(volumeID, 880401, 0x20304050) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + defer firstResp.Body.Close() + + time.Sleep(300 * time.Millisecond) + + secondResp := framework.ReadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid) + secondBody := framework.ReadAllAndClose(t, secondResp) + if secondResp.StatusCode != http.StatusOK { + t.Fatalf("second GET with disabled limit expected 200, got %d", secondResp.StatusCode) + } + if len(secondBody) != len(largePayload) { + t.Fatalf("second GET body size mismatch: got %d want %d", len(secondBody), len(largePayload)) + } +} + +func TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(110) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + largePayload := make([]byte, 12*1024*1024) + for i := range largePayload { + largePayload[i] = byte(i % 251) + } + fid := framework.NewFileID(volumeID, 880501, 0x50607080) + uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) + } + + firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) + if err != nil { + t.Fatalf("first GET failed: %v", err) + } + if firstResp.StatusCode != http.StatusOK { + _ = framework.ReadAllAndClose(t, firstResp) + t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) + } + defer firstResp.Body.Close() + + time.Sleep(300 * time.Millisecond) + + invalidReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/not-a-vid,1234567890ab") + invalidResp := framework.DoRequest(t, framework.NewHTTPClient(), invalidReq) + _ = framework.ReadAllAndClose(t, invalidResp) + if invalidResp.StatusCode != http.StatusBadRequest { + t.Fatalf("invalid vid while over limit expected 400, got %d", invalidResp.StatusCode) + } +} diff --git a/test/volume_server/http/write_delete_variants_test.go b/test/volume_server/http/write_delete_variants_test.go new file mode 100644 index 000000000..3355e7778 --- /dev/null +++ b/test/volume_server/http/write_delete_variants_test.go @@ -0,0 +1,118 @@ +package volume_server_http_test + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestWriteUnchangedAndDeleteEdgeVariants(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(87) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + const key = uint64(999001) + const cookie = uint32(0xDEADBEEF) + fid := framework.NewFileID(volumeID, key, cookie) + client := framework.NewHTTPClient() + payload := []byte("unchanged-write-content") + + firstUpload := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + firstUploadResp := framework.DoRequest(t, client, firstUpload) + _ = framework.ReadAllAndClose(t, firstUploadResp) + if firstUploadResp.StatusCode != http.StatusCreated { + t.Fatalf("first upload expected 201, got %d", firstUploadResp.StatusCode) + } + + secondUpload := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) + secondUploadResp := framework.DoRequest(t, client, secondUpload) + _ = framework.ReadAllAndClose(t, secondUploadResp) + if secondUploadResp.StatusCode != http.StatusNoContent { + t.Fatalf("second unchanged upload expected 204, got %d", secondUploadResp.StatusCode) + } + if secondUploadResp.Header.Get("ETag") == "" { + t.Fatalf("second unchanged upload expected ETag header") + } + + wrongCookieFid := framework.NewFileID(volumeID, key, cookie+1) + wrongCookieDelete := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+wrongCookieFid)) + _ = framework.ReadAllAndClose(t, wrongCookieDelete) + if wrongCookieDelete.StatusCode != http.StatusBadRequest { + t.Fatalf("delete with mismatched cookie expected 400, got %d", wrongCookieDelete.StatusCode) + } + + missingDelete := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+framework.NewFileID(volumeID, key+1, cookie))) + missingDeleteBody := framework.ReadAllAndClose(t, missingDelete) + if missingDelete.StatusCode != http.StatusNotFound { + t.Fatalf("delete missing needle expected 404, got %d", missingDelete.StatusCode) + } + + var payloadMap map[string]int64 + if err := json.Unmarshal(missingDeleteBody, &payloadMap); err != nil { + t.Fatalf("decode delete missing response: %v", err) + } + if payloadMap["size"] != 0 { + t.Fatalf("delete missing needle expected size=0, got %d", payloadMap["size"]) + } +} + +func TestDeleteTimestampOverrideKeepsReadDeletedLastModifiedParity(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(88) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 999002, 0xABCD1234) + uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("delete-ts-override")) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + beforeDeleteResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) + _ = framework.ReadAllAndClose(t, beforeDeleteResp) + if beforeDeleteResp.StatusCode != http.StatusOK { + t.Fatalf("pre-delete read expected 200, got %d", beforeDeleteResp.StatusCode) + } + lastModifiedBeforeDelete := beforeDeleteResp.Header.Get("Last-Modified") + if lastModifiedBeforeDelete == "" { + t.Fatalf("expected Last-Modified before delete") + } + + deleteReq := mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid+"?ts=1700000000") + deleteResp := framework.DoRequest(t, client, deleteReq) + _ = framework.ReadAllAndClose(t, deleteResp) + if deleteResp.StatusCode != http.StatusAccepted { + t.Fatalf("delete with ts override expected 202, got %d", deleteResp.StatusCode) + } + + readDeletedResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true")) + _ = framework.ReadAllAndClose(t, readDeletedResp) + if readDeletedResp.StatusCode != http.StatusOK { + t.Fatalf("readDeleted after ts override expected 200, got %d", readDeletedResp.StatusCode) + } + lastModified := readDeletedResp.Header.Get("Last-Modified") + if lastModified == "" { + t.Fatalf("expected Last-Modified header on readDeleted response") + } + if lastModified != lastModifiedBeforeDelete { + t.Fatalf("expected readDeleted Last-Modified parity with pre-delete header, got %q want %q", lastModified, lastModifiedBeforeDelete) + } +} diff --git a/test/volume_server/http/write_error_variants_test.go b/test/volume_server/http/write_error_variants_test.go new file mode 100644 index 000000000..ead11ed6c --- /dev/null +++ b/test/volume_server/http/write_error_variants_test.go @@ -0,0 +1,74 @@ +package volume_server_http_test + +import ( + "net/http" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" +) + +func TestWriteInvalidVidAndFidReturnBadRequest(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + client := framework.NewHTTPClient() + + invalidVidReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/invalid,12345678", []byte("x")) + invalidVidResp := framework.DoRequest(t, client, invalidVidReq) + _ = framework.ReadAllAndClose(t, invalidVidResp) + if invalidVidResp.StatusCode != http.StatusBadRequest { + t.Fatalf("write with invalid vid expected 400, got %d", invalidVidResp.StatusCode) + } + + invalidFidReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/1,bad", []byte("x")) + invalidFidResp := framework.DoRequest(t, client, invalidFidReq) + _ = framework.ReadAllAndClose(t, invalidFidResp) + if invalidFidResp.StatusCode != http.StatusBadRequest { + t.Fatalf("write with invalid fid expected 400, got %d", invalidFidResp.StatusCode) + } +} + +func TestWriteMalformedMultipartAndMD5Mismatch(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(98) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + client := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 772001, 0x1A2B3C4D) + + malformedMultipartReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+fid, strings.NewReader("not-a-valid-multipart-body")) + if err != nil { + t.Fatalf("create malformed multipart request: %v", err) + } + malformedMultipartReq.Header.Set("Content-Type", "multipart/form-data") + malformedMultipartResp := framework.DoRequest(t, client, malformedMultipartReq) + malformedMultipartBody := framework.ReadAllAndClose(t, malformedMultipartResp) + if malformedMultipartResp.StatusCode != http.StatusBadRequest { + t.Fatalf("malformed multipart write expected 400, got %d", malformedMultipartResp.StatusCode) + } + if !strings.Contains(strings.ToLower(string(malformedMultipartBody)), "boundary") { + t.Fatalf("malformed multipart response should mention boundary parse failure, got %q", string(malformedMultipartBody)) + } + + md5MismatchReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, []byte("content-md5-mismatch-body")) + md5MismatchReq.Header.Set("Content-MD5", "AAAAAAAAAAAAAAAAAAAAAA==") + md5MismatchResp := framework.DoRequest(t, client, md5MismatchReq) + md5MismatchBody := framework.ReadAllAndClose(t, md5MismatchResp) + if md5MismatchResp.StatusCode != http.StatusBadRequest { + t.Fatalf("content-md5 mismatch write expected 400, got %d", md5MismatchResp.StatusCode) + } + if !strings.Contains(string(md5MismatchBody), "Content-MD5") { + t.Fatalf("content-md5 mismatch response should mention Content-MD5, got %q", string(md5MismatchBody)) + } +} diff --git a/test/volume_server/matrix/config_profiles.go b/test/volume_server/matrix/config_profiles.go new file mode 100644 index 000000000..c359eb029 --- /dev/null +++ b/test/volume_server/matrix/config_profiles.go @@ -0,0 +1,63 @@ +package matrix + +import "time" + +// Profile describes one runtime test matrix configuration. +type Profile struct { + Name string + + ReadMode string + SplitPublicPort bool + + EnableJWT bool + JWTSigningKey string + JWTReadKey string + EnableMaintain bool + + ConcurrentUploadLimitMB int + ConcurrentDownloadLimitMB int + InflightUploadTimeout time.Duration + InflightDownloadTimeout time.Duration + + ReplicatedLayout bool + HasErasureCoding bool + HasRemoteTier bool +} + +// P1 is the baseline profile: one volume server, no JWT, proxy read mode. +func P1() Profile { + return Profile{ + Name: "P1", + ReadMode: "proxy", + SplitPublicPort: false, + } +} + +// P2 uses split public/admin ports to verify public read-only behavior. +func P2() Profile { + p := P1() + p.Name = "P2" + p.SplitPublicPort = true + return p +} + +// P3 enables JWT verification for read/write flows. +func P3() Profile { + p := P1() + p.Name = "P3" + p.EnableJWT = true + p.JWTSigningKey = "volume-server-write-key" + p.JWTReadKey = "volume-server-read-key" + return p +} + +// P8 enables upload/download throttling branches. +func P8() Profile { + p := P1() + p.Name = "P8" + p.ConcurrentUploadLimitMB = 1 + p.ConcurrentDownloadLimitMB = 1 + p.InflightUploadTimeout = 2 * time.Second + p.InflightDownloadTimeout = 2 * time.Second + return p +}