Browse Source
Add volume server integration test suite and CI workflow (#8322)
Add volume server integration test suite and CI workflow (#8322)
* docs(volume_server): add integration test development plan * test(volume_server): add integration harness and profile matrix * test(volume_server/http): add admin and options integration coverage * test(volume_server/grpc): add state and status integration coverage * test(volume_server): auto-build weed binary and harden cluster startup * test(volume_server/http): add upload read range head delete coverage * test(volume_server/grpc): expand admin lifecycle and state coverage * docs(volume_server): update progress tracker for implemented tests * test(volume_server/http): cover if-none-match and invalid-range branches * test(volume_server/grpc): add batch delete integration coverage * docs(volume_server): log latest HTTP and gRPC test coverage * ci(volume_server): run volume server integration tests in github actions * test(volume_server/grpc): add needle status configure ping and leave coverage * docs(volume_server): record additional grpc coverage progress * test(volume_server/grpc): add vacuum integration coverage * docs(volume_server): record vacuum test coverage progress * test(volume_server/grpc): add read and write needle blob error-path coverage * docs(volume_server): record data rw grpc coverage progress * test(volume_server/http): add jwt auth integration coverage * test(volume_server/grpc): add sync copy and stream error-path coverage * docs(volume_server): record jwt and sync/copy test coverage * test(volume_server/grpc): add scrub and query integration coverage * test(volume_server/grpc): add volume tail sender and receiver coverage * docs(volume_server): record scrub query and tail test progress * test(volume_server/grpc): add readonly writable and collection lifecycle coverage * test(volume_server/http): add public-port cors and method parity coverage * test(volume_server/grpc): add blob meta and read-all success path coverage * test(volume_server/grpc): expand scrub and query variation coverage * test(volume_server/grpc): add tiering and remote fetch error-path coverage * test(volume_server/http): add unchanged write and delete edge-case coverage * test(volume_server/grpc): add ping unknown and unreachable target coverage * test(volume_server/grpc): add volume delete only-empty variation coverage * test(volume_server/http): add jwt fid-mismatch auth coverage * test(volume_server/grpc): add scrub ec auto-select empty coverage * test(volume_server/grpc): stabilize ping timestamp assertion * docs(volume_server): update integration coverage progress log * test(volume_server/grpc): add tier remote backend and config variation coverage * docs(volume_server): record tier remote variation progress * test(volume_server/grpc): add incremental copy and receive-file protocol coverage * test(volume_server/http): add read path shape and if-modified-since coverage * test(volume_server/grpc): add copy-file compaction and receive-file success coverage * test(volume_server/http): add passthrough headers and static asset coverage * test(volume_server/grpc): add ping filer unreachable coverage * docs(volume_server): record copy receive and http variant progress * test(volume_server/grpc): add erasure coding maintenance and missing-path coverage * docs(volume_server): record initial erasure coding rpc coverage * test(volume_server/http): add multi-range multipart response coverage * docs(volume_server): record multi-range http coverage progress * test(volume_server/grpc): add query empty-stripe no-match coverage * docs(volume_server): record query no-match stream behavior coverage * test(volume_server/http): add upload throttling timeout and replicate bypass coverage * docs(volume_server): record upload throttling coverage progress * test(volume_server/http): add download throttling timeout coverage * docs(volume_server): record download throttling coverage progress * test(volume_server/http): add jwt wrong-cookie fid mismatch coverage * docs(volume_server): record jwt wrong-cookie mismatch coverage * test(volume_server/http): add jwt expired-token rejection coverage * docs(volume_server): record jwt expired-token coverage * test(volume_server/http): add jwt query and cookie transport coverage * docs(volume_server): record jwt token transport coverage * test(volume_server/http): add jwt token-source precedence coverage * docs(volume_server): record jwt token-source precedence coverage * test(volume_server/http): add jwt header-over-cookie precedence coverage * docs(volume_server): record jwt header cookie precedence coverage * test(volume_server/http): add jwt query-over-cookie precedence coverage * docs(volume_server): record jwt query cookie precedence coverage * test(volume_server/grpc): add setstate version mismatch and nil-state coverage * docs(volume_server): record setstate validation coverage * test(volume_server/grpc): add readonly persist-true lifecycle coverage * docs(volume_server): record readonly persist variation coverage * test(volume_server/http): add options origin cors header coverage * docs(volume_server): record options origin cors coverage * test(volume_server/http): add trace unsupported-method parity coverage * docs(volume_server): record trace method parity coverage * test(volume_server/grpc): add batch delete cookie-check variation coverage * docs(volume_server): record batch delete cookie-check coverage * test(volume_server/grpc): add admin lifecycle missing and maintenance variants * docs(volume_server): record admin lifecycle edge-case coverage * test(volume_server/grpc): add mixed batch delete status matrix coverage * docs(volume_server): record mixed batch delete matrix coverage * test(volume_server/http): add jwt-profile ui access gating coverage * docs(volume_server): record jwt ui-gating http coverage * test(volume_server/http): add propfind unsupported-method parity coverage * docs(volume_server): record propfind method parity coverage * test(volume_server/grpc): add volume configure success and rollback-path coverage * docs(volume_server): record volume configure branch coverage * test(volume_server/grpc): add volume needle status missing-path coverage * docs(volume_server): record volume needle status error-path coverage * test(volume_server/http): add readDeleted query behavior coverage * docs(volume_server): record readDeleted http behavior coverage * test(volume_server/http): add delete ts override parity coverage * docs(volume_server): record delete ts parity coverage * test(volume_server/grpc): add invalid blob/meta offset coverage * docs(volume_server): record invalid blob/meta offset coverage * test(volume_server/grpc): add read-all mixed volume abort coverage * docs(volume_server): record read-all mixed-volume abort coverage * test(volume_server/http): assert head response body parity * docs(volume_server): record head body parity assertion * test(volume_server/grpc): assert status state and memory payload completeness * docs(volume_server): record volume server status payload coverage * test(volume_server/grpc): add batch delete chunk-manifest rejection coverage * docs(volume_server): record batch delete chunk-manifest coverage * test(volume_server/grpc): add query cookie-mismatch eof parity coverage * docs(volume_server): record query cookie-mismatch parity coverage * test(volume_server/grpc): add ping master success target coverage * docs(volume_server): record ping master success coverage * test(volume_server/http): add head if-none-match conditional parity * docs(volume_server): record head if-none-match parity coverage * test(volume_server/http): add head if-modified-since parity coverage * docs(volume_server): record head if-modified-since parity coverage * test(volume_server/http): add connect unsupported-method parity coverage * docs(volume_server): record connect method parity coverage * test(volume_server/http): assert options allow-headers cors parity * docs(volume_server): record options allow-headers coverage * test(volume_server/framework): add dual volume cluster integration harness * test(volume_server/http): add missing-local read mode proxy redirect local coverage * docs(volume_server): record read mode missing-local matrix coverage * test(volume_server/http): add download over-limit replica proxy fallback coverage * docs(volume_server): record download replica fallback coverage * test(volume_server/http): add missing-local readDeleted proxy redirect parity coverage * docs(volume_server): record missing-local readDeleted mode coverage * test(volume_server/framework): add single-volume cluster with filer harness * test(volume_server/grpc): add ping filer success target coverage * docs(volume_server): record ping filer success coverage * test(volume_server/http): add proxied-loop guard download timeout coverage * docs(volume_server): record proxied-loop download coverage * test(volume_server/http): add disabled upload and download limit coverage * docs(volume_server): record disabled throttling path coverage * test(volume_server/grpc): add idempotent volume server leave coverage * docs(volume_server): record leave idempotence coverage * test(volume_server/http): add redirect collection query preservation coverage * docs(volume_server): record redirect collection query coverage * test(volume_server/http): assert admin server headers on status and health * docs(volume_server): record admin server header coverage * test(volume_server/http): assert healthz request-id echo parity * docs(volume_server): record healthz request-id parity coverage * test(volume_server/http): add over-limit invalid-vid download branch coverage * docs(volume_server): record over-limit invalid-vid branch coverage * test(volume_server/http): add public-port static asset coverage * docs(volume_server): record public static endpoint coverage * test(volume_server/http): add public head method parity coverage * docs(volume_server): record public head parity coverage * test(volume_server/http): add throttling wait-then-proceed path coverage * docs(volume_server): record throttling wait-then-proceed coverage * test(volume_server/http): add read cookie-mismatch not-found coverage * docs(volume_server): record read cookie-mismatch coverage * test(volume_server/http): add throttling timeout-recovery coverage * docs(volume_server): record throttling timeout-recovery coverage * test(volume_server/grpc): add ec generate mount info unmount lifecycle coverage * docs(volume_server): record ec positive lifecycle coverage * test(volume_server/grpc): add ec shard read and blob delete lifecycle coverage * docs(volume_server): record ec shard read/blob delete lifecycle coverage * test(volume_server/grpc): add ec rebuild and to-volume error branch coverage * docs(volume_server): record ec rebuild and to-volume branch coverage * test(volume_server/grpc): add ec shards-to-volume success roundtrip coverage * docs(volume_server): record ec shards-to-volume success coverage * test(volume_server/grpc): add ec receive and copy-file missing-source coverage * docs(volume_server): record ec receive and copy-file coverage * test(volume_server/grpc): add ec last-shard delete cleanup coverage * docs(volume_server): record ec last-shard delete cleanup coverage * test(volume_server/grpc): add volume copy success path coverage * docs(volume_server): record volume copy success coverage * test(volume_server/grpc): add volume copy overwrite-destination coverage * docs(volume_server): record volume copy overwrite coverage * test(volume_server/http): add write error-path variant coverage * docs(volume_server): record http write error-path coverage * test(volume_server/http): add conditional header precedence coverage * docs(volume_server): record conditional header precedence coverage * test(volume_server/http): add oversized combined range guard coverage * docs(volume_server): record oversized range guard coverage * test(volume_server/http): add image resize and crop read coverage * docs(volume_server): record image transform coverage * test(volume_server/http): add chunk-manifest expansion and bypass coverage * docs(volume_server): record chunk-manifest read coverage * test(volume_server/http): add compressed read encoding matrix coverage * docs(volume_server): record compressed read matrix coverage * test(volume_server/grpc): add tail receiver source replication coverage * docs(volume_server): record tail receiver replication coverage * test(volume_server/grpc): add tail sender large-needle chunking coverage * docs(volume_server): record tail sender chunking coverage * test(volume_server/grpc): add ec-backed volume needle status coverage * docs(volume_server): record ec-backed needle status coverage * test(volume_server/grpc): add ec shard copy from peer success coverage * docs(volume_server): record ec shard copy success coverage * test(volume_server/http): add chunk-manifest delete child cleanup coverage * docs(volume_server): record chunk-manifest delete cleanup coverage * test(volume_server/http): add chunk-manifest delete failure-path coverage * docs(volume_server): record chunk-manifest delete failure coverage * test(volume_server/grpc): add ec shard copy source-unavailable coverage * docs(volume_server): record ec shard copy source-unavailable coverage * parallelpull/8339/head
Chris Lu
4 weeks ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
41 changed files with 9459 additions and 0 deletions
-
122.github/workflows/volume-server-integration-tests.yml
-
1129test/volume_server/DEV_PLAN.md
-
7test/volume_server/Makefile
-
27test/volume_server/README.md
-
442test/volume_server/framework/cluster.go
-
293test/volume_server/framework/cluster_dual.go
-
91test/volume_server/framework/cluster_with_filer.go
-
8test/volume_server/framework/fault_injection.go
-
28test/volume_server/framework/grpc_client.go
-
34test/volume_server/framework/http_client.go
-
56test/volume_server/framework/volume_fixture.go
-
445test/volume_server/grpc/admin_extra_test.go
-
215test/volume_server/grpc/admin_lifecycle_test.go
-
177test/volume_server/grpc/admin_readonly_collection_test.go
-
264test/volume_server/grpc/batch_delete_test.go
-
431test/volume_server/grpc/copy_receive_variants_test.go
-
284test/volume_server/grpc/copy_sync_test.go
-
146test/volume_server/grpc/data_rw_test.go
-
273test/volume_server/grpc/data_stream_success_test.go
-
777test/volume_server/grpc/erasure_coding_test.go
-
139test/volume_server/grpc/health_state_test.go
-
385test/volume_server/grpc/scrub_query_test.go
-
206test/volume_server/grpc/tail_test.go
-
236test/volume_server/grpc/tiering_remote_test.go
-
87test/volume_server/grpc/vacuum_test.go
-
174test/volume_server/http/admin_test.go
-
419test/volume_server/http/auth_test.go
-
232test/volume_server/http/chunk_manifest_test.go
-
97test/volume_server/http/compressed_read_test.go
-
102test/volume_server/http/headers_static_test.go
-
92test/volume_server/http/image_transform_test.go
-
287test/volume_server/http/public_cors_methods_test.go
-
82test/volume_server/http/range_variants_test.go
-
54test/volume_server/http/read_deleted_test.go
-
319test/volume_server/http/read_mode_proxy_redirect_test.go
-
191test/volume_server/http/read_path_variants_test.go
-
123test/volume_server/http/read_write_delete_test.go
-
730test/volume_server/http/throttling_test.go
-
118test/volume_server/http/write_delete_variants_test.go
-
74test/volume_server/http/write_error_variants_test.go
-
63test/volume_server/matrix/config_profiles.go
@ -0,0 +1,122 @@ |
|||
name: "Volume Server Integration Tests" |
|||
|
|||
on: |
|||
pull_request: |
|||
branches: [ master ] |
|||
paths: |
|||
- 'test/volume_server/**' |
|||
- 'weed/server/**' |
|||
- 'weed/storage/**' |
|||
- 'weed/pb/volume_server.proto' |
|||
- 'weed/pb/volume_server_pb/**' |
|||
- '.github/workflows/volume-server-integration-tests.yml' |
|||
push: |
|||
branches: [ master, main ] |
|||
paths: |
|||
- 'test/volume_server/**' |
|||
- 'weed/server/**' |
|||
- 'weed/storage/**' |
|||
- 'weed/pb/volume_server.proto' |
|||
- 'weed/pb/volume_server_pb/**' |
|||
- '.github/workflows/volume-server-integration-tests.yml' |
|||
|
|||
concurrency: |
|||
group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests |
|||
cancel-in-progress: true |
|||
|
|||
permissions: |
|||
contents: read |
|||
|
|||
env: |
|||
GO_VERSION: '1.24' |
|||
TEST_TIMEOUT: '30m' |
|||
|
|||
jobs: |
|||
volume-server-integration-tests: |
|||
name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }}) |
|||
runs-on: ubuntu-22.04 |
|||
timeout-minutes: 45 |
|||
strategy: |
|||
fail-fast: false |
|||
matrix: |
|||
test-type: [grpc, http] |
|||
shard: [1, 2, 3] |
|||
|
|||
steps: |
|||
- name: Checkout code |
|||
uses: actions/checkout@v6 |
|||
|
|||
- name: Set up Go ${{ env.GO_VERSION }} |
|||
uses: actions/setup-go@v6 |
|||
with: |
|||
go-version: ${{ env.GO_VERSION }} |
|||
|
|||
- name: Build SeaweedFS binary |
|||
run: | |
|||
cd weed |
|||
go build -o weed . |
|||
chmod +x weed |
|||
./weed version |
|||
|
|||
- name: Run volume server integration tests |
|||
env: |
|||
WEED_BINARY: ${{ github.workspace }}/weed/weed |
|||
run: | |
|||
if [ "${{ matrix.test-type }}" == "grpc" ]; then |
|||
if [ "${{ matrix.shard }}" == "1" ]; then |
|||
TEST_PATTERN="^Test[A-H]" |
|||
elif [ "${{ matrix.shard }}" == "2" ]; then |
|||
TEST_PATTERN="^Test[I-S]" |
|||
else |
|||
TEST_PATTERN="^Test[T-Z]" |
|||
fi |
|||
else |
|||
if [ "${{ matrix.shard }}" == "1" ]; then |
|||
TEST_PATTERN="^Test[A-G]" |
|||
elif [ "${{ matrix.shard }}" == "2" ]; then |
|||
TEST_PATTERN="^Test[H-R]" |
|||
else |
|||
TEST_PATTERN="^Test[S-Z]" |
|||
fi |
|||
fi |
|||
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..." |
|||
go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}" |
|||
|
|||
- name: Collect logs on failure |
|||
if: failure() |
|||
run: | |
|||
mkdir -p /tmp/volume-server-it-logs |
|||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-it-logs/ \; || true |
|||
|
|||
- name: Archive logs on failure |
|||
if: failure() |
|||
uses: actions/upload-artifact@v6 |
|||
with: |
|||
name: volume-server-integration-test-logs |
|||
path: /tmp/volume-server-it-logs/ |
|||
if-no-files-found: warn |
|||
retention-days: 7 |
|||
|
|||
- name: Test summary |
|||
if: always() |
|||
run: | |
|||
if [ "${{ matrix.test-type }}" == "grpc" ]; then |
|||
if [ "${{ matrix.shard }}" == "1" ]; then |
|||
TEST_PATTERN="^Test[A-H]" |
|||
elif [ "${{ matrix.shard }}" == "2" ]; then |
|||
TEST_PATTERN="^Test[I-S]" |
|||
else |
|||
TEST_PATTERN="^Test[T-Z]" |
|||
fi |
|||
else |
|||
if [ "${{ matrix.shard }}" == "1" ]; then |
|||
TEST_PATTERN="^Test[A-G]" |
|||
elif [ "${{ matrix.shard }}" == "2" ]; then |
|||
TEST_PATTERN="^Test[H-R]" |
|||
else |
|||
TEST_PATTERN="^Test[S-Z]" |
|||
fi |
|||
fi |
|||
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY" |
|||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY" |
|||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY" |
|||
1129
test/volume_server/DEV_PLAN.md
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
@ -0,0 +1,7 @@ |
|||
.PHONY: test-volume-server test-volume-server-short |
|||
|
|||
test-volume-server: |
|||
go test ./test/volume_server/... -v |
|||
|
|||
test-volume-server-short: |
|||
go test ./test/volume_server/... -short -v |
|||
@ -0,0 +1,27 @@ |
|||
# Volume Server Integration Tests |
|||
|
|||
This package contains integration tests for SeaweedFS volume server HTTP and gRPC APIs. |
|||
|
|||
## Run Tests |
|||
|
|||
Run tests from repo root: |
|||
|
|||
```bash |
|||
go test ./test/volume_server/... -v |
|||
``` |
|||
|
|||
If a `weed` binary is not found, the harness will build one automatically. |
|||
|
|||
## Optional environment variables |
|||
|
|||
- `WEED_BINARY`: explicit path to the `weed` executable (disables auto-build). |
|||
- `VOLUME_SERVER_IT_KEEP_LOGS=1`: keep temporary test directories and process logs. |
|||
|
|||
## Current scope (Phase 0) |
|||
|
|||
- Shared cluster/framework utilities |
|||
- Matrix profile definitions |
|||
- Initial HTTP admin endpoint checks |
|||
- Initial gRPC state/status checks |
|||
|
|||
More API coverage is tracked in `/Users/chris/dev/seaweedfs2/test/volume_server/DEV_PLAN.md`. |
|||
@ -0,0 +1,442 @@ |
|||
package framework |
|||
|
|||
import ( |
|||
"bufio" |
|||
"bytes" |
|||
"errors" |
|||
"fmt" |
|||
"io" |
|||
"net" |
|||
"net/http" |
|||
"os" |
|||
"os/exec" |
|||
"path/filepath" |
|||
"runtime" |
|||
"strconv" |
|||
"strings" |
|||
"sync" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
const ( |
|||
defaultWaitTimeout = 30 * time.Second |
|||
defaultWaitTick = 200 * time.Millisecond |
|||
testVolumeSizeLimitMB = 32 |
|||
) |
|||
|
|||
// Cluster is a lightweight SeaweedFS master + one volume server test harness.
|
|||
type Cluster struct { |
|||
testingTB testing.TB |
|||
profile matrix.Profile |
|||
|
|||
weedBinary string |
|||
baseDir string |
|||
configDir string |
|||
logsDir string |
|||
keepLogs bool |
|||
|
|||
masterPort int |
|||
masterGrpcPort int |
|||
volumePort int |
|||
volumeGrpcPort int |
|||
volumePubPort int |
|||
|
|||
masterCmd *exec.Cmd |
|||
volumeCmd *exec.Cmd |
|||
|
|||
cleanupOnce sync.Once |
|||
} |
|||
|
|||
// StartSingleVolumeCluster boots one master and one volume server.
|
|||
func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster { |
|||
t.Helper() |
|||
|
|||
weedBinary, err := FindOrBuildWeedBinary() |
|||
if err != nil { |
|||
t.Fatalf("resolve weed binary: %v", err) |
|||
} |
|||
|
|||
baseDir, keepLogs, err := newWorkDir() |
|||
if err != nil { |
|||
t.Fatalf("create temp test directory: %v", err) |
|||
} |
|||
|
|||
configDir := filepath.Join(baseDir, "config") |
|||
logsDir := filepath.Join(baseDir, "logs") |
|||
masterDataDir := filepath.Join(baseDir, "master") |
|||
volumeDataDir := filepath.Join(baseDir, "volume") |
|||
for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir} { |
|||
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil { |
|||
t.Fatalf("create %s: %v", dir, mkErr) |
|||
} |
|||
} |
|||
|
|||
if err = writeSecurityConfig(configDir, profile); err != nil { |
|||
t.Fatalf("write security config: %v", err) |
|||
} |
|||
|
|||
masterPort, masterGrpcPort, err := allocateMasterPortPair() |
|||
if err != nil { |
|||
t.Fatalf("allocate master port pair: %v", err) |
|||
} |
|||
|
|||
ports, err := allocatePorts(3) |
|||
if err != nil { |
|||
t.Fatalf("allocate ports: %v", err) |
|||
} |
|||
|
|||
c := &Cluster{ |
|||
testingTB: t, |
|||
profile: profile, |
|||
weedBinary: weedBinary, |
|||
baseDir: baseDir, |
|||
configDir: configDir, |
|||
logsDir: logsDir, |
|||
keepLogs: keepLogs, |
|||
masterPort: masterPort, |
|||
masterGrpcPort: masterGrpcPort, |
|||
volumePort: ports[0], |
|||
volumeGrpcPort: ports[1], |
|||
volumePubPort: ports[0], |
|||
} |
|||
if profile.SplitPublicPort { |
|||
c.volumePubPort = ports[2] |
|||
} |
|||
|
|||
if err = c.startMaster(masterDataDir); err != nil { |
|||
c.Stop() |
|||
t.Fatalf("start master: %v", err) |
|||
} |
|||
if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil { |
|||
masterLog := c.tailLog("master.log") |
|||
c.Stop() |
|||
t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog) |
|||
} |
|||
|
|||
if err = c.startVolume(volumeDataDir); err != nil { |
|||
masterLog := c.tailLog("master.log") |
|||
c.Stop() |
|||
t.Fatalf("start volume: %v\nmaster log tail:\n%s", err, masterLog) |
|||
} |
|||
if err = c.waitForHTTP(c.VolumeAdminURL() + "/status"); err != nil { |
|||
volumeLog := c.tailLog("volume.log") |
|||
c.Stop() |
|||
t.Fatalf("wait for volume readiness: %v\nvolume log tail:\n%s", err, volumeLog) |
|||
} |
|||
if err = c.waitForTCP(c.VolumeGRPCAddress()); err != nil { |
|||
volumeLog := c.tailLog("volume.log") |
|||
c.Stop() |
|||
t.Fatalf("wait for volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog) |
|||
} |
|||
|
|||
t.Cleanup(func() { |
|||
c.Stop() |
|||
}) |
|||
|
|||
return c |
|||
} |
|||
|
|||
// Stop terminates all processes and cleans temporary files.
|
|||
func (c *Cluster) Stop() { |
|||
if c == nil { |
|||
return |
|||
} |
|||
c.cleanupOnce.Do(func() { |
|||
stopProcess(c.volumeCmd) |
|||
stopProcess(c.masterCmd) |
|||
if !c.keepLogs && !c.testingTB.Failed() { |
|||
_ = os.RemoveAll(c.baseDir) |
|||
} else if c.baseDir != "" { |
|||
c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir) |
|||
} |
|||
}) |
|||
} |
|||
|
|||
func (c *Cluster) startMaster(dataDir string) error { |
|||
logFile, err := os.Create(filepath.Join(c.logsDir, "master.log")) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
args := []string{ |
|||
"-config_dir=" + c.configDir, |
|||
"master", |
|||
"-ip=127.0.0.1", |
|||
"-port=" + strconv.Itoa(c.masterPort), |
|||
"-port.grpc=" + strconv.Itoa(c.masterGrpcPort), |
|||
"-mdir=" + dataDir, |
|||
"-peers=none", |
|||
"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB), |
|||
"-defaultReplication=000", |
|||
} |
|||
|
|||
c.masterCmd = exec.Command(c.weedBinary, args...) |
|||
c.masterCmd.Dir = c.baseDir |
|||
c.masterCmd.Stdout = logFile |
|||
c.masterCmd.Stderr = logFile |
|||
return c.masterCmd.Start() |
|||
} |
|||
|
|||
func (c *Cluster) startVolume(dataDir string) error { |
|||
logFile, err := os.Create(filepath.Join(c.logsDir, "volume.log")) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
args := []string{ |
|||
"-config_dir=" + c.configDir, |
|||
"volume", |
|||
"-ip=127.0.0.1", |
|||
"-port=" + strconv.Itoa(c.volumePort), |
|||
"-port.grpc=" + strconv.Itoa(c.volumeGrpcPort), |
|||
"-port.public=" + strconv.Itoa(c.volumePubPort), |
|||
"-dir=" + dataDir, |
|||
"-max=16", |
|||
"-master=127.0.0.1:" + strconv.Itoa(c.masterPort), |
|||
"-readMode=" + c.profile.ReadMode, |
|||
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB), |
|||
"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB), |
|||
} |
|||
if c.profile.InflightUploadTimeout > 0 { |
|||
args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String()) |
|||
} |
|||
if c.profile.InflightDownloadTimeout > 0 { |
|||
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String()) |
|||
} |
|||
|
|||
c.volumeCmd = exec.Command(c.weedBinary, args...) |
|||
c.volumeCmd.Dir = c.baseDir |
|||
c.volumeCmd.Stdout = logFile |
|||
c.volumeCmd.Stderr = logFile |
|||
return c.volumeCmd.Start() |
|||
} |
|||
|
|||
func (c *Cluster) waitForHTTP(url string) error { |
|||
client := &http.Client{Timeout: 1 * time.Second} |
|||
deadline := time.Now().Add(defaultWaitTimeout) |
|||
for time.Now().Before(deadline) { |
|||
resp, err := client.Get(url) |
|||
if err == nil { |
|||
_, _ = io.Copy(io.Discard, resp.Body) |
|||
resp.Body.Close() |
|||
if resp.StatusCode < 500 { |
|||
return nil |
|||
} |
|||
} |
|||
time.Sleep(defaultWaitTick) |
|||
} |
|||
return fmt.Errorf("timed out waiting for %s", url) |
|||
} |
|||
|
|||
func (c *Cluster) waitForTCP(addr string) error { |
|||
deadline := time.Now().Add(defaultWaitTimeout) |
|||
for time.Now().Before(deadline) { |
|||
conn, err := net.DialTimeout("tcp", addr, time.Second) |
|||
if err == nil { |
|||
_ = conn.Close() |
|||
return nil |
|||
} |
|||
time.Sleep(defaultWaitTick) |
|||
} |
|||
return fmt.Errorf("timed out waiting for tcp %s", addr) |
|||
} |
|||
|
|||
func stopProcess(cmd *exec.Cmd) { |
|||
if cmd == nil || cmd.Process == nil { |
|||
return |
|||
} |
|||
|
|||
_ = cmd.Process.Signal(os.Interrupt) |
|||
done := make(chan error, 1) |
|||
go func() { |
|||
done <- cmd.Wait() |
|||
}() |
|||
|
|||
select { |
|||
case <-time.After(10 * time.Second): |
|||
_ = cmd.Process.Kill() |
|||
<-done |
|||
case <-done: |
|||
} |
|||
} |
|||
|
|||
func allocatePorts(count int) ([]int, error) { |
|||
listeners := make([]net.Listener, 0, count) |
|||
ports := make([]int, 0, count) |
|||
for i := 0; i < count; i++ { |
|||
l, err := net.Listen("tcp", "127.0.0.1:0") |
|||
if err != nil { |
|||
for _, ll := range listeners { |
|||
_ = ll.Close() |
|||
} |
|||
return nil, err |
|||
} |
|||
listeners = append(listeners, l) |
|||
ports = append(ports, l.Addr().(*net.TCPAddr).Port) |
|||
} |
|||
for _, l := range listeners { |
|||
_ = l.Close() |
|||
} |
|||
return ports, nil |
|||
} |
|||
|
|||
func allocateMasterPortPair() (int, int, error) { |
|||
for masterPort := 10000; masterPort <= 55535; masterPort++ { |
|||
masterGrpcPort := masterPort + 10000 |
|||
l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort))) |
|||
if err != nil { |
|||
continue |
|||
} |
|||
l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterGrpcPort))) |
|||
if err != nil { |
|||
_ = l1.Close() |
|||
continue |
|||
} |
|||
_ = l2.Close() |
|||
_ = l1.Close() |
|||
return masterPort, masterGrpcPort, nil |
|||
} |
|||
return 0, 0, errors.New("unable to find available master port pair") |
|||
} |
|||
|
|||
func newWorkDir() (dir string, keepLogs bool, err error) { |
|||
keepLogs = os.Getenv("VOLUME_SERVER_IT_KEEP_LOGS") == "1" |
|||
dir, err = os.MkdirTemp("", "seaweedfs_volume_server_it_") |
|||
return dir, keepLogs, err |
|||
} |
|||
|
|||
func writeSecurityConfig(configDir string, profile matrix.Profile) error { |
|||
var b strings.Builder |
|||
if profile.EnableJWT { |
|||
if profile.JWTSigningKey == "" || profile.JWTReadKey == "" { |
|||
return errors.New("jwt profile requires both write and read keys") |
|||
} |
|||
b.WriteString("[jwt.signing]\n") |
|||
b.WriteString("key = \"") |
|||
b.WriteString(profile.JWTSigningKey) |
|||
b.WriteString("\"\n") |
|||
b.WriteString("expires_after_seconds = 60\n\n") |
|||
|
|||
b.WriteString("[jwt.signing.read]\n") |
|||
b.WriteString("key = \"") |
|||
b.WriteString(profile.JWTReadKey) |
|||
b.WriteString("\"\n") |
|||
b.WriteString("expires_after_seconds = 60\n") |
|||
} |
|||
if b.Len() == 0 { |
|||
b.WriteString("# optional security config generated for integration tests\n") |
|||
} |
|||
return os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(b.String()), 0o644) |
|||
} |
|||
|
|||
// FindOrBuildWeedBinary returns an executable weed binary, building one when needed.
|
|||
func FindOrBuildWeedBinary() (string, error) { |
|||
if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" { |
|||
if isExecutableFile(fromEnv) { |
|||
return fromEnv, nil |
|||
} |
|||
return "", fmt.Errorf("WEED_BINARY is set but not executable: %s", fromEnv) |
|||
} |
|||
|
|||
repoRoot := "" |
|||
if _, file, _, ok := runtime.Caller(0); ok { |
|||
repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..")) |
|||
candidate := filepath.Join(repoRoot, "weed", "weed") |
|||
if isExecutableFile(candidate) { |
|||
return candidate, nil |
|||
} |
|||
} |
|||
|
|||
if repoRoot == "" { |
|||
return "", errors.New("unable to detect repository root") |
|||
} |
|||
|
|||
binDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_bin") |
|||
if err := os.MkdirAll(binDir, 0o755); err != nil { |
|||
return "", fmt.Errorf("create binary directory %s: %w", binDir, err) |
|||
} |
|||
binPath := filepath.Join(binDir, "weed") |
|||
if isExecutableFile(binPath) { |
|||
return binPath, nil |
|||
} |
|||
|
|||
cmd := exec.Command("go", "build", "-o", binPath, ".") |
|||
cmd.Dir = filepath.Join(repoRoot, "weed") |
|||
var out bytes.Buffer |
|||
cmd.Stdout = &out |
|||
cmd.Stderr = &out |
|||
if err := cmd.Run(); err != nil { |
|||
return "", fmt.Errorf("build weed binary: %w\n%s", err, out.String()) |
|||
} |
|||
if !isExecutableFile(binPath) { |
|||
return "", fmt.Errorf("built weed binary is not executable: %s", binPath) |
|||
} |
|||
return binPath, nil |
|||
} |
|||
|
|||
func isExecutableFile(path string) bool { |
|||
info, err := os.Stat(path) |
|||
if err != nil || info.IsDir() { |
|||
return false |
|||
} |
|||
mode := info.Mode().Perm() |
|||
return mode&0o111 != 0 |
|||
} |
|||
|
|||
func (c *Cluster) tailLog(logName string) string { |
|||
f, err := os.Open(filepath.Join(c.logsDir, logName)) |
|||
if err != nil { |
|||
return "" |
|||
} |
|||
defer f.Close() |
|||
|
|||
scanner := bufio.NewScanner(f) |
|||
lines := make([]string, 0, 40) |
|||
for scanner.Scan() { |
|||
lines = append(lines, scanner.Text()) |
|||
if len(lines) > 40 { |
|||
lines = lines[1:] |
|||
} |
|||
} |
|||
return strings.Join(lines, "\n") |
|||
} |
|||
|
|||
func (c *Cluster) MasterAddress() string { |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort)) |
|||
} |
|||
|
|||
func (c *Cluster) VolumeAdminAddress() string { |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort)) |
|||
} |
|||
|
|||
func (c *Cluster) VolumePublicAddress() string { |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort)) |
|||
} |
|||
|
|||
func (c *Cluster) VolumeGRPCAddress() string { |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort)) |
|||
} |
|||
|
|||
// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
|
|||
func (c *Cluster) VolumeServerAddress() string { |
|||
return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(), c.volumeGrpcPort) |
|||
} |
|||
|
|||
func (c *Cluster) MasterURL() string { |
|||
return "http://" + c.MasterAddress() |
|||
} |
|||
|
|||
func (c *Cluster) VolumeAdminURL() string { |
|||
return "http://" + c.VolumeAdminAddress() |
|||
} |
|||
|
|||
func (c *Cluster) VolumePublicURL() string { |
|||
return "http://" + c.VolumePublicAddress() |
|||
} |
|||
|
|||
func (c *Cluster) BaseDir() string { |
|||
return c.baseDir |
|||
} |
|||
@ -0,0 +1,293 @@ |
|||
package framework |
|||
|
|||
import ( |
|||
"fmt" |
|||
"net" |
|||
"os" |
|||
"os/exec" |
|||
"path/filepath" |
|||
"strconv" |
|||
"sync" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
type DualVolumeCluster struct { |
|||
testingTB testing.TB |
|||
profile matrix.Profile |
|||
|
|||
weedBinary string |
|||
baseDir string |
|||
configDir string |
|||
logsDir string |
|||
keepLogs bool |
|||
|
|||
masterPort int |
|||
masterGrpcPort int |
|||
|
|||
volumePort0 int |
|||
volumeGrpcPort0 int |
|||
volumePubPort0 int |
|||
volumePort1 int |
|||
volumeGrpcPort1 int |
|||
volumePubPort1 int |
|||
|
|||
masterCmd *exec.Cmd |
|||
volumeCmd0 *exec.Cmd |
|||
volumeCmd1 *exec.Cmd |
|||
|
|||
cleanupOnce sync.Once |
|||
} |
|||
|
|||
func StartDualVolumeCluster(t testing.TB, profile matrix.Profile) *DualVolumeCluster { |
|||
t.Helper() |
|||
|
|||
weedBinary, err := FindOrBuildWeedBinary() |
|||
if err != nil { |
|||
t.Fatalf("resolve weed binary: %v", err) |
|||
} |
|||
|
|||
baseDir, keepLogs, err := newWorkDir() |
|||
if err != nil { |
|||
t.Fatalf("create temp test directory: %v", err) |
|||
} |
|||
|
|||
configDir := filepath.Join(baseDir, "config") |
|||
logsDir := filepath.Join(baseDir, "logs") |
|||
masterDataDir := filepath.Join(baseDir, "master") |
|||
volumeDataDir0 := filepath.Join(baseDir, "volume0") |
|||
volumeDataDir1 := filepath.Join(baseDir, "volume1") |
|||
for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir0, volumeDataDir1} { |
|||
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil { |
|||
t.Fatalf("create %s: %v", dir, mkErr) |
|||
} |
|||
} |
|||
|
|||
if err = writeSecurityConfig(configDir, profile); err != nil { |
|||
t.Fatalf("write security config: %v", err) |
|||
} |
|||
|
|||
masterPort, masterGrpcPort, err := allocateMasterPortPair() |
|||
if err != nil { |
|||
t.Fatalf("allocate master port pair: %v", err) |
|||
} |
|||
|
|||
ports, err := allocatePorts(6) |
|||
if err != nil { |
|||
t.Fatalf("allocate volume ports: %v", err) |
|||
} |
|||
|
|||
c := &DualVolumeCluster{ |
|||
testingTB: t, |
|||
profile: profile, |
|||
weedBinary: weedBinary, |
|||
baseDir: baseDir, |
|||
configDir: configDir, |
|||
logsDir: logsDir, |
|||
keepLogs: keepLogs, |
|||
masterPort: masterPort, |
|||
masterGrpcPort: masterGrpcPort, |
|||
volumePort0: ports[0], |
|||
volumeGrpcPort0: ports[1], |
|||
volumePubPort0: ports[0], |
|||
volumePort1: ports[2], |
|||
volumeGrpcPort1: ports[3], |
|||
volumePubPort1: ports[2], |
|||
} |
|||
if profile.SplitPublicPort { |
|||
c.volumePubPort0 = ports[4] |
|||
c.volumePubPort1 = ports[5] |
|||
} |
|||
|
|||
if err = c.startMaster(masterDataDir); err != nil { |
|||
c.Stop() |
|||
t.Fatalf("start master: %v", err) |
|||
} |
|||
if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil { |
|||
masterLog := c.tailLog("master.log") |
|||
c.Stop() |
|||
t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog) |
|||
} |
|||
|
|||
if err = c.startVolume(0, volumeDataDir0); err != nil { |
|||
masterLog := c.tailLog("master.log") |
|||
c.Stop() |
|||
t.Fatalf("start first volume server: %v\nmaster log tail:\n%s", err, masterLog) |
|||
} |
|||
if err = c.waitForHTTP(c.VolumeAdminURL(0) + "/status"); err != nil { |
|||
volumeLog := c.tailLog("volume0.log") |
|||
c.Stop() |
|||
t.Fatalf("wait for first volume readiness: %v\nvolume log tail:\n%s", err, volumeLog) |
|||
} |
|||
if err = c.waitForTCP(c.VolumeGRPCAddress(0)); err != nil { |
|||
volumeLog := c.tailLog("volume0.log") |
|||
c.Stop() |
|||
t.Fatalf("wait for first volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog) |
|||
} |
|||
|
|||
if err = c.startVolume(1, volumeDataDir1); err != nil { |
|||
volumeLog := c.tailLog("volume0.log") |
|||
c.Stop() |
|||
t.Fatalf("start second volume server: %v\nfirst volume log tail:\n%s", err, volumeLog) |
|||
} |
|||
if err = c.waitForHTTP(c.VolumeAdminURL(1) + "/status"); err != nil { |
|||
volumeLog := c.tailLog("volume1.log") |
|||
c.Stop() |
|||
t.Fatalf("wait for second volume readiness: %v\nvolume log tail:\n%s", err, volumeLog) |
|||
} |
|||
if err = c.waitForTCP(c.VolumeGRPCAddress(1)); err != nil { |
|||
volumeLog := c.tailLog("volume1.log") |
|||
c.Stop() |
|||
t.Fatalf("wait for second volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog) |
|||
} |
|||
|
|||
t.Cleanup(func() { |
|||
c.Stop() |
|||
}) |
|||
|
|||
return c |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) Stop() { |
|||
if c == nil { |
|||
return |
|||
} |
|||
c.cleanupOnce.Do(func() { |
|||
stopProcess(c.volumeCmd1) |
|||
stopProcess(c.volumeCmd0) |
|||
stopProcess(c.masterCmd) |
|||
if !c.keepLogs && !c.testingTB.Failed() { |
|||
_ = os.RemoveAll(c.baseDir) |
|||
} else if c.baseDir != "" { |
|||
c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir) |
|||
} |
|||
}) |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) startMaster(dataDir string) error { |
|||
logFile, err := os.Create(filepath.Join(c.logsDir, "master.log")) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
args := []string{ |
|||
"-config_dir=" + c.configDir, |
|||
"master", |
|||
"-ip=127.0.0.1", |
|||
"-port=" + strconv.Itoa(c.masterPort), |
|||
"-port.grpc=" + strconv.Itoa(c.masterGrpcPort), |
|||
"-mdir=" + dataDir, |
|||
"-peers=none", |
|||
"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB), |
|||
"-defaultReplication=000", |
|||
} |
|||
|
|||
c.masterCmd = exec.Command(c.weedBinary, args...) |
|||
c.masterCmd.Dir = c.baseDir |
|||
c.masterCmd.Stdout = logFile |
|||
c.masterCmd.Stderr = logFile |
|||
return c.masterCmd.Start() |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) startVolume(index int, dataDir string) error { |
|||
logName := fmt.Sprintf("volume%d.log", index) |
|||
logFile, err := os.Create(filepath.Join(c.logsDir, logName)) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
volumePort := c.volumePort0 |
|||
volumeGrpcPort := c.volumeGrpcPort0 |
|||
volumePubPort := c.volumePubPort0 |
|||
if index == 1 { |
|||
volumePort = c.volumePort1 |
|||
volumeGrpcPort = c.volumeGrpcPort1 |
|||
volumePubPort = c.volumePubPort1 |
|||
} |
|||
|
|||
args := []string{ |
|||
"-config_dir=" + c.configDir, |
|||
"volume", |
|||
"-ip=127.0.0.1", |
|||
"-port=" + strconv.Itoa(volumePort), |
|||
"-port.grpc=" + strconv.Itoa(volumeGrpcPort), |
|||
"-port.public=" + strconv.Itoa(volumePubPort), |
|||
"-dir=" + dataDir, |
|||
"-max=16", |
|||
"-master=127.0.0.1:" + strconv.Itoa(c.masterPort), |
|||
"-readMode=" + c.profile.ReadMode, |
|||
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB), |
|||
"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB), |
|||
} |
|||
if c.profile.InflightUploadTimeout > 0 { |
|||
args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String()) |
|||
} |
|||
if c.profile.InflightDownloadTimeout > 0 { |
|||
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String()) |
|||
} |
|||
|
|||
cmd := exec.Command(c.weedBinary, args...) |
|||
cmd.Dir = c.baseDir |
|||
cmd.Stdout = logFile |
|||
cmd.Stderr = logFile |
|||
|
|||
if err = cmd.Start(); err != nil { |
|||
return err |
|||
} |
|||
if index == 1 { |
|||
c.volumeCmd1 = cmd |
|||
} else { |
|||
c.volumeCmd0 = cmd |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) waitForHTTP(url string) error { |
|||
return (&Cluster{}).waitForHTTP(url) |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) waitForTCP(addr string) error { |
|||
return (&Cluster{}).waitForTCP(addr) |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) tailLog(logName string) string { |
|||
return (&Cluster{logsDir: c.logsDir}).tailLog(logName) |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) MasterAddress() string { |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort)) |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) MasterURL() string { |
|||
return "http://" + c.MasterAddress() |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) VolumeAdminAddress(index int) string { |
|||
if index == 1 { |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort1)) |
|||
} |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort0)) |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) VolumePublicAddress(index int) string { |
|||
if index == 1 { |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort1)) |
|||
} |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort0)) |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) VolumeGRPCAddress(index int) string { |
|||
if index == 1 { |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort1)) |
|||
} |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort0)) |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) VolumeAdminURL(index int) string { |
|||
return "http://" + c.VolumeAdminAddress(index) |
|||
} |
|||
|
|||
func (c *DualVolumeCluster) VolumePublicURL(index int) string { |
|||
return "http://" + c.VolumePublicAddress(index) |
|||
} |
|||
@ -0,0 +1,91 @@ |
|||
package framework |
|||
|
|||
import ( |
|||
"fmt" |
|||
"net" |
|||
"os" |
|||
"os/exec" |
|||
"path/filepath" |
|||
"strconv" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
type ClusterWithFiler struct { |
|||
*Cluster |
|||
|
|||
filerCmd *exec.Cmd |
|||
filerPort int |
|||
filerGrpcPort int |
|||
} |
|||
|
|||
func StartSingleVolumeClusterWithFiler(t testing.TB, profile matrix.Profile) *ClusterWithFiler { |
|||
t.Helper() |
|||
|
|||
baseCluster := StartSingleVolumeCluster(t, profile) |
|||
|
|||
ports, err := allocatePorts(2) |
|||
if err != nil { |
|||
t.Fatalf("allocate filer ports: %v", err) |
|||
} |
|||
|
|||
filerDataDir := filepath.Join(baseCluster.baseDir, "filer") |
|||
if mkErr := os.MkdirAll(filerDataDir, 0o755); mkErr != nil { |
|||
t.Fatalf("create filer data dir: %v", mkErr) |
|||
} |
|||
|
|||
logFile, err := os.Create(filepath.Join(baseCluster.logsDir, "filer.log")) |
|||
if err != nil { |
|||
t.Fatalf("create filer log file: %v", err) |
|||
} |
|||
|
|||
filerPort := ports[0] |
|||
filerGrpcPort := ports[1] |
|||
args := []string{ |
|||
"-config_dir=" + baseCluster.configDir, |
|||
"filer", |
|||
"-master=127.0.0.1:" + strconv.Itoa(baseCluster.masterPort), |
|||
"-ip=127.0.0.1", |
|||
"-port=" + strconv.Itoa(filerPort), |
|||
"-port.grpc=" + strconv.Itoa(filerGrpcPort), |
|||
"-defaultStoreDir=" + filerDataDir, |
|||
} |
|||
|
|||
filerCmd := exec.Command(baseCluster.weedBinary, args...) |
|||
filerCmd.Dir = baseCluster.baseDir |
|||
filerCmd.Stdout = logFile |
|||
filerCmd.Stderr = logFile |
|||
if err = filerCmd.Start(); err != nil { |
|||
t.Fatalf("start filer: %v", err) |
|||
} |
|||
|
|||
if err = baseCluster.waitForTCP(net.JoinHostPort("127.0.0.1", strconv.Itoa(filerGrpcPort))); err != nil { |
|||
filerLogTail := baseCluster.tailLog("filer.log") |
|||
stopProcess(filerCmd) |
|||
t.Fatalf("wait for filer grpc readiness: %v\nfiler log tail:\n%s", err, filerLogTail) |
|||
} |
|||
|
|||
t.Cleanup(func() { |
|||
stopProcess(filerCmd) |
|||
}) |
|||
|
|||
return &ClusterWithFiler{ |
|||
Cluster: baseCluster, |
|||
filerCmd: filerCmd, |
|||
filerPort: filerPort, |
|||
filerGrpcPort: filerGrpcPort, |
|||
} |
|||
} |
|||
|
|||
func (c *ClusterWithFiler) FilerAddress() string { |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerPort)) |
|||
} |
|||
|
|||
func (c *ClusterWithFiler) FilerGRPCAddress() string { |
|||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerGrpcPort)) |
|||
} |
|||
|
|||
func (c *ClusterWithFiler) FilerServerAddress() string { |
|||
return fmt.Sprintf("%s.%d", c.FilerAddress(), c.filerGrpcPort) |
|||
} |
|||
@ -0,0 +1,8 @@ |
|||
package framework |
|||
|
|||
// Phase 0 placeholder for future fault injection utilities.
|
|||
//
|
|||
// Planned extensions:
|
|||
// - restart/kill selected processes
|
|||
// - temporary network isolation hooks
|
|||
// - master or peer outage helpers for proxy/replication branch coverage
|
|||
@ -0,0 +1,28 @@ |
|||
package framework |
|||
|
|||
import ( |
|||
"context" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
"google.golang.org/grpc" |
|||
"google.golang.org/grpc/credentials/insecure" |
|||
) |
|||
|
|||
func DialVolumeServer(t testing.TB, address string) (*grpc.ClientConn, volume_server_pb.VolumeServerClient) { |
|||
t.Helper() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
conn, err := grpc.DialContext(ctx, address, |
|||
grpc.WithTransportCredentials(insecure.NewCredentials()), |
|||
grpc.WithBlock(), |
|||
) |
|||
if err != nil { |
|||
t.Fatalf("dial volume grpc %s: %v", address, err) |
|||
} |
|||
|
|||
return conn, volume_server_pb.NewVolumeServerClient(conn) |
|||
} |
|||
@ -0,0 +1,34 @@ |
|||
package framework |
|||
|
|||
import ( |
|||
"io" |
|||
"net/http" |
|||
"testing" |
|||
"time" |
|||
) |
|||
|
|||
func NewHTTPClient() *http.Client { |
|||
return &http.Client{Timeout: 10 * time.Second} |
|||
} |
|||
|
|||
func DoRequest(t testing.TB, client *http.Client, req *http.Request) *http.Response { |
|||
t.Helper() |
|||
resp, err := client.Do(req) |
|||
if err != nil { |
|||
t.Fatalf("http request %s %s: %v", req.Method, req.URL.String(), err) |
|||
} |
|||
return resp |
|||
} |
|||
|
|||
func ReadAllAndClose(t testing.TB, resp *http.Response) []byte { |
|||
t.Helper() |
|||
if resp == nil { |
|||
return nil |
|||
} |
|||
defer resp.Body.Close() |
|||
body, err := io.ReadAll(resp.Body) |
|||
if err != nil { |
|||
t.Fatalf("read response body: %v", err) |
|||
} |
|||
return body |
|||
} |
|||
@ -0,0 +1,56 @@ |
|||
package framework |
|||
|
|||
import ( |
|||
"bytes" |
|||
"context" |
|||
"fmt" |
|||
"net/http" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/needle" |
|||
) |
|||
|
|||
func AllocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string) { |
|||
t.Helper() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: collection, |
|||
Replication: "000", |
|||
Version: uint32(needle.GetCurrentVersion()), |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("allocate volume %d: %v", volumeID, err) |
|||
} |
|||
} |
|||
|
|||
func NewFileID(volumeID uint32, key uint64, cookie uint32) string { |
|||
return needle.NewFileId(needle.VolumeId(volumeID), key, cookie).String() |
|||
} |
|||
|
|||
func UploadBytes(t testing.TB, client *http.Client, volumeURL, fid string, data []byte) *http.Response { |
|||
t.Helper() |
|||
|
|||
req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/%s", volumeURL, fid), bytes.NewReader(data)) |
|||
if err != nil { |
|||
t.Fatalf("build upload request: %v", err) |
|||
} |
|||
req.Header.Set("Content-Type", "application/octet-stream") |
|||
req.Header.Set("Content-Length", fmt.Sprintf("%d", len(data))) |
|||
return DoRequest(t, client, req) |
|||
} |
|||
|
|||
func ReadBytes(t testing.TB, client *http.Client, volumeURL, fid string) *http.Response { |
|||
t.Helper() |
|||
|
|||
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", volumeURL, fid), nil) |
|||
if err != nil { |
|||
t.Fatalf("build read request: %v", err) |
|||
} |
|||
return DoRequest(t, client, req) |
|||
} |
|||
@ -0,0 +1,445 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"context" |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/cluster" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
) |
|||
|
|||
func TestVolumeNeedleStatusForUploadedFile(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(21) |
|||
const needleID = uint64(778899) |
|||
const cookie = uint32(0xA1B2C3D4) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
client := framework.NewHTTPClient() |
|||
payload := []byte("needle-status-payload") |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ |
|||
VolumeId: volumeID, |
|||
NeedleId: needleID, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeNeedleStatus failed: %v", err) |
|||
} |
|||
if statusResp.GetNeedleId() != needleID { |
|||
t.Fatalf("needle id mismatch: got %d want %d", statusResp.GetNeedleId(), needleID) |
|||
} |
|||
if statusResp.GetCookie() != cookie { |
|||
t.Fatalf("cookie mismatch: got %d want %d", statusResp.GetCookie(), cookie) |
|||
} |
|||
if statusResp.GetSize() == 0 { |
|||
t.Fatalf("expected non-zero needle size") |
|||
} |
|||
} |
|||
|
|||
func TestVolumeNeedleStatusViaEcShardsWhenNormalVolumeUnmounted(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(26) |
|||
const needleID = uint64(778900) |
|||
const cookie = uint32(0xA1B2C3D5) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
payload := []byte("needle-status-ec-path-payload") |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsMount data shards failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{ |
|||
VolumeId: volumeID, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeUnmount failed: %v", err) |
|||
} |
|||
|
|||
statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ |
|||
VolumeId: volumeID, |
|||
NeedleId: needleID, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeNeedleStatus via EC shards failed: %v", err) |
|||
} |
|||
if statusResp.GetNeedleId() != needleID { |
|||
t.Fatalf("needle id mismatch: got %d want %d", statusResp.GetNeedleId(), needleID) |
|||
} |
|||
if statusResp.GetCookie() != cookie { |
|||
t.Fatalf("cookie mismatch: got %d want %d", statusResp.GetCookie(), cookie) |
|||
} |
|||
if statusResp.GetSize() == 0 { |
|||
t.Fatalf("expected non-zero needle size from EC-backed needle status") |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ |
|||
VolumeId: volumeID, |
|||
NeedleId: needleID + 999999, |
|||
}) |
|||
if err == nil || !strings.Contains(strings.ToLower(err.Error()), "not found") { |
|||
t.Fatalf("VolumeNeedleStatus via EC shards missing-needle error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeNeedleStatusMissingVolumeAndNeedle(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(25) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ |
|||
VolumeId: 99925, |
|||
NeedleId: 1, |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("VolumeNeedleStatus should fail for missing volume") |
|||
} |
|||
if !strings.Contains(strings.ToLower(err.Error()), "volume not found") { |
|||
t.Fatalf("VolumeNeedleStatus missing-volume error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ |
|||
VolumeId: volumeID, |
|||
NeedleId: 123456789, |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("VolumeNeedleStatus should fail for missing needle") |
|||
} |
|||
if !strings.Contains(strings.ToLower(err.Error()), "not found") { |
|||
t.Fatalf("VolumeNeedleStatus missing-needle error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func mustNewRequest(t testing.TB, method, url string) *http.Request { |
|||
t.Helper() |
|||
req, err := http.NewRequest(method, url, nil) |
|||
if err != nil { |
|||
t.Fatalf("create request %s %s: %v", method, url, err) |
|||
} |
|||
return req |
|||
} |
|||
|
|||
func TestVolumeConfigureInvalidReplication(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(22) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
resp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{ |
|||
VolumeId: volumeID, |
|||
Replication: "bad-replication", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeConfigure returned grpc error: %v", err) |
|||
} |
|||
if resp.GetError() == "" { |
|||
t.Fatalf("VolumeConfigure expected response error for invalid replication") |
|||
} |
|||
if !strings.Contains(strings.ToLower(resp.GetError()), "replication") { |
|||
t.Fatalf("VolumeConfigure error should mention replication, got: %q", resp.GetError()) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeConfigureSuccessAndMissingRollbackPath(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(24) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
successResp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{ |
|||
VolumeId: volumeID, |
|||
Replication: "000", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeConfigure success path returned grpc error: %v", err) |
|||
} |
|||
if successResp.GetError() != "" { |
|||
t.Fatalf("VolumeConfigure success path expected empty response error, got: %q", successResp.GetError()) |
|||
} |
|||
|
|||
statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeStatus after successful configure failed: %v", err) |
|||
} |
|||
if statusResp.GetIsReadOnly() { |
|||
t.Fatalf("VolumeStatus after configure expected writable volume") |
|||
} |
|||
|
|||
missingResp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{ |
|||
VolumeId: 99024, |
|||
Replication: "000", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeConfigure missing-volume branch should return response error, got grpc error: %v", err) |
|||
} |
|||
if missingResp.GetError() == "" { |
|||
t.Fatalf("VolumeConfigure missing-volume expected non-empty response error") |
|||
} |
|||
lower := strings.ToLower(missingResp.GetError()) |
|||
if !strings.Contains(lower, "not found on disk") { |
|||
t.Fatalf("VolumeConfigure missing-volume error should mention not found on disk, got: %q", missingResp.GetError()) |
|||
} |
|||
if !strings.Contains(lower, "failed to restore mount") { |
|||
t.Fatalf("VolumeConfigure missing-volume error should include remount rollback failure, got: %q", missingResp.GetError()) |
|||
} |
|||
} |
|||
|
|||
func TestPingVolumeTargetAndLeaveAffectsHealthz(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
pingResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ |
|||
TargetType: cluster.VolumeServerType, |
|||
Target: clusterHarness.VolumeServerAddress(), |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("Ping target volume server failed: %v", err) |
|||
} |
|||
if pingResp.GetRemoteTimeNs() == 0 { |
|||
t.Fatalf("expected remote timestamp from ping target volume server") |
|||
} |
|||
|
|||
if _, err = grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil { |
|||
t.Fatalf("VolumeServerLeave failed: %v", err) |
|||
} |
|||
|
|||
client := framework.NewHTTPClient() |
|||
healthURL := clusterHarness.VolumeAdminURL() + "/healthz" |
|||
deadline := time.Now().Add(5 * time.Second) |
|||
for { |
|||
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, healthURL)) |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
if resp.StatusCode == http.StatusServiceUnavailable { |
|||
return |
|||
} |
|||
if time.Now().After(deadline) { |
|||
t.Fatalf("expected healthz to return 503 after leave, got %d", resp.StatusCode) |
|||
} |
|||
time.Sleep(100 * time.Millisecond) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeServerLeaveIsIdempotent(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
if _, err := grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil { |
|||
t.Fatalf("first VolumeServerLeave failed: %v", err) |
|||
} |
|||
if _, err := grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil { |
|||
t.Fatalf("second VolumeServerLeave should be idempotent success, got: %v", err) |
|||
} |
|||
|
|||
client := framework.NewHTTPClient() |
|||
healthURL := clusterHarness.VolumeAdminURL() + "/healthz" |
|||
deadline := time.Now().Add(5 * time.Second) |
|||
for { |
|||
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, healthURL)) |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
if resp.StatusCode == http.StatusServiceUnavailable { |
|||
return |
|||
} |
|||
if time.Now().After(deadline) { |
|||
t.Fatalf("expected healthz to stay 503 after repeated leave, got %d", resp.StatusCode) |
|||
} |
|||
time.Sleep(100 * time.Millisecond) |
|||
} |
|||
} |
|||
|
|||
func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
unknownResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ |
|||
TargetType: "unknown-type", |
|||
Target: "127.0.0.1:12345", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("Ping unknown target type should not return grpc error, got: %v", err) |
|||
} |
|||
if unknownResp.GetRemoteTimeNs() != 0 { |
|||
t.Fatalf("Ping unknown target type expected remote_time_ns=0, got %d", unknownResp.GetRemoteTimeNs()) |
|||
} |
|||
if unknownResp.GetStopTimeNs() < unknownResp.GetStartTimeNs() { |
|||
t.Fatalf("Ping unknown target type expected stop_time_ns >= start_time_ns") |
|||
} |
|||
|
|||
_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ |
|||
TargetType: cluster.MasterType, |
|||
Target: "127.0.0.1:1", |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("Ping master target should fail when target is unreachable") |
|||
} |
|||
if !strings.Contains(err.Error(), "ping master") { |
|||
t.Fatalf("Ping master unreachable error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ |
|||
TargetType: cluster.FilerType, |
|||
Target: "127.0.0.1:1", |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("Ping filer target should fail when target is unreachable") |
|||
} |
|||
if !strings.Contains(err.Error(), "ping filer") { |
|||
t.Fatalf("Ping filer unreachable error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestPingMasterTargetSuccess(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ |
|||
TargetType: cluster.MasterType, |
|||
Target: clusterHarness.MasterAddress(), |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("Ping master target success path failed: %v", err) |
|||
} |
|||
if resp.GetRemoteTimeNs() == 0 { |
|||
t.Fatalf("Ping master target expected non-zero remote time") |
|||
} |
|||
if resp.GetStopTimeNs() < resp.GetStartTimeNs() { |
|||
t.Fatalf("Ping master target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs()) |
|||
} |
|||
} |
|||
|
|||
func TestPingFilerTargetSuccess(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeClusterWithFiler(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{ |
|||
TargetType: cluster.FilerType, |
|||
Target: clusterHarness.FilerServerAddress(), |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("Ping filer target success path failed: %v", err) |
|||
} |
|||
if resp.GetRemoteTimeNs() == 0 { |
|||
t.Fatalf("Ping filer target expected non-zero remote time") |
|||
} |
|||
if resp.GetStopTimeNs() < resp.GetStartTimeNs() { |
|||
t.Fatalf("Ping filer target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs()) |
|||
} |
|||
} |
|||
@ -0,0 +1,215 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"context" |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
"google.golang.org/grpc/codes" |
|||
"google.golang.org/grpc/status" |
|||
) |
|||
|
|||
func TestVolumeAdminLifecycleRPCs(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
const volumeID = uint32(11) |
|||
framework.AllocateVolume(t, client, volumeID, "") |
|||
|
|||
statusResp, err := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeStatus failed: %v", err) |
|||
} |
|||
if statusResp.GetFileCount() != 0 { |
|||
t.Fatalf("new volume should be empty, got file_count=%d", statusResp.GetFileCount()) |
|||
} |
|||
|
|||
if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil { |
|||
t.Fatalf("VolumeUnmount failed: %v", err) |
|||
} |
|||
if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil { |
|||
t.Fatalf("VolumeMount failed: %v", err) |
|||
} |
|||
|
|||
if _, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}); err != nil { |
|||
t.Fatalf("VolumeDelete failed: %v", err) |
|||
} |
|||
|
|||
_, err = client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) |
|||
if err == nil { |
|||
t.Fatalf("VolumeStatus should fail after delete") |
|||
} |
|||
if st, ok := status.FromError(err); !ok || st.Code() == codes.OK { |
|||
t.Fatalf("VolumeStatus error should be a non-OK grpc status, got: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeDeleteOnlyEmptyVariants(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(13) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 66001, 0x11223344) |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("volume-delete-only-empty")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}) |
|||
if err == nil || !strings.Contains(err.Error(), "volume not empty") { |
|||
t.Fatalf("VolumeDelete only_empty=true expected volume-not-empty error, got: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: false}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeDelete only_empty=false failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) |
|||
if err == nil { |
|||
t.Fatalf("VolumeStatus should fail after non-empty delete with only_empty=false") |
|||
} |
|||
} |
|||
|
|||
func TestMaintenanceModeRejectsAllocateVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("GetState failed: %v", err) |
|||
} |
|||
_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("SetState maintenance=true failed: %v", err) |
|||
} |
|||
|
|||
_, err = client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{VolumeId: 12, Replication: "000"}) |
|||
if err == nil { |
|||
t.Fatalf("AllocateVolume should fail when maintenance mode is enabled") |
|||
} |
|||
if !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("expected maintenance mode error, got: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestAllocateDuplicateAndMountUnmountMissingVariants(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
const missingVolumeID = uint32(99331) |
|||
const volumeID = uint32(14) |
|||
|
|||
if _, err := client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: missingVolumeID}); err != nil { |
|||
t.Fatalf("VolumeUnmount missing volume should be idempotent success, got: %v", err) |
|||
} |
|||
|
|||
_, err := client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: missingVolumeID}) |
|||
if err == nil { |
|||
t.Fatalf("VolumeMount missing volume should fail") |
|||
} |
|||
if !strings.Contains(err.Error(), "not found on disk") { |
|||
t.Fatalf("VolumeMount missing volume error mismatch: %v", err) |
|||
} |
|||
|
|||
framework.AllocateVolume(t, client, volumeID, "") |
|||
|
|||
_, err = client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{ |
|||
VolumeId: volumeID, |
|||
Replication: "000", |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("AllocateVolume duplicate should fail") |
|||
} |
|||
if !strings.Contains(strings.ToLower(err.Error()), "already exists") { |
|||
t.Fatalf("AllocateVolume duplicate error mismatch: %v", err) |
|||
} |
|||
|
|||
if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil { |
|||
t.Fatalf("VolumeUnmount existing volume failed: %v", err) |
|||
} |
|||
if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil { |
|||
t.Fatalf("VolumeUnmount already-unmounted volume should be idempotent success, got: %v", err) |
|||
} |
|||
if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil { |
|||
t.Fatalf("VolumeMount remount failed: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestMaintenanceModeRejectsVolumeDelete(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(15) |
|||
framework.AllocateVolume(t, client, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("GetState failed: %v", err) |
|||
} |
|||
_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("SetState maintenance=true failed: %v", err) |
|||
} |
|||
|
|||
_, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}) |
|||
if err == nil { |
|||
t.Fatalf("VolumeDelete should fail when maintenance mode is enabled") |
|||
} |
|||
if !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("expected maintenance mode error, got: %v", err) |
|||
} |
|||
} |
|||
@ -0,0 +1,177 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"context" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
) |
|||
|
|||
func TestVolumeMarkReadonlyAndWritableLifecycle(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(72) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{ |
|||
VolumeId: volumeID, |
|||
Persist: false, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeMarkReadonly failed: %v", err) |
|||
} |
|||
|
|||
readOnlyStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeStatus after readonly failed: %v", err) |
|||
} |
|||
if !readOnlyStatus.GetIsReadOnly() { |
|||
t.Fatalf("VolumeStatus expected readonly=true after VolumeMarkReadonly") |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeMarkWritable failed: %v", err) |
|||
} |
|||
|
|||
writableStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeStatus after writable failed: %v", err) |
|||
} |
|||
if writableStatus.GetIsReadOnly() { |
|||
t.Fatalf("VolumeStatus expected readonly=false after VolumeMarkWritable") |
|||
} |
|||
} |
|||
|
|||
func TestVolumeMarkReadonlyPersistTrue(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(74) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{ |
|||
VolumeId: volumeID, |
|||
Persist: true, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeMarkReadonly persist=true failed: %v", err) |
|||
} |
|||
|
|||
statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeStatus after persist readonly failed: %v", err) |
|||
} |
|||
if !statusResp.GetIsReadOnly() { |
|||
t.Fatalf("VolumeStatus expected readonly=true after persist readonly") |
|||
} |
|||
} |
|||
|
|||
func TestVolumeMarkReadonlyWritableErrorPaths(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 98771, Persist: true}) |
|||
if err == nil || !strings.Contains(err.Error(), "not found") { |
|||
t.Fatalf("VolumeMarkReadonly missing-volume error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 98772}) |
|||
if err == nil || !strings.Contains(err.Error(), "not found") { |
|||
t.Fatalf("VolumeMarkWritable missing-volume error mismatch: %v", err) |
|||
} |
|||
|
|||
stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("GetState failed: %v", err) |
|||
} |
|||
_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{ |
|||
Maintenance: true, |
|||
Version: stateResp.GetState().GetVersion(), |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("SetState maintenance=true failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 1, Persist: true}) |
|||
if err == nil || !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("VolumeMarkReadonly maintenance error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 1}) |
|||
if err == nil || !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("VolumeMarkWritable maintenance error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestDeleteCollectionRemovesVolumeAndIsIdempotent(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(73) |
|||
const collection = "it-delete-collection" |
|||
|
|||
framework.AllocateVolume(t, grpcClient, volumeID, collection) |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeStatus before DeleteCollection failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection}) |
|||
if err != nil { |
|||
t.Fatalf("DeleteCollection existing collection failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID}) |
|||
if err == nil { |
|||
t.Fatalf("VolumeStatus should fail after collection delete") |
|||
} |
|||
if !strings.Contains(err.Error(), "not found volume") { |
|||
t.Fatalf("VolumeStatus after DeleteCollection error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection}) |
|||
if err != nil { |
|||
t.Fatalf("DeleteCollection idempotent retry failed: %v", err) |
|||
} |
|||
} |
|||
@ -0,0 +1,264 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"bytes" |
|||
"context" |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
) |
|||
|
|||
func TestBatchDeleteInvalidFidAndMaintenanceMode(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
resp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"bad-fid"}}) |
|||
if err != nil { |
|||
t.Fatalf("BatchDelete invalid fid should return response, got error: %v", err) |
|||
} |
|||
if len(resp.GetResults()) != 1 { |
|||
t.Fatalf("expected one batch delete result, got %d", len(resp.GetResults())) |
|||
} |
|||
if got := resp.GetResults()[0].GetStatus(); got != 400 { |
|||
t.Fatalf("invalid fid expected status 400, got %d", got) |
|||
} |
|||
|
|||
stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("GetState failed: %v", err) |
|||
} |
|||
_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("SetState maintenance=true failed: %v", err) |
|||
} |
|||
|
|||
_, err = client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"1,1234567890ab"}}) |
|||
if err == nil { |
|||
t.Fatalf("BatchDelete should fail when maintenance mode is enabled") |
|||
} |
|||
if !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("expected maintenance mode error, got: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestBatchDeleteCookieMismatchAndSkipCheck(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(31) |
|||
const needleID = uint64(900001) |
|||
const correctCookie = uint32(0x1122AABB) |
|||
const wrongCookie = uint32(0x1122AABC) |
|||
framework.AllocateVolume(t, client, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, needleID, correctCookie) |
|||
uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, []byte("batch-delete-cookie-check")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
wrongCookieFid := framework.NewFileID(volumeID, needleID, wrongCookie) |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
mismatchResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{ |
|||
FileIds: []string{wrongCookieFid}, |
|||
SkipCookieCheck: false, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("BatchDelete with cookie check failed: %v", err) |
|||
} |
|||
if len(mismatchResp.GetResults()) != 1 { |
|||
t.Fatalf("BatchDelete cookie mismatch expected 1 result, got %d", len(mismatchResp.GetResults())) |
|||
} |
|||
if mismatchResp.GetResults()[0].GetStatus() != http.StatusBadRequest { |
|||
t.Fatalf("BatchDelete cookie mismatch expected status 400, got %d", mismatchResp.GetResults()[0].GetStatus()) |
|||
} |
|||
|
|||
skipCheckResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{ |
|||
FileIds: []string{wrongCookieFid}, |
|||
SkipCookieCheck: true, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("BatchDelete skip cookie check failed: %v", err) |
|||
} |
|||
if len(skipCheckResp.GetResults()) != 1 { |
|||
t.Fatalf("BatchDelete skip check expected 1 result, got %d", len(skipCheckResp.GetResults())) |
|||
} |
|||
if skipCheckResp.GetResults()[0].GetStatus() != http.StatusAccepted { |
|||
t.Fatalf("BatchDelete skip check expected status 202, got %d", skipCheckResp.GetResults()[0].GetStatus()) |
|||
} |
|||
|
|||
readAfterDelete := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid) |
|||
_ = framework.ReadAllAndClose(t, readAfterDelete) |
|||
if readAfterDelete.StatusCode != http.StatusNotFound { |
|||
t.Fatalf("read after skip-check batch delete expected 404, got %d", readAfterDelete.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestBatchDeleteMixedStatusesAndMismatchStopsProcessing(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(32) |
|||
framework.AllocateVolume(t, client, volumeID, "") |
|||
|
|||
const needleA = uint64(910001) |
|||
const needleB = uint64(910002) |
|||
const needleC = uint64(910003) |
|||
const cookieA = uint32(0x11111111) |
|||
const cookieB = uint32(0x22222222) |
|||
const cookieC = uint32(0x33333333) |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fidA := framework.NewFileID(volumeID, needleA, cookieA) |
|||
fidB := framework.NewFileID(volumeID, needleB, cookieB) |
|||
fidC := framework.NewFileID(volumeID, needleC, cookieC) |
|||
|
|||
for _, tc := range []struct { |
|||
fid string |
|||
body string |
|||
}{ |
|||
{fid: fidA, body: "batch-delete-mixed-a"}, |
|||
{fid: fidB, body: "batch-delete-mixed-b"}, |
|||
{fid: fidC, body: "batch-delete-mixed-c"}, |
|||
} { |
|||
uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, []byte(tc.body)) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload %s expected 201, got %d", tc.fid, uploadResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
missingFid := framework.NewFileID(volumeID, 919999, 0x44444444) |
|||
mixedResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{ |
|||
FileIds: []string{"bad-fid", fidA, missingFid}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("BatchDelete mixed status request failed: %v", err) |
|||
} |
|||
if len(mixedResp.GetResults()) != 3 { |
|||
t.Fatalf("BatchDelete mixed status expected 3 results, got %d", len(mixedResp.GetResults())) |
|||
} |
|||
if mixedResp.GetResults()[0].GetStatus() != http.StatusBadRequest { |
|||
t.Fatalf("BatchDelete mixed result[0] expected 400, got %d", mixedResp.GetResults()[0].GetStatus()) |
|||
} |
|||
if mixedResp.GetResults()[1].GetStatus() != http.StatusAccepted { |
|||
t.Fatalf("BatchDelete mixed result[1] expected 202, got %d", mixedResp.GetResults()[1].GetStatus()) |
|||
} |
|||
if mixedResp.GetResults()[2].GetStatus() != http.StatusNotFound { |
|||
t.Fatalf("BatchDelete mixed result[2] expected 404, got %d", mixedResp.GetResults()[2].GetStatus()) |
|||
} |
|||
|
|||
readDeletedA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA) |
|||
_ = framework.ReadAllAndClose(t, readDeletedA) |
|||
if readDeletedA.StatusCode != http.StatusNotFound { |
|||
t.Fatalf("fidA should be deleted after batch delete, got status %d", readDeletedA.StatusCode) |
|||
} |
|||
|
|||
wrongCookieB := framework.NewFileID(volumeID, needleB, cookieB+1) |
|||
stopResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{ |
|||
FileIds: []string{wrongCookieB, fidC}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("BatchDelete mismatch-stop request failed: %v", err) |
|||
} |
|||
if len(stopResp.GetResults()) != 1 { |
|||
t.Fatalf("BatchDelete mismatch-stop expected 1 result due early break, got %d", len(stopResp.GetResults())) |
|||
} |
|||
if stopResp.GetResults()[0].GetStatus() != http.StatusBadRequest { |
|||
t.Fatalf("BatchDelete mismatch-stop expected 400, got %d", stopResp.GetResults()[0].GetStatus()) |
|||
} |
|||
|
|||
readB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB) |
|||
_ = framework.ReadAllAndClose(t, readB) |
|||
if readB.StatusCode != http.StatusOK { |
|||
t.Fatalf("fidB should remain after cookie mismatch path, got %d", readB.StatusCode) |
|||
} |
|||
|
|||
readC := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidC) |
|||
_ = framework.ReadAllAndClose(t, readC) |
|||
if readC.StatusCode != http.StatusOK { |
|||
t.Fatalf("fidC should remain when batch processing stops on mismatch, got %d", readC.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestBatchDeleteRejectsChunkManifestNeedles(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(33) |
|||
framework.AllocateVolume(t, client, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 920001, 0x5555AAAA) |
|||
req, err := http.NewRequest(http.MethodPost, cluster.VolumeAdminURL()+"/"+fid+"?cm=true", bytes.NewReader([]byte("manifest-placeholder-payload"))) |
|||
if err != nil { |
|||
t.Fatalf("create chunk manifest upload request: %v", err) |
|||
} |
|||
req.Header.Set("Content-Type", "application/octet-stream") |
|||
uploadResp := framework.DoRequest(t, httpClient, req) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("chunk manifest upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
resp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{fid}}) |
|||
if err != nil { |
|||
t.Fatalf("BatchDelete chunk manifest should return response, got grpc error: %v", err) |
|||
} |
|||
if len(resp.GetResults()) != 1 { |
|||
t.Fatalf("BatchDelete chunk manifest expected one result, got %d", len(resp.GetResults())) |
|||
} |
|||
if resp.GetResults()[0].GetStatus() != http.StatusNotAcceptable { |
|||
t.Fatalf("BatchDelete chunk manifest expected status 406, got %d", resp.GetResults()[0].GetStatus()) |
|||
} |
|||
if !strings.Contains(resp.GetResults()[0].GetError(), "ChunkManifest") { |
|||
t.Fatalf("BatchDelete chunk manifest expected error mentioning ChunkManifest, got %q", resp.GetResults()[0].GetError()) |
|||
} |
|||
|
|||
readResp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid) |
|||
_ = framework.ReadAllAndClose(t, readResp) |
|||
if readResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("chunk manifest should not be deleted by BatchDelete reject path, got %d", readResp.StatusCode) |
|||
} |
|||
} |
|||
@ -0,0 +1,431 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"context" |
|||
"io" |
|||
"math" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
) |
|||
|
|||
func TestVolumeIncrementalCopyDataAndNoDataPaths(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(91) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 770001, 0x1122AABB) |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("incremental-copy-content")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != 201 { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
dataStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{ |
|||
VolumeId: volumeID, |
|||
SinceNs: 0, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeIncrementalCopy start failed: %v", err) |
|||
} |
|||
|
|||
totalBytes := 0 |
|||
for { |
|||
msg, recvErr := dataStream.Recv() |
|||
if recvErr == io.EOF { |
|||
break |
|||
} |
|||
if recvErr != nil { |
|||
t.Fatalf("VolumeIncrementalCopy recv failed: %v", recvErr) |
|||
} |
|||
totalBytes += len(msg.GetFileContent()) |
|||
} |
|||
if totalBytes == 0 { |
|||
t.Fatalf("VolumeIncrementalCopy expected streamed bytes for since_ns=0") |
|||
} |
|||
|
|||
noDataStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{ |
|||
VolumeId: volumeID, |
|||
SinceNs: math.MaxUint64, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeIncrementalCopy no-data start failed: %v", err) |
|||
} |
|||
_, err = noDataStream.Recv() |
|||
if err != io.EOF { |
|||
t.Fatalf("VolumeIncrementalCopy no-data expected EOF, got: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestCopyFileIgnoreNotFoundAndStopOffsetZeroPaths(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(92) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
missingNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: volumeID, |
|||
Ext: ".definitely-missing", |
|||
CompactionRevision: math.MaxUint32, |
|||
StopOffset: 1, |
|||
IgnoreSourceFileNotFound: false, |
|||
}) |
|||
if err == nil { |
|||
_, err = missingNoIgnore.Recv() |
|||
} |
|||
if err == nil { |
|||
t.Fatalf("CopyFile should fail for missing source file when ignore_source_file_not_found=false") |
|||
} |
|||
|
|||
missingIgnored, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: volumeID, |
|||
Ext: ".definitely-missing", |
|||
CompactionRevision: math.MaxUint32, |
|||
StopOffset: 1, |
|||
IgnoreSourceFileNotFound: true, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("CopyFile ignore-not-found start failed: %v", err) |
|||
} |
|||
_, err = missingIgnored.Recv() |
|||
if err != io.EOF { |
|||
t.Fatalf("CopyFile ignore-not-found expected EOF, got: %v", err) |
|||
} |
|||
|
|||
stopZeroStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: volumeID, |
|||
Ext: ".definitely-missing", |
|||
CompactionRevision: math.MaxUint32, |
|||
StopOffset: 0, |
|||
IgnoreSourceFileNotFound: false, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("CopyFile stop_offset=0 start failed: %v", err) |
|||
} |
|||
_, err = stopZeroStream.Recv() |
|||
if err != io.EOF { |
|||
t.Fatalf("CopyFile stop_offset=0 expected EOF, got: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestCopyFileCompactionRevisionMismatch(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(94) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: volumeID, |
|||
Ext: ".idx", |
|||
CompactionRevision: 1, // fresh volume starts at revision 0
|
|||
StopOffset: 1, |
|||
}) |
|||
if err == nil { |
|||
_, err = stream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "is compacted") { |
|||
t.Fatalf("CopyFile compaction mismatch error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestReceiveFileProtocolViolationResponses(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
contentFirstStream, err := grpcClient.ReceiveFile(ctx) |
|||
if err != nil { |
|||
t.Fatalf("ReceiveFile stream create failed: %v", err) |
|||
} |
|||
if err = contentFirstStream.Send(&volume_server_pb.ReceiveFileRequest{ |
|||
Data: &volume_server_pb.ReceiveFileRequest_FileContent{ |
|||
FileContent: []byte("content-before-info"), |
|||
}, |
|||
}); err != nil { |
|||
t.Fatalf("ReceiveFile send content-first failed: %v", err) |
|||
} |
|||
contentFirstResp, err := contentFirstStream.CloseAndRecv() |
|||
if err != nil { |
|||
t.Fatalf("ReceiveFile content-first close failed: %v", err) |
|||
} |
|||
if !strings.Contains(contentFirstResp.GetError(), "file info must be sent first") { |
|||
t.Fatalf("ReceiveFile content-first response mismatch: %+v", contentFirstResp) |
|||
} |
|||
|
|||
unknownTypeStream, err := grpcClient.ReceiveFile(ctx) |
|||
if err != nil { |
|||
t.Fatalf("ReceiveFile stream create for unknown-type failed: %v", err) |
|||
} |
|||
if err = unknownTypeStream.Send(&volume_server_pb.ReceiveFileRequest{}); err != nil { |
|||
t.Fatalf("ReceiveFile send unknown-type request failed: %v", err) |
|||
} |
|||
unknownTypeResp, err := unknownTypeStream.CloseAndRecv() |
|||
if err != nil { |
|||
t.Fatalf("ReceiveFile unknown-type close failed: %v", err) |
|||
} |
|||
if !strings.Contains(unknownTypeResp.GetError(), "unknown message type") { |
|||
t.Fatalf("ReceiveFile unknown-type response mismatch: %+v", unknownTypeResp) |
|||
} |
|||
} |
|||
|
|||
func TestReceiveFileSuccessForRegularVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(95) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
payloadA := []byte("receive-file-chunk-a:") |
|||
payloadB := []byte("receive-file-chunk-b") |
|||
expected := append(append([]byte{}, payloadA...), payloadB...) |
|||
|
|||
receiveStream, err := grpcClient.ReceiveFile(ctx) |
|||
if err != nil { |
|||
t.Fatalf("ReceiveFile stream create failed: %v", err) |
|||
} |
|||
|
|||
if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ |
|||
Data: &volume_server_pb.ReceiveFileRequest_Info{ |
|||
Info: &volume_server_pb.ReceiveFileInfo{ |
|||
VolumeId: volumeID, |
|||
Ext: ".tmprecv", |
|||
Collection: "", |
|||
IsEcVolume: false, |
|||
FileSize: uint64(len(expected)), |
|||
}, |
|||
}, |
|||
}); err != nil { |
|||
t.Fatalf("ReceiveFile send info failed: %v", err) |
|||
} |
|||
if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ |
|||
Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadA}, |
|||
}); err != nil { |
|||
t.Fatalf("ReceiveFile send payloadA failed: %v", err) |
|||
} |
|||
if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ |
|||
Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadB}, |
|||
}); err != nil { |
|||
t.Fatalf("ReceiveFile send payloadB failed: %v", err) |
|||
} |
|||
|
|||
resp, err := receiveStream.CloseAndRecv() |
|||
if err != nil { |
|||
t.Fatalf("ReceiveFile close failed: %v", err) |
|||
} |
|||
if resp.GetError() != "" { |
|||
t.Fatalf("ReceiveFile unexpected error response: %+v", resp) |
|||
} |
|||
if resp.GetBytesWritten() != uint64(len(expected)) { |
|||
t.Fatalf("ReceiveFile bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(expected)) |
|||
} |
|||
|
|||
copyStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: volumeID, |
|||
Ext: ".tmprecv", |
|||
CompactionRevision: math.MaxUint32, |
|||
StopOffset: uint64(len(expected)), |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("CopyFile for received data start failed: %v", err) |
|||
} |
|||
|
|||
var copied []byte |
|||
for { |
|||
msg, recvErr := copyStream.Recv() |
|||
if recvErr == io.EOF { |
|||
break |
|||
} |
|||
if recvErr != nil { |
|||
t.Fatalf("CopyFile for received data recv failed: %v", recvErr) |
|||
} |
|||
copied = append(copied, msg.GetFileContent()...) |
|||
} |
|||
|
|||
if string(copied) != string(expected) { |
|||
t.Fatalf("received file data mismatch: got %q want %q", string(copied), string(expected)) |
|||
} |
|||
} |
|||
|
|||
func TestReceiveFileSuccessForEcVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
const volumeID = uint32(96) |
|||
const collection = "ec-receive-success" |
|||
const ext = ".ec00" |
|||
|
|||
payloadA := []byte("receive-ec-file-chunk-a:") |
|||
payloadB := []byte("receive-ec-file-chunk-b") |
|||
expected := append(append([]byte{}, payloadA...), payloadB...) |
|||
|
|||
receiveStream, err := grpcClient.ReceiveFile(ctx) |
|||
if err != nil { |
|||
t.Fatalf("ReceiveFile stream create failed: %v", err) |
|||
} |
|||
|
|||
if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ |
|||
Data: &volume_server_pb.ReceiveFileRequest_Info{ |
|||
Info: &volume_server_pb.ReceiveFileInfo{ |
|||
VolumeId: volumeID, |
|||
Ext: ext, |
|||
Collection: collection, |
|||
IsEcVolume: true, |
|||
ShardId: 0, |
|||
FileSize: uint64(len(expected)), |
|||
}, |
|||
}, |
|||
}); err != nil { |
|||
t.Fatalf("ReceiveFile send EC info failed: %v", err) |
|||
} |
|||
if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ |
|||
Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadA}, |
|||
}); err != nil { |
|||
t.Fatalf("ReceiveFile send EC payloadA failed: %v", err) |
|||
} |
|||
if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{ |
|||
Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadB}, |
|||
}); err != nil { |
|||
t.Fatalf("ReceiveFile send EC payloadB failed: %v", err) |
|||
} |
|||
|
|||
resp, err := receiveStream.CloseAndRecv() |
|||
if err != nil { |
|||
t.Fatalf("ReceiveFile EC close failed: %v", err) |
|||
} |
|||
if resp.GetError() != "" { |
|||
t.Fatalf("ReceiveFile EC unexpected error response: %+v", resp) |
|||
} |
|||
if resp.GetBytesWritten() != uint64(len(expected)) { |
|||
t.Fatalf("ReceiveFile EC bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(expected)) |
|||
} |
|||
|
|||
copyStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: collection, |
|||
IsEcVolume: true, |
|||
Ext: ext, |
|||
CompactionRevision: math.MaxUint32, |
|||
StopOffset: uint64(len(expected)), |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("CopyFile for received EC data start failed: %v", err) |
|||
} |
|||
|
|||
var copied []byte |
|||
for { |
|||
msg, recvErr := copyStream.Recv() |
|||
if recvErr == io.EOF { |
|||
break |
|||
} |
|||
if recvErr != nil { |
|||
t.Fatalf("CopyFile for received EC data recv failed: %v", recvErr) |
|||
} |
|||
copied = append(copied, msg.GetFileContent()...) |
|||
} |
|||
|
|||
if string(copied) != string(expected) { |
|||
t.Fatalf("received EC file data mismatch: got %q want %q", string(copied), string(expected)) |
|||
} |
|||
} |
|||
|
|||
func TestCopyFileEcVolumeIgnoreMissingSourcePaths(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
streamNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: 99601, |
|||
Collection: "ec-copy-missing", |
|||
IsEcVolume: true, |
|||
Ext: ".ec00", |
|||
CompactionRevision: math.MaxUint32, |
|||
StopOffset: 1, |
|||
IgnoreSourceFileNotFound: false, |
|||
}) |
|||
if err == nil { |
|||
_, err = streamNoIgnore.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "not found ec volume id") { |
|||
t.Fatalf("CopyFile EC missing source error mismatch: %v", err) |
|||
} |
|||
|
|||
streamIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: 99602, |
|||
Collection: "ec-copy-missing", |
|||
IsEcVolume: true, |
|||
Ext: ".ec00", |
|||
CompactionRevision: math.MaxUint32, |
|||
StopOffset: 1, |
|||
IgnoreSourceFileNotFound: true, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("CopyFile EC ignore-missing start failed: %v", err) |
|||
} |
|||
_, err = streamIgnore.Recv() |
|||
if err != io.EOF { |
|||
t.Fatalf("CopyFile EC ignore-missing expected EOF, got: %v", err) |
|||
} |
|||
} |
|||
@ -0,0 +1,284 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"context" |
|||
"io" |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
) |
|||
|
|||
func TestVolumeSyncStatusAndReadVolumeFileStatus(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(41) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
syncResp, err := grpcClient.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{VolumeId: volumeID}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeSyncStatus failed: %v", err) |
|||
} |
|||
if syncResp.GetVolumeId() != volumeID { |
|||
t.Fatalf("VolumeSyncStatus volume id mismatch: got %d want %d", syncResp.GetVolumeId(), volumeID) |
|||
} |
|||
|
|||
statusResp, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID}) |
|||
if err != nil { |
|||
t.Fatalf("ReadVolumeFileStatus failed: %v", err) |
|||
} |
|||
if statusResp.GetVolumeId() != volumeID { |
|||
t.Fatalf("ReadVolumeFileStatus volume id mismatch: got %d want %d", statusResp.GetVolumeId(), volumeID) |
|||
} |
|||
if statusResp.GetVersion() == 0 { |
|||
t.Fatalf("ReadVolumeFileStatus expected non-zero version") |
|||
} |
|||
} |
|||
|
|||
func TestCopyAndStreamMethodsMissingVolumePaths(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{VolumeId: 98761}) |
|||
if err == nil { |
|||
t.Fatalf("VolumeSyncStatus should fail for missing volume") |
|||
} |
|||
|
|||
incrementalStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{VolumeId: 98762, SinceNs: 0}) |
|||
if err == nil { |
|||
_, err = incrementalStream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "not found volume") { |
|||
t.Fatalf("VolumeIncrementalCopy missing-volume error mismatch: %v", err) |
|||
} |
|||
|
|||
readAllStream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{98763}}) |
|||
if err == nil { |
|||
_, err = readAllStream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "not found volume") { |
|||
t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", err) |
|||
} |
|||
|
|||
copyFileStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{VolumeId: 98764, Ext: ".dat", StopOffset: 1}) |
|||
if err == nil { |
|||
_, err = copyFileStream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "not found volume") { |
|||
t.Fatalf("CopyFile missing-volume error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: 98765}) |
|||
if err == nil || !strings.Contains(err.Error(), "not found volume") { |
|||
t.Fatalf("ReadVolumeFileStatus missing-volume error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeCopyAndReceiveFileMaintenanceRejection(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("GetState failed: %v", err) |
|||
} |
|||
_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("SetState maintenance=true failed: %v", err) |
|||
} |
|||
|
|||
copyStream, err := grpcClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{VolumeId: 1, SourceDataNode: "127.0.0.1:1234"}) |
|||
if err == nil { |
|||
_, err = copyStream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("VolumeCopy maintenance error mismatch: %v", err) |
|||
} |
|||
|
|||
receiveClient, err := grpcClient.ReceiveFile(ctx) |
|||
if err != nil { |
|||
t.Fatalf("ReceiveFile client creation failed: %v", err) |
|||
} |
|||
_ = receiveClient.Send(&volume_server_pb.ReceiveFileRequest{ |
|||
Data: &volume_server_pb.ReceiveFileRequest_Info{ |
|||
Info: &volume_server_pb.ReceiveFileInfo{VolumeId: 1, Ext: ".dat"}, |
|||
}, |
|||
}) |
|||
_, err = receiveClient.CloseAndRecv() |
|||
if err == nil || !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("ReceiveFile maintenance error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeCopySuccessFromPeerAndMountsDestination(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1()) |
|||
sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) |
|||
defer sourceConn.Close() |
|||
destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) |
|||
defer destConn.Close() |
|||
|
|||
const volumeID = uint32(42) |
|||
framework.AllocateVolume(t, sourceClient, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 880001, 0x12345678) |
|||
payload := []byte("volume-copy-success-payload") |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload to source expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
|||
defer cancel() |
|||
|
|||
copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1], |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeCopy start failed: %v", err) |
|||
} |
|||
|
|||
sawFinalAppendTimestamp := false |
|||
for { |
|||
msg, recvErr := copyStream.Recv() |
|||
if recvErr == io.EOF { |
|||
break |
|||
} |
|||
if recvErr != nil { |
|||
t.Fatalf("VolumeCopy recv failed: %v", recvErr) |
|||
} |
|||
if msg.GetLastAppendAtNs() > 0 { |
|||
sawFinalAppendTimestamp = true |
|||
} |
|||
} |
|||
if !sawFinalAppendTimestamp { |
|||
t.Fatalf("VolumeCopy expected final response with last_append_at_ns") |
|||
} |
|||
|
|||
destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid) |
|||
destReadBody := framework.ReadAllAndClose(t, destReadResp) |
|||
if destReadResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("read from copied destination expected 200, got %d", destReadResp.StatusCode) |
|||
} |
|||
if string(destReadBody) != string(payload) { |
|||
t.Fatalf("destination copied payload mismatch: got %q want %q", string(destReadBody), string(payload)) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeCopyOverwritesExistingDestinationVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1()) |
|||
sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) |
|||
defer sourceConn.Close() |
|||
destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) |
|||
defer destConn.Close() |
|||
|
|||
const volumeID = uint32(43) |
|||
framework.AllocateVolume(t, sourceClient, volumeID, "") |
|||
framework.AllocateVolume(t, destClient, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 880002, 0x23456789) |
|||
sourcePayload := []byte("volume-copy-overwrite-source") |
|||
destPayload := []byte("volume-copy-overwrite-destination-old") |
|||
|
|||
sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, sourcePayload) |
|||
_ = framework.ReadAllAndClose(t, sourceUploadResp) |
|||
if sourceUploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload to source expected 201, got %d", sourceUploadResp.StatusCode) |
|||
} |
|||
|
|||
destUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid, destPayload) |
|||
_ = framework.ReadAllAndClose(t, destUploadResp) |
|||
if destUploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload to destination expected 201, got %d", destUploadResp.StatusCode) |
|||
} |
|||
|
|||
destReadBeforeResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid) |
|||
destReadBeforeBody := framework.ReadAllAndClose(t, destReadBeforeResp) |
|||
if destReadBeforeResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("destination pre-copy read expected 200, got %d", destReadBeforeResp.StatusCode) |
|||
} |
|||
if string(destReadBeforeBody) != string(destPayload) { |
|||
t.Fatalf("destination pre-copy payload mismatch: got %q want %q", string(destReadBeforeBody), string(destPayload)) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
|||
defer cancel() |
|||
|
|||
copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1], |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeCopy overwrite start failed: %v", err) |
|||
} |
|||
|
|||
sawFinalAppendTimestamp := false |
|||
for { |
|||
msg, recvErr := copyStream.Recv() |
|||
if recvErr == io.EOF { |
|||
break |
|||
} |
|||
if recvErr != nil { |
|||
t.Fatalf("VolumeCopy overwrite recv failed: %v", recvErr) |
|||
} |
|||
if msg.GetLastAppendAtNs() > 0 { |
|||
sawFinalAppendTimestamp = true |
|||
} |
|||
} |
|||
if !sawFinalAppendTimestamp { |
|||
t.Fatalf("VolumeCopy overwrite expected final response with last_append_at_ns") |
|||
} |
|||
|
|||
destReadAfterResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid) |
|||
destReadAfterBody := framework.ReadAllAndClose(t, destReadAfterResp) |
|||
if destReadAfterResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("destination post-copy read expected 200, got %d", destReadAfterResp.StatusCode) |
|||
} |
|||
if string(destReadAfterBody) != string(sourcePayload) { |
|||
t.Fatalf("destination post-copy payload mismatch: got %q want %q", string(destReadAfterBody), string(sourcePayload)) |
|||
} |
|||
} |
|||
@ -0,0 +1,146 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"context" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
) |
|||
|
|||
func TestReadNeedleBlobAndMetaMissingVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{ |
|||
VolumeId: 99111, |
|||
Offset: 0, |
|||
Size: 16, |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("ReadNeedleBlob should fail for missing volume") |
|||
} |
|||
if !strings.Contains(err.Error(), "not found volume") { |
|||
t.Fatalf("ReadNeedleBlob missing volume error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{ |
|||
VolumeId: 99112, |
|||
NeedleId: 1, |
|||
Offset: 0, |
|||
Size: 16, |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("ReadNeedleMeta should fail for missing volume") |
|||
} |
|||
if !strings.Contains(err.Error(), "not found volume") { |
|||
t.Fatalf("ReadNeedleMeta missing volume error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestWriteNeedleBlobMaintenanceAndMissingVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{ |
|||
VolumeId: 99113, |
|||
NeedleId: 1, |
|||
NeedleBlob: []byte("abc"), |
|||
Size: 3, |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("WriteNeedleBlob should fail for missing volume") |
|||
} |
|||
if !strings.Contains(err.Error(), "not found volume") { |
|||
t.Fatalf("WriteNeedleBlob missing volume error mismatch: %v", err) |
|||
} |
|||
|
|||
stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("GetState failed: %v", err) |
|||
} |
|||
_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("SetState maintenance=true failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{ |
|||
VolumeId: 1, |
|||
NeedleId: 2, |
|||
NeedleBlob: []byte("def"), |
|||
Size: 3, |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("WriteNeedleBlob should fail in maintenance mode") |
|||
} |
|||
if !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("WriteNeedleBlob maintenance mode error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestReadNeedleBlobAndMetaInvalidOffsets(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(92) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 880001, 0xCCDD1122) |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("invalid-offset-check")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != 201 { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{ |
|||
VolumeId: volumeID, |
|||
Offset: 1 << 40, |
|||
Size: 64, |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("ReadNeedleBlob should fail for invalid offset") |
|||
} |
|||
if !strings.Contains(strings.ToLower(err.Error()), "read needle blob") { |
|||
t.Fatalf("ReadNeedleBlob invalid offset error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{ |
|||
VolumeId: volumeID, |
|||
NeedleId: 880001, |
|||
Offset: 1 << 40, |
|||
Size: 64, |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("ReadNeedleMeta should fail for invalid offset") |
|||
} |
|||
} |
|||
@ -0,0 +1,273 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"context" |
|||
"io" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/idx" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/types" |
|||
) |
|||
|
|||
func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(83) |
|||
const sourceNeedleID = uint64(333333) |
|||
const sourceCookie = uint32(0xABCD0102) |
|||
const clonedNeedleID = uint64(333334) |
|||
|
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
payload := []byte("blob-roundtrip-content") |
|||
fid := framework.NewFileID(volumeID, sourceNeedleID, sourceCookie) |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != 201 { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
fileStatus, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID}) |
|||
if err != nil { |
|||
t.Fatalf("ReadVolumeFileStatus failed: %v", err) |
|||
} |
|||
if fileStatus.GetIdxFileSize() == 0 { |
|||
t.Fatalf("expected non-zero idx file size after upload") |
|||
} |
|||
|
|||
idxBytes := copyFileBytes(t, grpcClient, &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: volumeID, |
|||
Ext: ".idx", |
|||
CompactionRevision: fileStatus.GetCompactionRevision(), |
|||
StopOffset: fileStatus.GetIdxFileSize(), |
|||
}) |
|||
offset, size := findNeedleOffsetAndSize(t, idxBytes, sourceNeedleID) |
|||
|
|||
blobResp, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{ |
|||
VolumeId: volumeID, |
|||
Offset: offset, |
|||
Size: size, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("ReadNeedleBlob failed: %v", err) |
|||
} |
|||
if len(blobResp.GetNeedleBlob()) == 0 { |
|||
t.Fatalf("ReadNeedleBlob returned empty blob") |
|||
} |
|||
|
|||
metaResp, err := grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{ |
|||
VolumeId: volumeID, |
|||
NeedleId: sourceNeedleID, |
|||
Offset: offset, |
|||
Size: size, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("ReadNeedleMeta failed: %v", err) |
|||
} |
|||
if metaResp.GetCookie() != sourceCookie { |
|||
t.Fatalf("ReadNeedleMeta cookie mismatch: got %d want %d", metaResp.GetCookie(), sourceCookie) |
|||
} |
|||
|
|||
_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{ |
|||
VolumeId: volumeID, |
|||
NeedleId: clonedNeedleID, |
|||
Size: size, |
|||
NeedleBlob: blobResp.GetNeedleBlob(), |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("WriteNeedleBlob failed: %v", err) |
|||
} |
|||
|
|||
clonedStatus, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{ |
|||
VolumeId: volumeID, |
|||
NeedleId: clonedNeedleID, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeNeedleStatus for cloned needle failed: %v", err) |
|||
} |
|||
if clonedStatus.GetNeedleId() != sourceNeedleID { |
|||
t.Fatalf("cloned needle status id mismatch: got %d want %d", clonedStatus.GetNeedleId(), sourceNeedleID) |
|||
} |
|||
if clonedStatus.GetCookie() != sourceCookie { |
|||
t.Fatalf("cloned needle cookie mismatch: got %d want %d", clonedStatus.GetCookie(), sourceCookie) |
|||
} |
|||
|
|||
clonedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, clonedNeedleID, sourceCookie)) |
|||
clonedReadBody := framework.ReadAllAndClose(t, clonedReadResp) |
|||
if clonedReadResp.StatusCode != 200 { |
|||
t.Fatalf("cloned needle GET expected 200, got %d", clonedReadResp.StatusCode) |
|||
} |
|||
if string(clonedReadBody) != string(payload) { |
|||
t.Fatalf("cloned needle body mismatch: got %q want %q", string(clonedReadBody), string(payload)) |
|||
} |
|||
} |
|||
|
|||
func TestReadAllNeedlesStreamsUploadedRecords(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(84) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
expected := map[uint64]string{ |
|||
444441: "read-all-needle-one", |
|||
444442: "read-all-needle-two", |
|||
} |
|||
for key, body := range expected { |
|||
resp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, key, 0xA0B0C0D0), []byte(body)) |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
if resp.StatusCode != 201 { |
|||
t.Fatalf("upload for key %d expected 201, got %d", key, resp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{volumeID}}) |
|||
if err != nil { |
|||
t.Fatalf("ReadAllNeedles start failed: %v", err) |
|||
} |
|||
|
|||
seen := map[uint64]string{} |
|||
for { |
|||
msg, recvErr := stream.Recv() |
|||
if recvErr == io.EOF { |
|||
break |
|||
} |
|||
if recvErr != nil { |
|||
t.Fatalf("ReadAllNeedles recv failed: %v", recvErr) |
|||
} |
|||
if _, wanted := expected[msg.GetNeedleId()]; wanted { |
|||
seen[msg.GetNeedleId()] = string(msg.GetNeedleBlob()) |
|||
} |
|||
} |
|||
|
|||
for key, body := range expected { |
|||
got, found := seen[key] |
|||
if !found { |
|||
t.Fatalf("ReadAllNeedles missing key %d in stream", key) |
|||
} |
|||
if got != body { |
|||
t.Fatalf("ReadAllNeedles body mismatch for key %d: got %q want %q", key, got, body) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func TestReadAllNeedlesExistingThenMissingVolumeAbortsStream(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const existingVolumeID = uint32(85) |
|||
const missingVolumeID = uint32(98585) |
|||
const needleID = uint64(445551) |
|||
framework.AllocateVolume(t, grpcClient, existingVolumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(existingVolumeID, needleID, 0xAA11BB22) |
|||
payload := "read-all-existing-then-missing" |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte(payload)) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != 201 { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{ |
|||
VolumeIds: []uint32{existingVolumeID, missingVolumeID}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("ReadAllNeedles start failed: %v", err) |
|||
} |
|||
|
|||
seenUploadedNeedle := false |
|||
for { |
|||
msg, recvErr := stream.Recv() |
|||
if recvErr == io.EOF { |
|||
t.Fatalf("ReadAllNeedles expected stream error for missing volume, got EOF") |
|||
} |
|||
if recvErr != nil { |
|||
if !strings.Contains(recvErr.Error(), "not found volume id") { |
|||
t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", recvErr) |
|||
} |
|||
break |
|||
} |
|||
if msg.GetNeedleId() == needleID && string(msg.GetNeedleBlob()) == payload { |
|||
seenUploadedNeedle = true |
|||
} |
|||
} |
|||
|
|||
if !seenUploadedNeedle { |
|||
t.Fatalf("ReadAllNeedles should stream entries from existing volume before missing-volume abort") |
|||
} |
|||
} |
|||
|
|||
func copyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte { |
|||
t.Helper() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stream, err := grpcClient.CopyFile(ctx, req) |
|||
if err != nil { |
|||
t.Fatalf("CopyFile start failed: %v", err) |
|||
} |
|||
|
|||
var out []byte |
|||
for { |
|||
msg, recvErr := stream.Recv() |
|||
if recvErr == io.EOF { |
|||
return out |
|||
} |
|||
if recvErr != nil { |
|||
t.Fatalf("CopyFile recv failed: %v", recvErr) |
|||
} |
|||
out = append(out, msg.GetFileContent()...) |
|||
} |
|||
} |
|||
|
|||
func findNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) { |
|||
t.Helper() |
|||
|
|||
for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize { |
|||
key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize]) |
|||
if uint64(key) != needleID { |
|||
continue |
|||
} |
|||
if entryOffset.IsZero() || entrySize <= 0 { |
|||
continue |
|||
} |
|||
return entryOffset.ToActualOffset(), int32(entrySize) |
|||
} |
|||
|
|||
t.Fatalf("needle id %d not found in idx entries", needleID) |
|||
return 0, 0 |
|||
} |
|||
@ -0,0 +1,777 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"context" |
|||
"io" |
|||
"math" |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/needle" |
|||
"google.golang.org/grpc/codes" |
|||
"google.golang.org/grpc/status" |
|||
) |
|||
|
|||
func TestEcMaintenanceModeRejections(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("GetState failed: %v", err) |
|||
} |
|||
_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{ |
|||
Maintenance: true, |
|||
Version: stateResp.GetState().GetVersion(), |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("SetState maintenance=true failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: 1, Collection: ""}) |
|||
if err == nil || !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("VolumeEcShardsGenerate maintenance error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ |
|||
VolumeId: 1, |
|||
Collection: "", |
|||
SourceDataNode: "127.0.0.1:1", |
|||
ShardIds: []uint32{0}, |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("VolumeEcShardsCopy maintenance error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ |
|||
VolumeId: 1, |
|||
Collection: "", |
|||
ShardIds: []uint32{0}, |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("VolumeEcShardsDelete maintenance error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{ |
|||
VolumeId: 1, |
|||
Collection: "", |
|||
FileKey: 1, |
|||
Version: 3, |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("VolumeEcBlobDelete maintenance error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ |
|||
VolumeId: 1, |
|||
Collection: "", |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("VolumeEcShardsToVolume maintenance error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestEcMissingInvalidAndNoopPaths(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ |
|||
VolumeId: 98791, |
|||
Collection: "", |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "not found") { |
|||
t.Fatalf("VolumeEcShardsGenerate missing-volume error mismatch: %v", err) |
|||
} |
|||
|
|||
rebuildResp, err := grpcClient.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{ |
|||
VolumeId: 98792, |
|||
Collection: "ec-rebuild", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsRebuild missing-volume should return empty success, got: %v", err) |
|||
} |
|||
if len(rebuildResp.GetRebuiltShardIds()) != 0 { |
|||
t.Fatalf("VolumeEcShardsRebuild expected no rebuilt shards for missing volume, got %v", rebuildResp.GetRebuiltShardIds()) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ |
|||
VolumeId: 98793, |
|||
Collection: "ec-copy", |
|||
SourceDataNode: "127.0.0.1:1", |
|||
ShardIds: []uint32{0}, |
|||
DiskId: 99, |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "invalid disk_id") { |
|||
t.Fatalf("VolumeEcShardsCopy invalid-disk error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ |
|||
VolumeId: 98794, |
|||
Collection: "ec-delete", |
|||
ShardIds: []uint32{0, 1}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsDelete missing-volume should be no-op success, got: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ |
|||
VolumeId: 98795, |
|||
Collection: "ec-mount", |
|||
ShardIds: []uint32{0}, |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("VolumeEcShardsMount should fail for missing EC shards") |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{ |
|||
VolumeId: 98796, |
|||
ShardIds: []uint32{0}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsUnmount missing shards should be no-op success, got: %v", err) |
|||
} |
|||
|
|||
readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{ |
|||
VolumeId: 98797, |
|||
ShardId: 0, |
|||
Offset: 0, |
|||
Size: 1, |
|||
}) |
|||
if err == nil { |
|||
_, err = readStream.Recv() |
|||
} |
|||
if err == nil || err == io.EOF { |
|||
t.Fatalf("VolumeEcShardRead should fail for missing EC volume") |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{ |
|||
VolumeId: 98798, |
|||
Collection: "ec-blob", |
|||
FileKey: 1, |
|||
Version: 3, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcBlobDelete missing local EC volume should be no-op success, got: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ |
|||
VolumeId: 98799, |
|||
Collection: "ec-to-volume", |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "not found") { |
|||
t.Fatalf("VolumeEcShardsToVolume missing-volume error mismatch: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{ |
|||
VolumeId: 98800, |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "not found") { |
|||
t.Fatalf("VolumeEcShardsInfo missing-volume error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestEcGenerateMountInfoUnmountLifecycle(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(115) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 990001, 0x1234ABCD) |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-generate-lifecycle-content")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsGenerate success path failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
ShardIds: []uint32{0}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsMount success path failed: %v", err) |
|||
} |
|||
|
|||
infoResp, err := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{ |
|||
VolumeId: volumeID, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsInfo after mount failed: %v", err) |
|||
} |
|||
if len(infoResp.GetEcShardInfos()) == 0 { |
|||
t.Fatalf("VolumeEcShardsInfo expected non-empty shard infos after mount") |
|||
} |
|||
if infoResp.GetVolumeSize() == 0 { |
|||
t.Fatalf("VolumeEcShardsInfo expected non-zero volume size after mount") |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{ |
|||
VolumeId: volumeID, |
|||
ShardIds: []uint32{0}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsUnmount success path failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{ |
|||
VolumeId: volumeID, |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "not found") { |
|||
t.Fatalf("VolumeEcShardsInfo after unmount expected not-found error, got: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestEcShardReadAndBlobDeleteLifecycle(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(116) |
|||
const fileKey = uint64(990002) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, fileKey, 0x2233CCDD) |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-shard-read-delete-content")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
ShardIds: []uint32{0}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsMount failed: %v", err) |
|||
} |
|||
|
|||
readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{ |
|||
VolumeId: volumeID, |
|||
ShardId: 0, |
|||
Offset: 0, |
|||
Size: 1, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardRead start failed: %v", err) |
|||
} |
|||
firstChunk, err := readStream.Recv() |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardRead recv failed: %v", err) |
|||
} |
|||
if len(firstChunk.GetData()) == 0 { |
|||
t.Fatalf("VolumeEcShardRead expected non-empty data chunk before deletion") |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
FileKey: fileKey, |
|||
Version: uint32(needle.GetCurrentVersion()), |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcBlobDelete first delete failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
FileKey: fileKey, |
|||
Version: uint32(needle.GetCurrentVersion()), |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcBlobDelete second delete should be idempotent success, got: %v", err) |
|||
} |
|||
|
|||
deletedStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{ |
|||
VolumeId: volumeID, |
|||
ShardId: 0, |
|||
FileKey: fileKey, |
|||
Offset: 0, |
|||
Size: 1, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardRead deleted-check start failed: %v", err) |
|||
} |
|||
deletedMsg, err := deletedStream.Recv() |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardRead deleted-check recv failed: %v", err) |
|||
} |
|||
if !deletedMsg.GetIsDeleted() { |
|||
t.Fatalf("VolumeEcShardRead expected IsDeleted=true after blob delete") |
|||
} |
|||
_, err = deletedStream.Recv() |
|||
if err != io.EOF { |
|||
t.Fatalf("VolumeEcShardRead deleted-check expected EOF after deleted marker, got: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestEcRebuildMissingShardLifecycle(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(117) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 990003, 0x3344DDEE) |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-rebuild-shard-content")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
ShardIds: []uint32{0}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsDelete shard 0 failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
ShardIds: []uint32{0}, |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("VolumeEcShardsMount should fail when shard 0 has been deleted") |
|||
} |
|||
|
|||
rebuildResp, err := grpcClient.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsRebuild failed: %v", err) |
|||
} |
|||
if len(rebuildResp.GetRebuiltShardIds()) == 0 { |
|||
t.Fatalf("VolumeEcShardsRebuild expected rebuilt shard ids") |
|||
} |
|||
foundShard0 := false |
|||
for _, shardID := range rebuildResp.GetRebuiltShardIds() { |
|||
if shardID == 0 { |
|||
foundShard0 = true |
|||
break |
|||
} |
|||
} |
|||
if !foundShard0 { |
|||
t.Fatalf("VolumeEcShardsRebuild expected shard 0 to be rebuilt, got %v", rebuildResp.GetRebuiltShardIds()) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
ShardIds: []uint32{0}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsMount shard 0 after rebuild failed: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestEcShardsToVolumeMissingShardAndNoLiveEntries(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
|||
defer cancel() |
|||
|
|||
t.Run("missing shard returns error", func(t *testing.T) { |
|||
const volumeID = uint32(118) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, 990004, 0x4455EEFF) |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-to-volume-missing-shard-content")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
ShardIds: []uint32{0}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsDelete shard 0 failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
ShardIds: []uint32{1}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsMount shard 1 failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "missing shard 0") { |
|||
t.Fatalf("VolumeEcShardsToVolume missing-shard error mismatch: %v", err) |
|||
} |
|||
}) |
|||
|
|||
t.Run("no live entries returns failed precondition", func(t *testing.T) { |
|||
const volumeID = uint32(119) |
|||
const needleID = uint64(990005) |
|||
const cookie = uint32(0x5566FF11) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-no-live-entries-content")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
deleteResp := framework.DoRequest(t, httpClient, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid)) |
|||
_ = framework.ReadAllAndClose(t, deleteResp) |
|||
if deleteResp.StatusCode != http.StatusAccepted { |
|||
t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode) |
|||
} |
|||
|
|||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsMount data shards failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("VolumeEcShardsToVolume expected failed-precondition error when no live entries") |
|||
} |
|||
if status.Code(err) != codes.FailedPrecondition { |
|||
t.Fatalf("VolumeEcShardsToVolume no-live-entries expected FailedPrecondition, got %v (%v)", status.Code(err), err) |
|||
} |
|||
if !strings.Contains(err.Error(), erasure_coding.EcNoLiveEntriesSubstring) { |
|||
t.Fatalf("VolumeEcShardsToVolume no-live-entries error should mention %q, got %v", erasure_coding.EcNoLiveEntriesSubstring, err) |
|||
} |
|||
}) |
|||
} |
|||
|
|||
func TestEcShardsToVolumeSuccessRoundTrip(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(120) |
|||
const needleID = uint64(990006) |
|||
const cookie = uint32(0x66771122) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
payload := []byte("ec-shards-to-volume-success-roundtrip-content") |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsMount data shards failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsToVolume success path failed: %v", err) |
|||
} |
|||
|
|||
readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid) |
|||
readBody := framework.ReadAllAndClose(t, readResp) |
|||
if readResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("post-conversion read expected 200, got %d", readResp.StatusCode) |
|||
} |
|||
if string(readBody) != string(payload) { |
|||
t.Fatalf("post-conversion payload mismatch: got %q want %q", string(readBody), string(payload)) |
|||
} |
|||
} |
|||
|
|||
func TestEcShardsDeleteLastShardRemovesEcx(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(121) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 990007, 0x77882233) |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-delete-all-shards-content")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err) |
|||
} |
|||
|
|||
// Verify .ecx is present before deleting all shards.
|
|||
ecxBeforeDelete, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
IsEcVolume: true, |
|||
Ext: ".ecx", |
|||
CompactionRevision: math.MaxUint32, |
|||
StopOffset: 1, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("CopyFile .ecx before shard deletion start failed: %v", err) |
|||
} |
|||
if _, err = ecxBeforeDelete.Recv(); err != nil { |
|||
t.Fatalf("CopyFile .ecx before shard deletion recv failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeEcShardsDelete all shards failed: %v", err) |
|||
} |
|||
|
|||
ecxAfterDelete, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
IsEcVolume: true, |
|||
Ext: ".ecx", |
|||
CompactionRevision: math.MaxUint32, |
|||
StopOffset: 1, |
|||
}) |
|||
if err == nil { |
|||
_, err = ecxAfterDelete.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "not found ec volume id") { |
|||
t.Fatalf("CopyFile .ecx after deleting all shards should fail not-found, got: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestEcShardsCopyFromPeerSuccess(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1()) |
|||
sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) |
|||
defer sourceConn.Close() |
|||
destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) |
|||
defer destConn.Close() |
|||
|
|||
const volumeID = uint32(122) |
|||
framework.AllocateVolume(t, sourceClient, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 990008, 0x88993344) |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, []byte("ec-copy-from-peer-content")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("source upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := sourceClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("source VolumeEcShardsGenerate failed: %v", err) |
|||
} |
|||
|
|||
sourceDataNode := clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1] |
|||
_, err = destClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
SourceDataNode: sourceDataNode, |
|||
ShardIds: []uint32{0}, |
|||
CopyEcxFile: true, |
|||
CopyVifFile: true, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("destination VolumeEcShardsCopy success path failed: %v", err) |
|||
} |
|||
|
|||
for _, ext := range []string{".ec00", ".ecx", ".vif"} { |
|||
copyStream, copyErr := destClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "", |
|||
IsEcVolume: true, |
|||
Ext: ext, |
|||
CompactionRevision: math.MaxUint32, |
|||
StopOffset: 1, |
|||
}) |
|||
if copyErr != nil { |
|||
t.Fatalf("destination CopyFile %s start failed: %v", ext, copyErr) |
|||
} |
|||
if _, copyErr = copyStream.Recv(); copyErr != nil { |
|||
t.Fatalf("destination CopyFile %s recv failed: %v", ext, copyErr) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func TestEcShardsCopyFailsWhenSourceUnavailable(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ |
|||
VolumeId: 12345, |
|||
Collection: "", |
|||
SourceDataNode: "127.0.0.1:1.1", |
|||
ShardIds: []uint32{0}, |
|||
CopyEcxFile: true, |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "VolumeEcShardsCopy volume") { |
|||
t.Fatalf("VolumeEcShardsCopy source-unavailable error mismatch: %v", err) |
|||
} |
|||
} |
|||
@ -0,0 +1,139 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"context" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
) |
|||
|
|||
func TestStateAndStatusRPCs(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
initialState, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("GetState failed: %v", err) |
|||
} |
|||
if initialState.GetState() == nil { |
|||
t.Fatalf("GetState returned nil state") |
|||
} |
|||
|
|||
setResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{ |
|||
Maintenance: true, |
|||
Version: initialState.GetState().GetVersion(), |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("SetState(maintenance=true) failed: %v", err) |
|||
} |
|||
if !setResp.GetState().GetMaintenance() { |
|||
t.Fatalf("expected maintenance=true after SetState") |
|||
} |
|||
|
|||
setResp, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{ |
|||
Maintenance: false, |
|||
Version: setResp.GetState().GetVersion(), |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("SetState(maintenance=false) failed: %v", err) |
|||
} |
|||
if setResp.GetState().GetMaintenance() { |
|||
t.Fatalf("expected maintenance=false after SetState") |
|||
} |
|||
|
|||
statusResp, err := client.VolumeServerStatus(ctx, &volume_server_pb.VolumeServerStatusRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeServerStatus failed: %v", err) |
|||
} |
|||
if statusResp.GetVersion() == "" { |
|||
t.Fatalf("VolumeServerStatus returned empty version") |
|||
} |
|||
if len(statusResp.GetDiskStatuses()) == 0 { |
|||
t.Fatalf("VolumeServerStatus returned no disk statuses") |
|||
} |
|||
if statusResp.GetState() == nil { |
|||
t.Fatalf("VolumeServerStatus returned nil state") |
|||
} |
|||
if statusResp.GetMemoryStatus() == nil { |
|||
t.Fatalf("VolumeServerStatus returned nil memory status") |
|||
} |
|||
if statusResp.GetMemoryStatus().GetGoroutines() <= 0 { |
|||
t.Fatalf("VolumeServerStatus memory status should report goroutines, got %d", statusResp.GetMemoryStatus().GetGoroutines()) |
|||
} |
|||
|
|||
pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("Ping failed: %v", err) |
|||
} |
|||
if pingResp.GetStartTimeNs() == 0 || pingResp.GetStopTimeNs() == 0 { |
|||
t.Fatalf("Ping timestamps should be non-zero: %+v", pingResp) |
|||
} |
|||
if pingResp.GetStopTimeNs() < pingResp.GetStartTimeNs() { |
|||
t.Fatalf("Ping stop time should be >= start time: %+v", pingResp) |
|||
} |
|||
} |
|||
|
|||
func TestSetStateVersionMismatchAndNilStateNoop(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
initialState, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("GetState failed: %v", err) |
|||
} |
|||
initialVersion := initialState.GetState().GetVersion() |
|||
|
|||
staleResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{ |
|||
Maintenance: true, |
|||
Version: initialVersion + 1, |
|||
}, |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("SetState with stale version should fail") |
|||
} |
|||
if !strings.Contains(err.Error(), "version mismatch") { |
|||
t.Fatalf("SetState stale version error mismatch: %v", err) |
|||
} |
|||
if staleResp.GetState().GetVersion() != initialVersion { |
|||
t.Fatalf("SetState stale version should not mutate server version: got %d want %d", staleResp.GetState().GetVersion(), initialVersion) |
|||
} |
|||
if staleResp.GetState().GetMaintenance() != initialState.GetState().GetMaintenance() { |
|||
t.Fatalf("SetState stale version should not mutate maintenance flag") |
|||
} |
|||
|
|||
nilResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("SetState nil-state request should be no-op success: %v", err) |
|||
} |
|||
if nilResp.GetState().GetVersion() != initialVersion { |
|||
t.Fatalf("SetState nil-state should keep version unchanged: got %d want %d", nilResp.GetState().GetVersion(), initialVersion) |
|||
} |
|||
if nilResp.GetState().GetMaintenance() != initialState.GetState().GetMaintenance() { |
|||
t.Fatalf("SetState nil-state should keep maintenance unchanged") |
|||
} |
|||
} |
|||
@ -0,0 +1,385 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"context" |
|||
"io" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
) |
|||
|
|||
func TestScrubVolumeIndexAndUnsupportedMode(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(61) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
indexResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ |
|||
VolumeIds: []uint32{volumeID}, |
|||
Mode: volume_server_pb.VolumeScrubMode_INDEX, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("ScrubVolume index mode failed: %v", err) |
|||
} |
|||
if indexResp.GetTotalVolumes() != 1 { |
|||
t.Fatalf("ScrubVolume expected total_volumes=1, got %d", indexResp.GetTotalVolumes()) |
|||
} |
|||
|
|||
_, err = grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ |
|||
VolumeIds: []uint32{volumeID}, |
|||
Mode: volume_server_pb.VolumeScrubMode(99), |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("ScrubVolume should fail for unsupported mode") |
|||
} |
|||
if !strings.Contains(err.Error(), "unsupported volume scrub mode") { |
|||
t.Fatalf("ScrubVolume unsupported mode error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestScrubEcVolumeMissingVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{ |
|||
VolumeIds: []uint32{98765}, |
|||
Mode: volume_server_pb.VolumeScrubMode_INDEX, |
|||
}) |
|||
if err == nil { |
|||
t.Fatalf("ScrubEcVolume should fail for missing EC volume") |
|||
} |
|||
if !strings.Contains(err.Error(), "EC volume id") { |
|||
t.Fatalf("ScrubEcVolume missing-volume error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestScrubEcVolumeAutoSelectNoEcVolumes(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{ |
|||
Mode: volume_server_pb.VolumeScrubMode_INDEX, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("ScrubEcVolume auto-select failed: %v", err) |
|||
} |
|||
if resp.GetTotalVolumes() != 0 { |
|||
t.Fatalf("ScrubEcVolume auto-select expected total_volumes=0 without EC data, got %d", resp.GetTotalVolumes()) |
|||
} |
|||
if len(resp.GetBrokenVolumeIds()) != 0 { |
|||
t.Fatalf("ScrubEcVolume auto-select expected no broken volumes, got %v", resp.GetBrokenVolumeIds()) |
|||
} |
|||
} |
|||
|
|||
func TestQueryInvalidAndMissingFileIDPaths(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
invalidStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ |
|||
FromFileIds: []string{"bad-fid"}, |
|||
Selections: []string{"name"}, |
|||
Filter: &volume_server_pb.QueryRequest_Filter{}, |
|||
InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ |
|||
JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{}, |
|||
}, |
|||
}) |
|||
if err == nil { |
|||
_, err = invalidStream.Recv() |
|||
} |
|||
if err == nil { |
|||
t.Fatalf("Query should fail for invalid file id") |
|||
} |
|||
|
|||
missingFid := framework.NewFileID(98766, 1, 1) |
|||
missingStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ |
|||
FromFileIds: []string{missingFid}, |
|||
Selections: []string{"name"}, |
|||
Filter: &volume_server_pb.QueryRequest_Filter{}, |
|||
InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ |
|||
JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{}, |
|||
}, |
|||
}) |
|||
if err == nil { |
|||
_, err = missingStream.Recv() |
|||
} |
|||
if err == nil { |
|||
t.Fatalf("Query should fail for missing file id volume") |
|||
} |
|||
} |
|||
|
|||
func TestScrubVolumeAutoSelectAndNotImplementedModes(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeIDA = uint32(62) |
|||
const volumeIDB = uint32(63) |
|||
framework.AllocateVolume(t, grpcClient, volumeIDA, "") |
|||
framework.AllocateVolume(t, grpcClient, volumeIDB, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
autoResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ |
|||
Mode: volume_server_pb.VolumeScrubMode_INDEX, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("ScrubVolume auto-select failed: %v", err) |
|||
} |
|||
if autoResp.GetTotalVolumes() < 2 { |
|||
t.Fatalf("ScrubVolume auto-select expected at least 2 volumes, got %d", autoResp.GetTotalVolumes()) |
|||
} |
|||
|
|||
localResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ |
|||
VolumeIds: []uint32{volumeIDA}, |
|||
Mode: volume_server_pb.VolumeScrubMode_LOCAL, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("ScrubVolume local mode failed: %v", err) |
|||
} |
|||
if localResp.GetTotalVolumes() != 1 { |
|||
t.Fatalf("ScrubVolume local mode expected total_volumes=1, got %d", localResp.GetTotalVolumes()) |
|||
} |
|||
if len(localResp.GetBrokenVolumeIds()) != 1 || localResp.GetBrokenVolumeIds()[0] != volumeIDA { |
|||
t.Fatalf("ScrubVolume local mode expected broken volume %d, got %v", volumeIDA, localResp.GetBrokenVolumeIds()) |
|||
} |
|||
if len(localResp.GetDetails()) == 0 || !strings.Contains(strings.Join(localResp.GetDetails(), " "), "not implemented") { |
|||
t.Fatalf("ScrubVolume local mode expected not-implemented details, got %v", localResp.GetDetails()) |
|||
} |
|||
|
|||
fullResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{ |
|||
VolumeIds: []uint32{volumeIDA}, |
|||
Mode: volume_server_pb.VolumeScrubMode_FULL, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("ScrubVolume full mode failed: %v", err) |
|||
} |
|||
if fullResp.GetTotalVolumes() != 1 { |
|||
t.Fatalf("ScrubVolume full mode expected total_volumes=1, got %d", fullResp.GetTotalVolumes()) |
|||
} |
|||
if len(fullResp.GetDetails()) == 0 || !strings.Contains(strings.Join(fullResp.GetDetails(), " "), "not implemented") { |
|||
t.Fatalf("ScrubVolume full mode expected not-implemented details, got %v", fullResp.GetDetails()) |
|||
} |
|||
} |
|||
|
|||
func TestQueryJsonSuccessAndCsvNoOutput(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(64) |
|||
const needleID = uint64(777001) |
|||
const cookie = uint32(0xAABBCCDD) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
jsonLines := []byte("{\"score\":3}\n{\"score\":12}\n{\"score\":18}\n") |
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != 201 { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
queryStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ |
|||
FromFileIds: []string{fid}, |
|||
Selections: []string{"score"}, |
|||
Filter: &volume_server_pb.QueryRequest_Filter{ |
|||
Field: "score", |
|||
Operand: ">", |
|||
Value: "10", |
|||
}, |
|||
InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ |
|||
JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"}, |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("Query json start failed: %v", err) |
|||
} |
|||
|
|||
firstStripe, err := queryStream.Recv() |
|||
if err != nil { |
|||
t.Fatalf("Query json recv failed: %v", err) |
|||
} |
|||
records := string(firstStripe.GetRecords()) |
|||
if !strings.Contains(records, "score:12") || !strings.Contains(records, "score:18") { |
|||
t.Fatalf("Query json records missing expected filtered scores: %q", records) |
|||
} |
|||
if strings.Contains(records, "score:3") { |
|||
t.Fatalf("Query json records should not include filtered-out score: %q", records) |
|||
} |
|||
_, err = queryStream.Recv() |
|||
if err != io.EOF { |
|||
t.Fatalf("Query json expected EOF after first stripe, got: %v", err) |
|||
} |
|||
|
|||
csvStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ |
|||
FromFileIds: []string{fid}, |
|||
Selections: []string{"score"}, |
|||
Filter: &volume_server_pb.QueryRequest_Filter{}, |
|||
InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ |
|||
CsvInput: &volume_server_pb.QueryRequest_InputSerialization_CSVInput{}, |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("Query csv start failed: %v", err) |
|||
} |
|||
_, err = csvStream.Recv() |
|||
if err != io.EOF { |
|||
t.Fatalf("Query csv expected EOF with no rows, got: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestQueryJsonNoMatchReturnsEmptyStripe(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(65) |
|||
const needleID = uint64(777002) |
|||
const cookie = uint32(0xABABCDCD) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
jsonLines := []byte("{\"score\":1}\n{\"score\":2}\n") |
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != 201 { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
queryStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ |
|||
FromFileIds: []string{fid}, |
|||
Selections: []string{"score"}, |
|||
Filter: &volume_server_pb.QueryRequest_Filter{ |
|||
Field: "score", |
|||
Operand: ">", |
|||
Value: "100", |
|||
}, |
|||
InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ |
|||
JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"}, |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("Query json no-match start failed: %v", err) |
|||
} |
|||
|
|||
firstStripe, err := queryStream.Recv() |
|||
if err != nil { |
|||
t.Fatalf("Query json no-match recv failed: %v", err) |
|||
} |
|||
if len(firstStripe.GetRecords()) != 0 { |
|||
t.Fatalf("Query json no-match expected empty records stripe, got: %q", string(firstStripe.GetRecords())) |
|||
} |
|||
|
|||
_, err = queryStream.Recv() |
|||
if err != io.EOF { |
|||
t.Fatalf("Query json no-match expected EOF after first empty stripe, got: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestQueryCookieMismatchReturnsEOFNoResults(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(66) |
|||
const needleID = uint64(777003) |
|||
const cookie = uint32(0xCDCDABAB) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
jsonLines := []byte("{\"score\":7}\n{\"score\":8}\n") |
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != 201 { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
wrongCookieFid := framework.NewFileID(volumeID, needleID, cookie+1) |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{ |
|||
FromFileIds: []string{wrongCookieFid}, |
|||
Selections: []string{"score"}, |
|||
Filter: &volume_server_pb.QueryRequest_Filter{ |
|||
Field: "score", |
|||
Operand: ">", |
|||
Value: "0", |
|||
}, |
|||
InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{ |
|||
JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"}, |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("Query start for cookie mismatch should not fail immediately, got: %v", err) |
|||
} |
|||
|
|||
_, err = stream.Recv() |
|||
if err != io.EOF { |
|||
t.Fatalf("Query cookie mismatch expected EOF with no streamed records, got: %v", err) |
|||
} |
|||
} |
|||
@ -0,0 +1,206 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"bytes" |
|||
"context" |
|||
"io" |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
) |
|||
|
|||
func TestVolumeTailSenderMissingVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{VolumeId: 77777, SinceNs: 0, IdleTimeoutSeconds: 1}) |
|||
if err == nil { |
|||
_, err = stream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "not found volume") { |
|||
t.Fatalf("VolumeTailSender missing-volume error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeTailSenderHeartbeatThenEOF(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(71) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{ |
|||
VolumeId: volumeID, |
|||
SinceNs: 0, |
|||
IdleTimeoutSeconds: 1, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeTailSender start failed: %v", err) |
|||
} |
|||
|
|||
msg, err := stream.Recv() |
|||
if err != nil { |
|||
t.Fatalf("VolumeTailSender first recv failed: %v", err) |
|||
} |
|||
if !msg.GetIsLastChunk() { |
|||
t.Fatalf("expected first tail message to be heartbeat IsLastChunk=true") |
|||
} |
|||
|
|||
_, err = stream.Recv() |
|||
if err != io.EOF { |
|||
t.Fatalf("expected EOF after idle timeout drain, got: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeTailReceiverMissingVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{VolumeId: 88888, SourceVolumeServer: clusterHarness.VolumeServerAddress(), SinceNs: 0, IdleTimeoutSeconds: 1}) |
|||
if err == nil || !strings.Contains(err.Error(), "receiver not found volume") { |
|||
t.Fatalf("VolumeTailReceiver missing-volume error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeTailReceiverReplicatesSourceUpdates(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1()) |
|||
sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) |
|||
defer sourceConn.Close() |
|||
destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) |
|||
defer destConn.Close() |
|||
|
|||
const volumeID = uint32(72) |
|||
framework.AllocateVolume(t, sourceClient, volumeID, "") |
|||
framework.AllocateVolume(t, destClient, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 880003, 0x3456789A) |
|||
payload := []byte("tail-receiver-replicates-source-updates") |
|||
|
|||
sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, sourceUploadResp) |
|||
if sourceUploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("source upload expected 201, got %d", sourceUploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := destClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{ |
|||
VolumeId: volumeID, |
|||
SourceVolumeServer: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1], |
|||
SinceNs: 0, |
|||
IdleTimeoutSeconds: 1, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeTailReceiver success path failed: %v", err) |
|||
} |
|||
|
|||
destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid) |
|||
destReadBody := framework.ReadAllAndClose(t, destReadResp) |
|||
if destReadResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("destination read after tail receive expected 200, got %d", destReadResp.StatusCode) |
|||
} |
|||
if string(destReadBody) != string(payload) { |
|||
t.Fatalf("destination tail-received payload mismatch: got %q want %q", string(destReadBody), string(payload)) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeTailSenderLargeNeedleChunking(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(73) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
httpClient := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 880004, 0x456789AB) |
|||
largePayload := bytes.Repeat([]byte("L"), 2*1024*1024+128*1024) |
|||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, largePayload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
|||
defer cancel() |
|||
|
|||
stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{ |
|||
VolumeId: volumeID, |
|||
SinceNs: 0, |
|||
IdleTimeoutSeconds: 1, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("VolumeTailSender start failed: %v", err) |
|||
} |
|||
|
|||
dataChunkCount := 0 |
|||
sawNonLastDataChunk := false |
|||
sawLastDataChunk := false |
|||
for { |
|||
msg, recvErr := stream.Recv() |
|||
if recvErr == io.EOF { |
|||
break |
|||
} |
|||
if recvErr != nil { |
|||
t.Fatalf("VolumeTailSender recv failed: %v", recvErr) |
|||
} |
|||
if len(msg.GetNeedleBody()) == 0 { |
|||
continue |
|||
} |
|||
dataChunkCount++ |
|||
if msg.GetIsLastChunk() { |
|||
sawLastDataChunk = true |
|||
} else { |
|||
sawNonLastDataChunk = true |
|||
} |
|||
} |
|||
|
|||
if dataChunkCount < 2 { |
|||
t.Fatalf("VolumeTailSender expected multiple chunks for large needle, got %d", dataChunkCount) |
|||
} |
|||
if !sawNonLastDataChunk { |
|||
t.Fatalf("VolumeTailSender expected at least one non-last data chunk") |
|||
} |
|||
if !sawLastDataChunk { |
|||
t.Fatalf("VolumeTailSender expected a final data chunk marked IsLastChunk=true") |
|||
} |
|||
} |
|||
@ -0,0 +1,236 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"context" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
) |
|||
|
|||
func TestFetchAndWriteNeedleMaintenanceAndMissingVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{ |
|||
VolumeId: 98781, |
|||
NeedleId: 1, |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "not found volume id") { |
|||
t.Fatalf("FetchAndWriteNeedle missing-volume error mismatch: %v", err) |
|||
} |
|||
|
|||
stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("GetState failed: %v", err) |
|||
} |
|||
_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{ |
|||
Maintenance: true, |
|||
Version: stateResp.GetState().GetVersion(), |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("SetState maintenance=true failed: %v", err) |
|||
} |
|||
|
|||
_, err = grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{ |
|||
VolumeId: 1, |
|||
NeedleId: 1, |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("FetchAndWriteNeedle maintenance error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestFetchAndWriteNeedleInvalidRemoteConfig(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(88) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
_, err := grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{ |
|||
VolumeId: volumeID, |
|||
NeedleId: 1, |
|||
Cookie: 1, |
|||
Size: 1, |
|||
RemoteConf: &remote_pb.RemoteConf{ |
|||
Name: "it-invalid-remote", |
|||
Type: "does-not-exist", |
|||
}, |
|||
RemoteLocation: &remote_pb.RemoteStorageLocation{ |
|||
Name: "it-invalid-remote", |
|||
Path: "/test", |
|||
}, |
|||
}) |
|||
if err == nil || !strings.Contains(err.Error(), "get remote client") { |
|||
t.Fatalf("FetchAndWriteNeedle invalid-remote error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeTierMoveDatToRemoteErrorPaths(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(85) |
|||
const collection = "tier-collection" |
|||
framework.AllocateVolume(t, grpcClient, volumeID, collection) |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
missingStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ |
|||
VolumeId: 98782, |
|||
Collection: collection, |
|||
DestinationBackendName: "dummy", |
|||
}) |
|||
if err == nil { |
|||
_, err = missingStream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "not found") { |
|||
t.Fatalf("VolumeTierMoveDatToRemote missing-volume error mismatch: %v", err) |
|||
} |
|||
|
|||
mismatchStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "wrong-collection", |
|||
DestinationBackendName: "dummy", |
|||
}) |
|||
if err == nil { |
|||
_, err = mismatchStream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "unexpected input") { |
|||
t.Fatalf("VolumeTierMoveDatToRemote collection mismatch error mismatch: %v", err) |
|||
} |
|||
|
|||
stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("GetState failed: %v", err) |
|||
} |
|||
_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{ |
|||
Maintenance: true, |
|||
Version: stateResp.GetState().GetVersion(), |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("SetState maintenance=true failed: %v", err) |
|||
} |
|||
|
|||
maintenanceStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: collection, |
|||
DestinationBackendName: "dummy", |
|||
}) |
|||
if err == nil { |
|||
_, err = maintenanceStream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("VolumeTierMoveDatToRemote maintenance error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeTierMoveDatToRemoteMissingBackend(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(89) |
|||
const collection = "tier-missing-backend" |
|||
framework.AllocateVolume(t, grpcClient, volumeID, collection) |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: collection, |
|||
DestinationBackendName: "definitely-missing-backend", |
|||
}) |
|||
if err == nil { |
|||
_, err = stream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "destination definitely-missing-backend not found") { |
|||
t.Fatalf("VolumeTierMoveDatToRemote missing-backend error mismatch: %v", err) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeTierMoveDatFromRemoteErrorPaths(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(86) |
|||
const collection = "tier-download-collection" |
|||
framework.AllocateVolume(t, grpcClient, volumeID, collection) |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
missingStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{ |
|||
VolumeId: 98783, |
|||
Collection: collection, |
|||
}) |
|||
if err == nil { |
|||
_, err = missingStream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "not found") { |
|||
t.Fatalf("VolumeTierMoveDatFromRemote missing-volume error mismatch: %v", err) |
|||
} |
|||
|
|||
mismatchStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: "wrong-collection", |
|||
}) |
|||
if err == nil { |
|||
_, err = mismatchStream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "unexpected input") { |
|||
t.Fatalf("VolumeTierMoveDatFromRemote collection mismatch error mismatch: %v", err) |
|||
} |
|||
|
|||
localDiskStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: collection, |
|||
}) |
|||
if err == nil { |
|||
_, err = localDiskStream.Recv() |
|||
} |
|||
if err == nil || !strings.Contains(err.Error(), "already on local disk") { |
|||
t.Fatalf("VolumeTierMoveDatFromRemote local-disk error mismatch: %v", err) |
|||
} |
|||
} |
|||
@ -0,0 +1,87 @@ |
|||
package volume_server_grpc_test |
|||
|
|||
import ( |
|||
"context" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
) |
|||
|
|||
func TestVacuumVolumeCheckSuccessAndMissingVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(31) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
resp, err := grpcClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{VolumeId: volumeID}) |
|||
if err != nil { |
|||
t.Fatalf("VacuumVolumeCheck existing volume failed: %v", err) |
|||
} |
|||
if resp.GetGarbageRatio() < 0 || resp.GetGarbageRatio() > 1 { |
|||
t.Fatalf("unexpected garbage ratio: %f", resp.GetGarbageRatio()) |
|||
} |
|||
|
|||
_, err = grpcClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{VolumeId: 99999}) |
|||
if err == nil { |
|||
t.Fatalf("VacuumVolumeCheck should fail for missing volume") |
|||
} |
|||
} |
|||
|
|||
func TestVacuumMaintenanceModeRejections(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
|
|||
stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{}) |
|||
if err != nil { |
|||
t.Fatalf("GetState failed: %v", err) |
|||
} |
|||
_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{ |
|||
State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()}, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("SetState maintenance=true failed: %v", err) |
|||
} |
|||
|
|||
assertMaintenanceErr := func(name string, err error) { |
|||
t.Helper() |
|||
if err == nil { |
|||
t.Fatalf("%s should fail in maintenance mode", name) |
|||
} |
|||
if !strings.Contains(err.Error(), "maintenance mode") { |
|||
t.Fatalf("%s expected maintenance mode error, got: %v", name, err) |
|||
} |
|||
} |
|||
|
|||
compactStream, err := grpcClient.VacuumVolumeCompact(ctx, &volume_server_pb.VacuumVolumeCompactRequest{VolumeId: 31}) |
|||
if err == nil { |
|||
_, err = compactStream.Recv() |
|||
} |
|||
assertMaintenanceErr("VacuumVolumeCompact", err) |
|||
|
|||
_, err = grpcClient.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: 31}) |
|||
assertMaintenanceErr("VacuumVolumeCommit", err) |
|||
|
|||
_, err = grpcClient.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{VolumeId: 31}) |
|||
assertMaintenanceErr("VacuumVolumeCleanup", err) |
|||
} |
|||
@ -0,0 +1,174 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/util/request_id" |
|||
) |
|||
|
|||
func TestAdminStatusAndHealthz(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
client := framework.NewHTTPClient() |
|||
|
|||
statusReq, err := http.NewRequest(http.MethodGet, cluster.VolumeAdminURL()+"/status", nil) |
|||
if err != nil { |
|||
t.Fatalf("create status request: %v", err) |
|||
} |
|||
statusReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-1") |
|||
|
|||
statusResp := framework.DoRequest(t, client, statusReq) |
|||
statusBody := framework.ReadAllAndClose(t, statusResp) |
|||
|
|||
if statusResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("expected /status code 200, got %d, body: %s", statusResp.StatusCode, string(statusBody)) |
|||
} |
|||
if got := statusResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") { |
|||
t.Fatalf("expected /status Server header to contain SeaweedFS Volume, got %q", got) |
|||
} |
|||
if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-1" { |
|||
t.Fatalf("expected echoed request id, got %q", got) |
|||
} |
|||
|
|||
var payload map[string]interface{} |
|||
if err := json.Unmarshal(statusBody, &payload); err != nil { |
|||
t.Fatalf("decode status response: %v", err) |
|||
} |
|||
for _, field := range []string{"Version", "DiskStatuses", "Volumes"} { |
|||
if _, found := payload[field]; !found { |
|||
t.Fatalf("status payload missing field %q", field) |
|||
} |
|||
} |
|||
|
|||
healthReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz") |
|||
healthReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-2") |
|||
healthResp := framework.DoRequest(t, client, healthReq) |
|||
_ = framework.ReadAllAndClose(t, healthResp) |
|||
if healthResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("expected /healthz code 200, got %d", healthResp.StatusCode) |
|||
} |
|||
if got := healthResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") { |
|||
t.Fatalf("expected /healthz Server header to contain SeaweedFS Volume, got %q", got) |
|||
} |
|||
if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-2" { |
|||
t.Fatalf("expected /healthz echoed request id, got %q", got) |
|||
} |
|||
|
|||
uiResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html")) |
|||
uiBody := framework.ReadAllAndClose(t, uiResp) |
|||
if uiResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("expected /ui/index.html code 200, got %d, body: %s", uiResp.StatusCode, string(uiBody)) |
|||
} |
|||
if !strings.Contains(strings.ToLower(string(uiBody)), "volume") { |
|||
t.Fatalf("ui page does not look like volume status page") |
|||
} |
|||
} |
|||
|
|||
func TestOptionsMethodsByPort(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P2()) |
|||
client := framework.NewHTTPClient() |
|||
|
|||
adminResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/")) |
|||
_ = framework.ReadAllAndClose(t, adminResp) |
|||
if adminResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("admin OPTIONS expected 200, got %d", adminResp.StatusCode) |
|||
} |
|||
adminAllowed := adminResp.Header.Get("Access-Control-Allow-Methods") |
|||
for _, expected := range []string{"PUT", "POST", "GET", "DELETE", "OPTIONS"} { |
|||
if !strings.Contains(adminAllowed, expected) { |
|||
t.Fatalf("admin allow methods missing %q, got %q", expected, adminAllowed) |
|||
} |
|||
} |
|||
if adminResp.Header.Get("Access-Control-Allow-Headers") != "*" { |
|||
t.Fatalf("admin allow headers expected '*', got %q", adminResp.Header.Get("Access-Control-Allow-Headers")) |
|||
} |
|||
|
|||
publicResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodOptions, cluster.VolumePublicURL()+"/")) |
|||
_ = framework.ReadAllAndClose(t, publicResp) |
|||
if publicResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("public OPTIONS expected 200, got %d", publicResp.StatusCode) |
|||
} |
|||
publicAllowed := publicResp.Header.Get("Access-Control-Allow-Methods") |
|||
if !strings.Contains(publicAllowed, "GET") || !strings.Contains(publicAllowed, "OPTIONS") { |
|||
t.Fatalf("public allow methods expected GET and OPTIONS, got %q", publicAllowed) |
|||
} |
|||
if strings.Contains(publicAllowed, "POST") { |
|||
t.Fatalf("public allow methods should not include POST, got %q", publicAllowed) |
|||
} |
|||
if publicResp.Header.Get("Access-Control-Allow-Headers") != "*" { |
|||
t.Fatalf("public allow headers expected '*', got %q", publicResp.Header.Get("Access-Control-Allow-Headers")) |
|||
} |
|||
} |
|||
|
|||
func TestOptionsWithOriginIncludesCorsHeaders(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P2()) |
|||
client := framework.NewHTTPClient() |
|||
|
|||
adminReq := mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/") |
|||
adminReq.Header.Set("Origin", "https://example.com") |
|||
adminResp := framework.DoRequest(t, client, adminReq) |
|||
_ = framework.ReadAllAndClose(t, adminResp) |
|||
if adminResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("admin OPTIONS expected 200, got %d", adminResp.StatusCode) |
|||
} |
|||
if adminResp.Header.Get("Access-Control-Allow-Origin") != "*" { |
|||
t.Fatalf("admin OPTIONS expected Access-Control-Allow-Origin=*, got %q", adminResp.Header.Get("Access-Control-Allow-Origin")) |
|||
} |
|||
if adminResp.Header.Get("Access-Control-Allow-Credentials") != "true" { |
|||
t.Fatalf("admin OPTIONS expected Access-Control-Allow-Credentials=true, got %q", adminResp.Header.Get("Access-Control-Allow-Credentials")) |
|||
} |
|||
|
|||
publicReq := mustNewRequest(t, http.MethodOptions, cluster.VolumePublicURL()+"/") |
|||
publicReq.Header.Set("Origin", "https://example.com") |
|||
publicResp := framework.DoRequest(t, client, publicReq) |
|||
_ = framework.ReadAllAndClose(t, publicResp) |
|||
if publicResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("public OPTIONS expected 200, got %d", publicResp.StatusCode) |
|||
} |
|||
if publicResp.Header.Get("Access-Control-Allow-Origin") != "*" { |
|||
t.Fatalf("public OPTIONS expected Access-Control-Allow-Origin=*, got %q", publicResp.Header.Get("Access-Control-Allow-Origin")) |
|||
} |
|||
if publicResp.Header.Get("Access-Control-Allow-Credentials") != "true" { |
|||
t.Fatalf("public OPTIONS expected Access-Control-Allow-Credentials=true, got %q", publicResp.Header.Get("Access-Control-Allow-Credentials")) |
|||
} |
|||
} |
|||
|
|||
func TestUiIndexNotExposedWhenJwtSigningEnabled(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P3()) |
|||
client := framework.NewHTTPClient() |
|||
|
|||
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html")) |
|||
body := framework.ReadAllAndClose(t, resp) |
|||
if resp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("expected /ui/index.html to be gated by auth under JWT profile (401), got %d body=%s", resp.StatusCode, string(body)) |
|||
} |
|||
} |
|||
|
|||
func mustNewRequest(t testing.TB, method, url string) *http.Request { |
|||
t.Helper() |
|||
req, err := http.NewRequest(method, url, nil) |
|||
if err != nil { |
|||
t.Fatalf("create request %s %s: %v", method, url, err) |
|||
} |
|||
return req |
|||
} |
|||
@ -0,0 +1,419 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"bytes" |
|||
"net/http" |
|||
"testing" |
|||
"time" |
|||
|
|||
jwt "github.com/golang-jwt/jwt/v5" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/security" |
|||
) |
|||
|
|||
func TestJWTAuthForWriteAndRead(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P3() |
|||
clusterHarness := framework.StartSingleVolumeCluster(t, profile) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(51) |
|||
const needleID = uint64(123456) |
|||
const cookie = uint32(0xABCDEF12) |
|||
|
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
payload := []byte("jwt-protected-content") |
|||
client := framework.NewHTTPClient() |
|||
|
|||
unauthWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
unauthWriteResp := framework.DoRequest(t, client, unauthWrite) |
|||
_ = framework.ReadAllAndClose(t, unauthWriteResp) |
|||
if unauthWriteResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("unauthorized write expected 401, got %d", unauthWriteResp.StatusCode) |
|||
} |
|||
|
|||
invalidWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
invalidWrite.Header.Set("Authorization", "Bearer invalid") |
|||
invalidWriteResp := framework.DoRequest(t, client, invalidWrite) |
|||
_ = framework.ReadAllAndClose(t, invalidWriteResp) |
|||
if invalidWriteResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("invalid write token expected 401, got %d", invalidWriteResp.StatusCode) |
|||
} |
|||
|
|||
writeToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) |
|||
authWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
authWrite.Header.Set("Authorization", "Bearer "+string(writeToken)) |
|||
authWriteResp := framework.DoRequest(t, client, authWrite) |
|||
_ = framework.ReadAllAndClose(t, authWriteResp) |
|||
if authWriteResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("authorized write expected 201, got %d", authWriteResp.StatusCode) |
|||
} |
|||
|
|||
unauthReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
unauthReadResp := framework.DoRequest(t, client, unauthReadReq) |
|||
_ = framework.ReadAllAndClose(t, unauthReadResp) |
|||
if unauthReadResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("unauthorized read expected 401, got %d", unauthReadResp.StatusCode) |
|||
} |
|||
|
|||
readToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) |
|||
authReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
authReadReq.Header.Set("Authorization", "Bearer "+string(readToken)) |
|||
authReadResp := framework.DoRequest(t, client, authReadReq) |
|||
authReadBody := framework.ReadAllAndClose(t, authReadResp) |
|||
if authReadResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("authorized read expected 200, got %d", authReadResp.StatusCode) |
|||
} |
|||
if string(authReadBody) != string(payload) { |
|||
t.Fatalf("authorized read content mismatch: got %q want %q", string(authReadBody), string(payload)) |
|||
} |
|||
} |
|||
|
|||
func TestJWTAuthRejectsFidMismatch(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P3() |
|||
clusterHarness := framework.StartSingleVolumeCluster(t, profile) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(52) |
|||
const needleID = uint64(223344) |
|||
const cookie = uint32(0x10203040) |
|||
const otherNeedleID = uint64(223345) |
|||
const otherCookie = uint32(0x50607080) |
|||
const wrongCookie = uint32(0x10203041) |
|||
|
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
otherFid := framework.NewFileID(volumeID, otherNeedleID, otherCookie) |
|||
payload := []byte("jwt-fid-mismatch-content") |
|||
client := framework.NewHTTPClient() |
|||
|
|||
writeTokenForOtherFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFid) |
|||
mismatchedWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
mismatchedWrite.Header.Set("Authorization", "Bearer "+string(writeTokenForOtherFid)) |
|||
mismatchedWriteResp := framework.DoRequest(t, client, mismatchedWrite) |
|||
_ = framework.ReadAllAndClose(t, mismatchedWriteResp) |
|||
if mismatchedWriteResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("write with mismatched fid token expected 401, got %d", mismatchedWriteResp.StatusCode) |
|||
} |
|||
|
|||
wrongCookieFid := framework.NewFileID(volumeID, needleID, wrongCookie) |
|||
writeTokenWrongCookie := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, wrongCookieFid) |
|||
wrongCookieWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
wrongCookieWrite.Header.Set("Authorization", "Bearer "+string(writeTokenWrongCookie)) |
|||
wrongCookieWriteResp := framework.DoRequest(t, client, wrongCookieWrite) |
|||
_ = framework.ReadAllAndClose(t, wrongCookieWriteResp) |
|||
if wrongCookieWriteResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("write with wrong-cookie fid token expected 401, got %d", wrongCookieWriteResp.StatusCode) |
|||
} |
|||
|
|||
writeTokenForFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) |
|||
validWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
validWrite.Header.Set("Authorization", "Bearer "+string(writeTokenForFid)) |
|||
validWriteResp := framework.DoRequest(t, client, validWrite) |
|||
_ = framework.ReadAllAndClose(t, validWriteResp) |
|||
if validWriteResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("authorized write expected 201, got %d", validWriteResp.StatusCode) |
|||
} |
|||
|
|||
readTokenForOtherFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFid) |
|||
mismatchedReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
mismatchedReadReq.Header.Set("Authorization", "Bearer "+string(readTokenForOtherFid)) |
|||
mismatchedReadResp := framework.DoRequest(t, client, mismatchedReadReq) |
|||
_ = framework.ReadAllAndClose(t, mismatchedReadResp) |
|||
if mismatchedReadResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("read with mismatched fid token expected 401, got %d", mismatchedReadResp.StatusCode) |
|||
} |
|||
|
|||
readTokenWrongCookie := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, wrongCookieFid) |
|||
wrongCookieReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
wrongCookieReadReq.Header.Set("Authorization", "Bearer "+string(readTokenWrongCookie)) |
|||
wrongCookieReadResp := framework.DoRequest(t, client, wrongCookieReadReq) |
|||
_ = framework.ReadAllAndClose(t, wrongCookieReadResp) |
|||
if wrongCookieReadResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("read with wrong-cookie fid token expected 401, got %d", wrongCookieReadResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func newUploadRequest(t testing.TB, url string, payload []byte) *http.Request { |
|||
t.Helper() |
|||
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(payload)) |
|||
if err != nil { |
|||
t.Fatalf("create upload request %s: %v", url, err) |
|||
} |
|||
req.Header.Set("Content-Type", "application/octet-stream") |
|||
return req |
|||
} |
|||
|
|||
func TestJWTAuthRejectsExpiredTokens(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P3() |
|||
clusterHarness := framework.StartSingleVolumeCluster(t, profile) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(53) |
|||
const needleID = uint64(334455) |
|||
const cookie = uint32(0x22334455) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
payload := []byte("expired-token-content") |
|||
client := framework.NewHTTPClient() |
|||
|
|||
expiredWriteToken := mustGenExpiredToken(t, []byte(profile.JWTSigningKey), fid) |
|||
writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
writeReq.Header.Set("Authorization", "Bearer "+expiredWriteToken) |
|||
writeResp := framework.DoRequest(t, client, writeReq) |
|||
_ = framework.ReadAllAndClose(t, writeResp) |
|||
if writeResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("expired write token expected 401, got %d", writeResp.StatusCode) |
|||
} |
|||
|
|||
// Seed data with a valid token so read auth path can be exercised against existing content.
|
|||
validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) |
|||
validWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
validWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) |
|||
validWriteResp := framework.DoRequest(t, client, validWriteReq) |
|||
_ = framework.ReadAllAndClose(t, validWriteResp) |
|||
if validWriteResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("valid write expected 201, got %d", validWriteResp.StatusCode) |
|||
} |
|||
|
|||
expiredReadToken := mustGenExpiredToken(t, []byte(profile.JWTReadKey), fid) |
|||
readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
readReq.Header.Set("Authorization", "Bearer "+expiredReadToken) |
|||
readResp := framework.DoRequest(t, client, readReq) |
|||
_ = framework.ReadAllAndClose(t, readResp) |
|||
if readResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("expired read token expected 401, got %d", readResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestJWTAuthViaQueryParamAndCookie(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P3() |
|||
clusterHarness := framework.StartSingleVolumeCluster(t, profile) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(54) |
|||
const needleID = uint64(445566) |
|||
const cookie = uint32(0x31415926) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
payload := []byte("jwt-query-cookie-content") |
|||
client := framework.NewHTTPClient() |
|||
|
|||
writeToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) |
|||
writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(writeToken), payload) |
|||
writeResp := framework.DoRequest(t, client, writeReq) |
|||
_ = framework.ReadAllAndClose(t, writeResp) |
|||
if writeResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("query-jwt write expected 201, got %d", writeResp.StatusCode) |
|||
} |
|||
|
|||
readToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) |
|||
readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(readToken)}) |
|||
readResp := framework.DoRequest(t, client, readReq) |
|||
readBody := framework.ReadAllAndClose(t, readResp) |
|||
if readResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("cookie-jwt read expected 200, got %d", readResp.StatusCode) |
|||
} |
|||
if string(readBody) != string(payload) { |
|||
t.Fatalf("cookie-jwt read body mismatch: got %q want %q", string(readBody), string(payload)) |
|||
} |
|||
} |
|||
|
|||
func TestJWTTokenSourcePrecedenceQueryOverHeader(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P3() |
|||
clusterHarness := framework.StartSingleVolumeCluster(t, profile) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(55) |
|||
const needleID = uint64(556677) |
|||
const cookie = uint32(0x99887766) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1) |
|||
payload := []byte("jwt-precedence-content") |
|||
client := framework.NewHTTPClient() |
|||
|
|||
validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) |
|||
invalidWriteQueryToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID) |
|||
writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidWriteQueryToken), payload) |
|||
writeReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) |
|||
writeResp := framework.DoRequest(t, client, writeReq) |
|||
_ = framework.ReadAllAndClose(t, writeResp) |
|||
if writeResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("query token should take precedence over header token for write, expected 401 got %d", writeResp.StatusCode) |
|||
} |
|||
|
|||
// Seed data with valid write token, then exercise read precedence.
|
|||
seedWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
seedWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) |
|||
seedWriteResp := framework.DoRequest(t, client, seedWriteReq) |
|||
_ = framework.ReadAllAndClose(t, seedWriteResp) |
|||
if seedWriteResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("seed write expected 201, got %d", seedWriteResp.StatusCode) |
|||
} |
|||
|
|||
validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) |
|||
invalidReadQueryToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID) |
|||
readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidReadQueryToken)) |
|||
readReq.Header.Set("Authorization", "Bearer "+string(validReadToken)) |
|||
readResp := framework.DoRequest(t, client, readReq) |
|||
_ = framework.ReadAllAndClose(t, readResp) |
|||
if readResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("query token should take precedence over header token for read, expected 401 got %d", readResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestJWTTokenSourcePrecedenceHeaderOverCookie(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P3() |
|||
clusterHarness := framework.StartSingleVolumeCluster(t, profile) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(56) |
|||
const needleID = uint64(667788) |
|||
const cookie = uint32(0x11229988) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1) |
|||
payload := []byte("jwt-precedence-header-cookie") |
|||
client := framework.NewHTTPClient() |
|||
|
|||
validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) |
|||
invalidCookieWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID) |
|||
writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
writeReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) |
|||
writeReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieWriteToken)}) |
|||
writeResp := framework.DoRequest(t, client, writeReq) |
|||
_ = framework.ReadAllAndClose(t, writeResp) |
|||
if writeResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("header token should take precedence over cookie token for write, expected 201 got %d", writeResp.StatusCode) |
|||
} |
|||
|
|||
validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) |
|||
invalidCookieReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID) |
|||
readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
readReq.Header.Set("Authorization", "Bearer "+string(validReadToken)) |
|||
readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieReadToken)}) |
|||
readResp := framework.DoRequest(t, client, readReq) |
|||
readBody := framework.ReadAllAndClose(t, readResp) |
|||
if readResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("header token should take precedence over cookie token for read, expected 200 got %d", readResp.StatusCode) |
|||
} |
|||
if string(readBody) != string(payload) { |
|||
t.Fatalf("header-over-cookie read body mismatch: got %q want %q", string(readBody), string(payload)) |
|||
} |
|||
} |
|||
|
|||
func TestJWTTokenSourcePrecedenceQueryOverCookie(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P3() |
|||
clusterHarness := framework.StartSingleVolumeCluster(t, profile) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(57) |
|||
const needleID = uint64(778899) |
|||
const cookie = uint32(0x88776655) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1) |
|||
payload := []byte("jwt-precedence-query-cookie") |
|||
client := framework.NewHTTPClient() |
|||
|
|||
validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid) |
|||
invalidQueryWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID) |
|||
writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidQueryWriteToken), payload) |
|||
writeReq.AddCookie(&http.Cookie{Name: "AT", Value: string(validWriteToken)}) |
|||
writeResp := framework.DoRequest(t, client, writeReq) |
|||
_ = framework.ReadAllAndClose(t, writeResp) |
|||
if writeResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("query token should take precedence over cookie token for write, expected 401 got %d", writeResp.StatusCode) |
|||
} |
|||
|
|||
// Seed data with valid write token so read precedence can be exercised.
|
|||
seedWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
seedWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken)) |
|||
seedWriteResp := framework.DoRequest(t, client, seedWriteReq) |
|||
_ = framework.ReadAllAndClose(t, seedWriteResp) |
|||
if seedWriteResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("seed write expected 201, got %d", seedWriteResp.StatusCode) |
|||
} |
|||
|
|||
validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid) |
|||
invalidQueryReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID) |
|||
readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidQueryReadToken)) |
|||
readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(validReadToken)}) |
|||
readResp := framework.DoRequest(t, client, readReq) |
|||
_ = framework.ReadAllAndClose(t, readResp) |
|||
if readResp.StatusCode != http.StatusUnauthorized { |
|||
t.Fatalf("query token should take precedence over cookie token for read, expected 401 got %d", readResp.StatusCode) |
|||
} |
|||
|
|||
// Validate positive path: valid query token should succeed even if cookie token is invalid.
|
|||
validQueryReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(validReadToken)) |
|||
invalidCookieReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID) |
|||
validQueryReadReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieReadToken)}) |
|||
validQueryReadResp := framework.DoRequest(t, client, validQueryReadReq) |
|||
validQueryReadBody := framework.ReadAllAndClose(t, validQueryReadResp) |
|||
if validQueryReadResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("valid query token should succeed over invalid cookie token, expected 200 got %d", validQueryReadResp.StatusCode) |
|||
} |
|||
if string(validQueryReadBody) != string(payload) { |
|||
t.Fatalf("query-over-cookie read body mismatch: got %q want %q", string(validQueryReadBody), string(payload)) |
|||
} |
|||
} |
|||
|
|||
func mustGenExpiredToken(t testing.TB, key []byte, fid string) string { |
|||
t.Helper() |
|||
claims := security.SeaweedFileIdClaims{ |
|||
Fid: fid, |
|||
RegisteredClaims: jwt.RegisteredClaims{ |
|||
ExpiresAt: jwt.NewNumericDate(time.Now().Add(-1 * time.Minute)), |
|||
}, |
|||
} |
|||
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) |
|||
signed, err := token.SignedString(key) |
|||
if err != nil { |
|||
t.Fatalf("sign expired token: %v", err) |
|||
} |
|||
return signed |
|||
} |
|||
@ -0,0 +1,232 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"bytes" |
|||
"encoding/json" |
|||
"net/http" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/operation" |
|||
) |
|||
|
|||
func TestChunkManifestExpansionAndBypass(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(102) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
|
|||
chunkFID := framework.NewFileID(volumeID, 772005, 0x5E6F7081) |
|||
chunkPayload := []byte("chunk-manifest-expanded-content") |
|||
chunkUploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID, chunkPayload) |
|||
_ = framework.ReadAllAndClose(t, chunkUploadResp) |
|||
if chunkUploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("chunk upload expected 201, got %d", chunkUploadResp.StatusCode) |
|||
} |
|||
|
|||
manifest := &operation.ChunkManifest{ |
|||
Name: "manifest.bin", |
|||
Mime: "application/octet-stream", |
|||
Size: int64(len(chunkPayload)), |
|||
Chunks: []*operation.ChunkInfo{ |
|||
{ |
|||
Fid: chunkFID, |
|||
Offset: 0, |
|||
Size: int64(len(chunkPayload)), |
|||
}, |
|||
}, |
|||
} |
|||
manifestBytes, err := json.Marshal(manifest) |
|||
if err != nil { |
|||
t.Fatalf("marshal chunk manifest: %v", err) |
|||
} |
|||
|
|||
manifestFID := framework.NewFileID(volumeID, 772006, 0x6F708192) |
|||
manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes)) |
|||
if err != nil { |
|||
t.Fatalf("create manifest upload request: %v", err) |
|||
} |
|||
manifestUploadReq.Header.Set("Content-Type", "application/json") |
|||
manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq) |
|||
_ = framework.ReadAllAndClose(t, manifestUploadResp) |
|||
if manifestUploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode) |
|||
} |
|||
|
|||
expandedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), manifestFID) |
|||
expandedReadBody := framework.ReadAllAndClose(t, expandedReadResp) |
|||
if expandedReadResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("manifest expanded read expected 200, got %d", expandedReadResp.StatusCode) |
|||
} |
|||
if string(expandedReadBody) != string(chunkPayload) { |
|||
t.Fatalf("manifest expanded read mismatch: got %q want %q", string(expandedReadBody), string(chunkPayload)) |
|||
} |
|||
if expandedReadResp.Header.Get("X-File-Store") != "chunked" { |
|||
t.Fatalf("manifest expanded read expected X-File-Store=chunked, got %q", expandedReadResp.Header.Get("X-File-Store")) |
|||
} |
|||
|
|||
bypassReadResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=false")) |
|||
bypassReadBody := framework.ReadAllAndClose(t, bypassReadResp) |
|||
if bypassReadResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("manifest bypass read expected 200, got %d", bypassReadResp.StatusCode) |
|||
} |
|||
if bypassReadResp.Header.Get("X-File-Store") != "" { |
|||
t.Fatalf("manifest bypass read expected empty X-File-Store header, got %q", bypassReadResp.Header.Get("X-File-Store")) |
|||
} |
|||
|
|||
var gotManifest operation.ChunkManifest |
|||
if err = json.Unmarshal(bypassReadBody, &gotManifest); err != nil { |
|||
t.Fatalf("manifest bypass read expected JSON payload, got decode error: %v body=%q", err, string(bypassReadBody)) |
|||
} |
|||
if len(gotManifest.Chunks) != 1 || gotManifest.Chunks[0].Fid != chunkFID { |
|||
t.Fatalf("manifest bypass read payload mismatch: %+v", gotManifest) |
|||
} |
|||
} |
|||
|
|||
func TestChunkManifestDeleteRemovesChildChunks(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(104) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
|
|||
chunkFID := framework.NewFileID(volumeID, 772008, 0x8192A3B4) |
|||
chunkPayload := []byte("chunk-manifest-delete-content") |
|||
chunkUploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID, chunkPayload) |
|||
_ = framework.ReadAllAndClose(t, chunkUploadResp) |
|||
if chunkUploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("chunk upload expected 201, got %d", chunkUploadResp.StatusCode) |
|||
} |
|||
|
|||
manifest := &operation.ChunkManifest{ |
|||
Name: "manifest-delete.bin", |
|||
Mime: "application/octet-stream", |
|||
Size: int64(len(chunkPayload)), |
|||
Chunks: []*operation.ChunkInfo{ |
|||
{ |
|||
Fid: chunkFID, |
|||
Offset: 0, |
|||
Size: int64(len(chunkPayload)), |
|||
}, |
|||
}, |
|||
} |
|||
manifestBytes, err := json.Marshal(manifest) |
|||
if err != nil { |
|||
t.Fatalf("marshal chunk manifest: %v", err) |
|||
} |
|||
|
|||
manifestFID := framework.NewFileID(volumeID, 772009, 0x92A3B4C5) |
|||
manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes)) |
|||
if err != nil { |
|||
t.Fatalf("create manifest upload request: %v", err) |
|||
} |
|||
manifestUploadReq.Header.Set("Content-Type", "application/json") |
|||
manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq) |
|||
_ = framework.ReadAllAndClose(t, manifestUploadResp) |
|||
if manifestUploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode) |
|||
} |
|||
|
|||
deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+manifestFID)) |
|||
deleteBody := framework.ReadAllAndClose(t, deleteResp) |
|||
if deleteResp.StatusCode != http.StatusAccepted { |
|||
t.Fatalf("manifest delete expected 202, got %d", deleteResp.StatusCode) |
|||
} |
|||
var deleteResult map[string]int64 |
|||
if err = json.Unmarshal(deleteBody, &deleteResult); err != nil { |
|||
t.Fatalf("decode manifest delete response: %v body=%q", err, string(deleteBody)) |
|||
} |
|||
if deleteResult["size"] != int64(len(chunkPayload)) { |
|||
t.Fatalf("manifest delete expected size=%d, got %d", len(chunkPayload), deleteResult["size"]) |
|||
} |
|||
|
|||
manifestReadAfterDelete := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), manifestFID) |
|||
_ = framework.ReadAllAndClose(t, manifestReadAfterDelete) |
|||
if manifestReadAfterDelete.StatusCode != http.StatusNotFound { |
|||
t.Fatalf("manifest read after delete expected 404, got %d", manifestReadAfterDelete.StatusCode) |
|||
} |
|||
|
|||
chunkReadAfterDelete := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID) |
|||
_ = framework.ReadAllAndClose(t, chunkReadAfterDelete) |
|||
if chunkReadAfterDelete.StatusCode != http.StatusNotFound { |
|||
t.Fatalf("chunk read after manifest delete expected 404, got %d", chunkReadAfterDelete.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestChunkManifestDeleteFailsWhenChildDeletionFails(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(105) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
manifest := &operation.ChunkManifest{ |
|||
Name: "manifest-delete-failure.bin", |
|||
Mime: "application/octet-stream", |
|||
Size: 1, |
|||
Chunks: []*operation.ChunkInfo{ |
|||
{ |
|||
Fid: "not-a-valid-fid", |
|||
Offset: 0, |
|||
Size: 1, |
|||
}, |
|||
}, |
|||
} |
|||
manifestBytes, err := json.Marshal(manifest) |
|||
if err != nil { |
|||
t.Fatalf("marshal chunk manifest: %v", err) |
|||
} |
|||
|
|||
manifestFID := framework.NewFileID(volumeID, 772010, 0xA3B4C5D6) |
|||
manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes)) |
|||
if err != nil { |
|||
t.Fatalf("create manifest upload request: %v", err) |
|||
} |
|||
manifestUploadReq.Header.Set("Content-Type", "application/json") |
|||
manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq) |
|||
_ = framework.ReadAllAndClose(t, manifestUploadResp) |
|||
if manifestUploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode) |
|||
} |
|||
|
|||
deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+manifestFID)) |
|||
deleteBody := framework.ReadAllAndClose(t, deleteResp) |
|||
if deleteResp.StatusCode != http.StatusInternalServerError { |
|||
t.Fatalf("manifest delete with invalid child fid expected 500, got %d body=%q", deleteResp.StatusCode, string(deleteBody)) |
|||
} |
|||
|
|||
manifestBypassRead := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=false")) |
|||
manifestBypassBody := framework.ReadAllAndClose(t, manifestBypassRead) |
|||
if manifestBypassRead.StatusCode != http.StatusOK { |
|||
t.Fatalf("manifest bypass read after failed delete expected 200, got %d", manifestBypassRead.StatusCode) |
|||
} |
|||
var gotManifest operation.ChunkManifest |
|||
if err = json.Unmarshal(manifestBypassBody, &gotManifest); err != nil { |
|||
t.Fatalf("manifest bypass read expected JSON payload, got decode error: %v body=%q", err, string(manifestBypassBody)) |
|||
} |
|||
if len(gotManifest.Chunks) != 1 || gotManifest.Chunks[0].Fid != "not-a-valid-fid" { |
|||
t.Fatalf("manifest payload mismatch after failed delete: %+v", gotManifest) |
|||
} |
|||
} |
|||
@ -0,0 +1,97 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"bytes" |
|||
"compress/gzip" |
|||
"io" |
|||
"net/http" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
func gzipData(t testing.TB, data []byte) []byte { |
|||
t.Helper() |
|||
var buf bytes.Buffer |
|||
zw := gzip.NewWriter(&buf) |
|||
if _, err := zw.Write(data); err != nil { |
|||
t.Fatalf("gzip write: %v", err) |
|||
} |
|||
if err := zw.Close(); err != nil { |
|||
t.Fatalf("gzip close: %v", err) |
|||
} |
|||
return buf.Bytes() |
|||
} |
|||
|
|||
func gunzipData(t testing.TB, data []byte) []byte { |
|||
t.Helper() |
|||
zr, err := gzip.NewReader(bytes.NewReader(data)) |
|||
if err != nil { |
|||
t.Fatalf("gunzip new reader: %v", err) |
|||
} |
|||
defer zr.Close() |
|||
out, err := io.ReadAll(zr) |
|||
if err != nil { |
|||
t.Fatalf("gunzip read: %v", err) |
|||
} |
|||
return out |
|||
} |
|||
|
|||
func TestCompressedReadAcceptEncodingMatrix(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(103) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 772007, 0x708192A3) |
|||
plainPayload := []byte("compressed-read-accept-encoding-matrix-content-compressed-read-accept-encoding-matrix-content") |
|||
compressedPayload := gzipData(t, plainPayload) |
|||
|
|||
uploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+fid, bytes.NewReader(compressedPayload)) |
|||
if err != nil { |
|||
t.Fatalf("create compressed upload request: %v", err) |
|||
} |
|||
uploadReq.Header.Set("Content-Type", "text/plain") |
|||
uploadReq.Header.Set("Content-Encoding", "gzip") |
|||
uploadResp := framework.DoRequest(t, client, uploadReq) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("compressed upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
gzipReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
gzipReadReq.Header.Set("Accept-Encoding", "gzip") |
|||
gzipReadResp := framework.DoRequest(t, client, gzipReadReq) |
|||
gzipReadBody := framework.ReadAllAndClose(t, gzipReadResp) |
|||
if gzipReadResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("gzip-accepted read expected 200, got %d", gzipReadResp.StatusCode) |
|||
} |
|||
if gzipReadResp.Header.Get("Content-Encoding") != "gzip" { |
|||
t.Fatalf("gzip-accepted read expected Content-Encoding=gzip, got %q", gzipReadResp.Header.Get("Content-Encoding")) |
|||
} |
|||
if string(gunzipData(t, gzipReadBody)) != string(plainPayload) { |
|||
t.Fatalf("gzip-accepted read body mismatch after gunzip") |
|||
} |
|||
|
|||
identityReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
identityReadReq.Header.Set("Accept-Encoding", "identity") |
|||
identityReadResp := framework.DoRequest(t, client, identityReadReq) |
|||
identityReadBody := framework.ReadAllAndClose(t, identityReadResp) |
|||
if identityReadResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("identity read expected 200, got %d", identityReadResp.StatusCode) |
|||
} |
|||
if identityReadResp.Header.Get("Content-Encoding") != "" { |
|||
t.Fatalf("identity read expected no Content-Encoding header, got %q", identityReadResp.Header.Get("Content-Encoding")) |
|||
} |
|||
if string(identityReadBody) != string(plainPayload) { |
|||
t.Fatalf("identity read body mismatch: got %q want %q", string(identityReadBody), string(plainPayload)) |
|||
} |
|||
} |
|||
@ -0,0 +1,102 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"fmt" |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
func TestReadPassthroughHeadersAndDownloadDisposition(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(96) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fullFileID := framework.NewFileID(volumeID, 661122, 0x55667788) |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fullFileID, []byte("passthrough-header-content")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
parts := strings.SplitN(fullFileID, ",", 2) |
|||
if len(parts) != 2 { |
|||
t.Fatalf("unexpected file id format: %q", fullFileID) |
|||
} |
|||
fidOnly := parts[1] |
|||
|
|||
url := fmt.Sprintf("%s/%d/%s/%s?response-content-type=text/plain&response-cache-control=no-store&dl=true", |
|||
clusterHarness.VolumeAdminURL(), |
|||
volumeID, |
|||
fidOnly, |
|||
"report.txt", |
|||
) |
|||
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, url)) |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
if resp.StatusCode != http.StatusOK { |
|||
t.Fatalf("passthrough read expected 200, got %d", resp.StatusCode) |
|||
} |
|||
if resp.Header.Get("Content-Type") != "text/plain" { |
|||
t.Fatalf("response-content-type override mismatch: %q", resp.Header.Get("Content-Type")) |
|||
} |
|||
if resp.Header.Get("Cache-Control") != "no-store" { |
|||
t.Fatalf("response-cache-control override mismatch: %q", resp.Header.Get("Cache-Control")) |
|||
} |
|||
contentDisposition := resp.Header.Get("Content-Disposition") |
|||
if !strings.Contains(contentDisposition, "attachment") || !strings.Contains(contentDisposition, "report.txt") { |
|||
t.Fatalf("download disposition header mismatch: %q", contentDisposition) |
|||
} |
|||
} |
|||
|
|||
func TestStaticAssetEndpoints(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
client := framework.NewHTTPClient() |
|||
|
|||
faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/favicon.ico")) |
|||
_ = framework.ReadAllAndClose(t, faviconResp) |
|||
if faviconResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("/favicon.ico expected 200, got %d", faviconResp.StatusCode) |
|||
} |
|||
|
|||
staticResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/seaweedfsstatic/seaweed50x50.png")) |
|||
_ = framework.ReadAllAndClose(t, staticResp) |
|||
if staticResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("/seaweedfsstatic/seaweed50x50.png expected 200, got %d", staticResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestStaticAssetEndpointsOnPublicPort(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) |
|||
client := framework.NewHTTPClient() |
|||
|
|||
faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/favicon.ico")) |
|||
_ = framework.ReadAllAndClose(t, faviconResp) |
|||
if faviconResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("public /favicon.ico expected 200, got %d", faviconResp.StatusCode) |
|||
} |
|||
|
|||
staticResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/seaweedfsstatic/seaweed50x50.png")) |
|||
_ = framework.ReadAllAndClose(t, staticResp) |
|||
if staticResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("public /seaweedfsstatic/seaweed50x50.png expected 200, got %d", staticResp.StatusCode) |
|||
} |
|||
} |
|||
@ -0,0 +1,92 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"bytes" |
|||
"fmt" |
|||
"image" |
|||
"image/color" |
|||
"image/png" |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
func makePNGFixture(t testing.TB, width, height int) []byte { |
|||
t.Helper() |
|||
|
|||
img := image.NewRGBA(image.Rect(0, 0, width, height)) |
|||
for y := 0; y < height; y++ { |
|||
for x := 0; x < width; x++ { |
|||
img.Set(x, y, color.RGBA{R: uint8(x * 20), G: uint8(y * 20), B: 200, A: 255}) |
|||
} |
|||
} |
|||
|
|||
var buf bytes.Buffer |
|||
if err := png.Encode(&buf, img); err != nil { |
|||
t.Fatalf("encode png fixture: %v", err) |
|||
} |
|||
return buf.Bytes() |
|||
} |
|||
|
|||
func decodeImageConfig(t testing.TB, data []byte) image.Config { |
|||
t.Helper() |
|||
cfg, _, err := image.DecodeConfig(bytes.NewReader(data)) |
|||
if err != nil { |
|||
t.Fatalf("decode image config: %v", err) |
|||
} |
|||
return cfg |
|||
} |
|||
|
|||
func TestImageResizeAndCropReadVariants(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(101) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fullFileID := framework.NewFileID(volumeID, 772004, 0x4D5E6F70) |
|||
uploadReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fullFileID, makePNGFixture(t, 6, 4)) |
|||
uploadReq.Header.Set("Content-Type", "image/png") |
|||
uploadResp := framework.DoRequest(t, client, uploadReq) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("image upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
parts := strings.SplitN(fullFileID, ",", 2) |
|||
if len(parts) != 2 { |
|||
t.Fatalf("unexpected file id format: %q", fullFileID) |
|||
} |
|||
fidOnly := parts[1] |
|||
|
|||
resizeURL := fmt.Sprintf("%s/%d/%s/%s?width=2&height=1", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "fixture.png") |
|||
resizeResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, resizeURL)) |
|||
resizeBody := framework.ReadAllAndClose(t, resizeResp) |
|||
if resizeResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("image resize read expected 200, got %d", resizeResp.StatusCode) |
|||
} |
|||
resizeCfg := decodeImageConfig(t, resizeBody) |
|||
if resizeCfg.Width > 2 || resizeCfg.Height > 1 { |
|||
t.Fatalf("image resize expected dimensions <= 2x1, got %dx%d", resizeCfg.Width, resizeCfg.Height) |
|||
} |
|||
|
|||
cropURL := fmt.Sprintf("%s/%d/%s/%s?crop_x1=1&crop_y1=1&crop_x2=4&crop_y2=3", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "fixture.png") |
|||
cropResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cropURL)) |
|||
cropBody := framework.ReadAllAndClose(t, cropResp) |
|||
if cropResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("image crop read expected 200, got %d", cropResp.StatusCode) |
|||
} |
|||
cropCfg := decodeImageConfig(t, cropBody) |
|||
if cropCfg.Width != 3 || cropCfg.Height != 2 { |
|||
t.Fatalf("image crop expected 3x2, got %dx%d", cropCfg.Width, cropCfg.Height) |
|||
} |
|||
} |
|||
@ -0,0 +1,287 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"bytes" |
|||
"net/http" |
|||
"strconv" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
func TestPublicPortReadOnlyMethodBehavior(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(81) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, 123321, 0x01020304) |
|||
originalData := []byte("public-port-original") |
|||
replacementData := []byte("public-port-replacement") |
|||
client := framework.NewHTTPClient() |
|||
|
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, originalData) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("admin upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
publicReadResp := framework.ReadBytes(t, client, clusterHarness.VolumePublicURL(), fid) |
|||
publicReadBody := framework.ReadAllAndClose(t, publicReadResp) |
|||
if publicReadResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("public GET expected 200, got %d", publicReadResp.StatusCode) |
|||
} |
|||
if string(publicReadBody) != string(originalData) { |
|||
t.Fatalf("public GET body mismatch: got %q want %q", string(publicReadBody), string(originalData)) |
|||
} |
|||
|
|||
publicPostReq := newUploadRequest(t, clusterHarness.VolumePublicURL()+"/"+fid, replacementData) |
|||
publicPostResp := framework.DoRequest(t, client, publicPostReq) |
|||
_ = framework.ReadAllAndClose(t, publicPostResp) |
|||
if publicPostResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("public POST expected passthrough 200, got %d", publicPostResp.StatusCode) |
|||
} |
|||
|
|||
publicDeleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumePublicURL()+"/"+fid)) |
|||
_ = framework.ReadAllAndClose(t, publicDeleteResp) |
|||
if publicDeleteResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("public DELETE expected passthrough 200, got %d", publicDeleteResp.StatusCode) |
|||
} |
|||
|
|||
adminReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) |
|||
adminReadBody := framework.ReadAllAndClose(t, adminReadResp) |
|||
if adminReadResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("admin GET after public POST/DELETE expected 200, got %d", adminReadResp.StatusCode) |
|||
} |
|||
if string(adminReadBody) != string(originalData) { |
|||
t.Fatalf("public port should not mutate data: got %q want %q", string(adminReadBody), string(originalData)) |
|||
} |
|||
} |
|||
|
|||
func TestCorsAndUnsupportedMethodBehavior(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(82) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, 789789, 0x0A0B0C0D) |
|||
client := framework.NewHTTPClient() |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("cors-check")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("admin upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
adminOriginReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
adminOriginReq.Header.Set("Origin", "https://example.com") |
|||
adminOriginResp := framework.DoRequest(t, client, adminOriginReq) |
|||
_ = framework.ReadAllAndClose(t, adminOriginResp) |
|||
if adminOriginResp.Header.Get("Access-Control-Allow-Origin") != "*" { |
|||
t.Fatalf("admin GET origin header mismatch: %q", adminOriginResp.Header.Get("Access-Control-Allow-Origin")) |
|||
} |
|||
if adminOriginResp.Header.Get("Access-Control-Allow-Credentials") != "true" { |
|||
t.Fatalf("admin GET credentials header mismatch: %q", adminOriginResp.Header.Get("Access-Control-Allow-Credentials")) |
|||
} |
|||
|
|||
publicOriginReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/"+fid) |
|||
publicOriginReq.Header.Set("Origin", "https://example.com") |
|||
publicOriginResp := framework.DoRequest(t, client, publicOriginReq) |
|||
_ = framework.ReadAllAndClose(t, publicOriginResp) |
|||
if publicOriginResp.Header.Get("Access-Control-Allow-Origin") != "*" { |
|||
t.Fatalf("public GET origin header mismatch: %q", publicOriginResp.Header.Get("Access-Control-Allow-Origin")) |
|||
} |
|||
if publicOriginResp.Header.Get("Access-Control-Allow-Credentials") != "true" { |
|||
t.Fatalf("public GET credentials header mismatch: %q", publicOriginResp.Header.Get("Access-Control-Allow-Credentials")) |
|||
} |
|||
|
|||
adminPatchReq, err := http.NewRequest(http.MethodPatch, clusterHarness.VolumeAdminURL()+"/"+fid, bytes.NewReader([]byte("patch"))) |
|||
if err != nil { |
|||
t.Fatalf("create admin PATCH request: %v", err) |
|||
} |
|||
adminPatchResp := framework.DoRequest(t, client, adminPatchReq) |
|||
_ = framework.ReadAllAndClose(t, adminPatchResp) |
|||
if adminPatchResp.StatusCode != http.StatusBadRequest { |
|||
t.Fatalf("admin PATCH expected 400, got %d", adminPatchResp.StatusCode) |
|||
} |
|||
|
|||
publicPatchReq, err := http.NewRequest(http.MethodPatch, clusterHarness.VolumePublicURL()+"/"+fid, bytes.NewReader([]byte("patch"))) |
|||
if err != nil { |
|||
t.Fatalf("create public PATCH request: %v", err) |
|||
} |
|||
publicPatchResp := framework.DoRequest(t, client, publicPatchReq) |
|||
_ = framework.ReadAllAndClose(t, publicPatchResp) |
|||
if publicPatchResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("public PATCH expected passthrough 200, got %d", publicPatchResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestUnsupportedMethodTraceParity(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(83) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, 123999, 0x01010101) |
|||
client := framework.NewHTTPClient() |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("trace-method-check")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
adminTraceReq := mustNewRequest(t, http.MethodTrace, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
adminTraceResp := framework.DoRequest(t, client, adminTraceReq) |
|||
_ = framework.ReadAllAndClose(t, adminTraceResp) |
|||
if adminTraceResp.StatusCode != http.StatusBadRequest { |
|||
t.Fatalf("admin TRACE expected 400, got %d", adminTraceResp.StatusCode) |
|||
} |
|||
|
|||
publicTraceReq := mustNewRequest(t, http.MethodTrace, clusterHarness.VolumePublicURL()+"/"+fid) |
|||
publicTraceResp := framework.DoRequest(t, client, publicTraceReq) |
|||
_ = framework.ReadAllAndClose(t, publicTraceResp) |
|||
if publicTraceResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("public TRACE expected passthrough 200, got %d", publicTraceResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestUnsupportedMethodPropfindParity(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(84) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, 124000, 0x02020202) |
|||
client := framework.NewHTTPClient() |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("propfind-method-check")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
adminReq := mustNewRequest(t, "PROPFIND", clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
adminResp := framework.DoRequest(t, client, adminReq) |
|||
_ = framework.ReadAllAndClose(t, adminResp) |
|||
if adminResp.StatusCode != http.StatusBadRequest { |
|||
t.Fatalf("admin PROPFIND expected 400, got %d", adminResp.StatusCode) |
|||
} |
|||
|
|||
publicReq := mustNewRequest(t, "PROPFIND", clusterHarness.VolumePublicURL()+"/"+fid) |
|||
publicResp := framework.DoRequest(t, client, publicReq) |
|||
_ = framework.ReadAllAndClose(t, publicResp) |
|||
if publicResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("public PROPFIND expected passthrough 200, got %d", publicResp.StatusCode) |
|||
} |
|||
|
|||
verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) |
|||
verifyBody := framework.ReadAllAndClose(t, verifyResp) |
|||
if verifyResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode) |
|||
} |
|||
if string(verifyBody) != "propfind-method-check" { |
|||
t.Fatalf("PROPFIND should not mutate data, got %q", string(verifyBody)) |
|||
} |
|||
} |
|||
|
|||
func TestUnsupportedMethodConnectParity(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(85) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, 124001, 0x03030303) |
|||
client := framework.NewHTTPClient() |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("connect-method-check")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
adminReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
adminResp := framework.DoRequest(t, client, adminReq) |
|||
_ = framework.ReadAllAndClose(t, adminResp) |
|||
if adminResp.StatusCode != http.StatusBadRequest { |
|||
t.Fatalf("admin CONNECT expected 400, got %d", adminResp.StatusCode) |
|||
} |
|||
|
|||
publicReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumePublicURL()+"/"+fid) |
|||
publicResp := framework.DoRequest(t, client, publicReq) |
|||
_ = framework.ReadAllAndClose(t, publicResp) |
|||
if publicResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("public CONNECT expected passthrough 200, got %d", publicResp.StatusCode) |
|||
} |
|||
|
|||
verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) |
|||
verifyBody := framework.ReadAllAndClose(t, verifyResp) |
|||
if verifyResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode) |
|||
} |
|||
if string(verifyBody) != "connect-method-check" { |
|||
t.Fatalf("CONNECT should not mutate data, got %q", string(verifyBody)) |
|||
} |
|||
} |
|||
|
|||
func TestPublicPortHeadReadParity(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(86) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, 124002, 0x04040404) |
|||
payload := []byte("public-head-parity-content") |
|||
client := framework.NewHTTPClient() |
|||
|
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumePublicURL()+"/"+fid)) |
|||
headBody := framework.ReadAllAndClose(t, headResp) |
|||
if headResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("public HEAD expected 200, got %d", headResp.StatusCode) |
|||
} |
|||
if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) { |
|||
t.Fatalf("public HEAD content-length mismatch: got %q want %d", got, len(payload)) |
|||
} |
|||
if len(headBody) != 0 { |
|||
t.Fatalf("public HEAD body should be empty, got %d bytes", len(headBody)) |
|||
} |
|||
} |
|||
@ -0,0 +1,82 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
func TestMultiRangeReadReturnsMultipartPayload(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(97) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 771999, 0x0A1B2C3D) |
|||
payload := []byte("0123456789abcdef") |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
multiRangeReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
multiRangeReq.Header.Set("Range", "bytes=0-1,4-5") |
|||
multiRangeResp := framework.DoRequest(t, client, multiRangeReq) |
|||
multiRangeBody := framework.ReadAllAndClose(t, multiRangeResp) |
|||
if multiRangeResp.StatusCode != http.StatusPartialContent { |
|||
t.Fatalf("multi-range expected 206, got %d", multiRangeResp.StatusCode) |
|||
} |
|||
if !strings.Contains(multiRangeResp.Header.Get("Content-Type"), "multipart/byteranges") { |
|||
t.Fatalf("multi-range content-type mismatch: %q", multiRangeResp.Header.Get("Content-Type")) |
|||
} |
|||
|
|||
bodyText := string(multiRangeBody) |
|||
if !strings.Contains(bodyText, "01") || !strings.Contains(bodyText, "45") { |
|||
t.Fatalf("multi-range body missing expected segments: %q", bodyText) |
|||
} |
|||
} |
|||
|
|||
func TestOversizedCombinedRangesAreIgnored(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(100) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 772003, 0x3C4D5E6F) |
|||
payload := []byte("0123456789abcdef") |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
// Range bytes sum is 22 (> payload size 16), which exercises the oversized-range guard path.
|
|||
oversizedRangeReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
oversizedRangeReq.Header.Set("Range", "bytes=0-10,5-15") |
|||
oversizedRangeResp := framework.DoRequest(t, client, oversizedRangeReq) |
|||
oversizedRangeBody := framework.ReadAllAndClose(t, oversizedRangeResp) |
|||
if oversizedRangeResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("oversized combined range expected 200, got %d", oversizedRangeResp.StatusCode) |
|||
} |
|||
if len(oversizedRangeBody) != 0 { |
|||
t.Fatalf("oversized combined range expected empty body, got %d bytes", len(oversizedRangeBody)) |
|||
} |
|||
} |
|||
@ -0,0 +1,54 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"net/http" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
func TestReadDeletedQueryReturnsDeletedNeedleData(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(94) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 551234, 0xCAFE1234) |
|||
payload := []byte("read-deleted-needle-payload") |
|||
|
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid)) |
|||
_ = framework.ReadAllAndClose(t, deleteResp) |
|||
if deleteResp.StatusCode != http.StatusAccepted { |
|||
t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode) |
|||
} |
|||
|
|||
normalRead := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) |
|||
_ = framework.ReadAllAndClose(t, normalRead) |
|||
if normalRead.StatusCode != http.StatusNotFound { |
|||
t.Fatalf("normal read after delete expected 404, got %d", normalRead.StatusCode) |
|||
} |
|||
|
|||
readDeletedReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true") |
|||
readDeletedResp := framework.DoRequest(t, client, readDeletedReq) |
|||
readDeletedBody := framework.ReadAllAndClose(t, readDeletedResp) |
|||
if readDeletedResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("read with readDeleted=true expected 200, got %d", readDeletedResp.StatusCode) |
|||
} |
|||
if string(readDeletedBody) != string(payload) { |
|||
t.Fatalf("readDeleted body mismatch: got %q want %q", string(readDeletedBody), string(payload)) |
|||
} |
|||
} |
|||
@ -0,0 +1,319 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
func TestReadModeProxyMissingLocalVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P1() |
|||
profile.ReadMode = "proxy" |
|||
clusterHarness := framework.StartDualVolumeCluster(t, profile) |
|||
|
|||
conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) |
|||
defer conn0.Close() |
|||
|
|||
const volumeID = uint32(101) |
|||
framework.AllocateVolume(t, grpc0, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 120001, 0x0102ABCD) |
|||
payload := []byte("proxy-read-mode-forwarded-content") |
|||
|
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid |
|||
var finalBody []byte |
|||
if !waitForHTTPStatus(t, client, readURL, http.StatusOK, 10*time.Second, func(resp *http.Response) { |
|||
finalBody = framework.ReadAllAndClose(t, resp) |
|||
}) { |
|||
t.Fatalf("proxy read mode did not return 200 from non-owning volume server within deadline") |
|||
} |
|||
if string(finalBody) != string(payload) { |
|||
t.Fatalf("proxy read mode body mismatch: got %q want %q", string(finalBody), string(payload)) |
|||
} |
|||
} |
|||
|
|||
func TestReadModeRedirectMissingLocalVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P1() |
|||
profile.ReadMode = "redirect" |
|||
clusterHarness := framework.StartDualVolumeCluster(t, profile) |
|||
|
|||
conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) |
|||
defer conn0.Close() |
|||
|
|||
const volumeID = uint32(102) |
|||
framework.AllocateVolume(t, grpc0, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 120002, 0x0102DCBA) |
|||
payload := []byte("redirect-read-mode-content") |
|||
|
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
noRedirectClient := &http.Client{ |
|||
Timeout: 10 * time.Second, |
|||
CheckRedirect: func(req *http.Request, via []*http.Request) error { |
|||
return http.ErrUseLastResponse |
|||
}, |
|||
} |
|||
|
|||
readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid |
|||
var redirectLocation string |
|||
if !waitForHTTPStatus(t, noRedirectClient, readURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) { |
|||
redirectLocation = resp.Header.Get("Location") |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
}) { |
|||
t.Fatalf("redirect read mode did not return 301 from non-owning volume server within deadline") |
|||
} |
|||
if redirectLocation == "" { |
|||
t.Fatalf("redirect response missing Location header") |
|||
} |
|||
if !strings.Contains(redirectLocation, "proxied=true") { |
|||
t.Fatalf("redirect Location should include proxied=true, got %q", redirectLocation) |
|||
} |
|||
|
|||
followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, redirectLocation)) |
|||
followBody := framework.ReadAllAndClose(t, followResp) |
|||
if followResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("following redirect expected 200, got %d", followResp.StatusCode) |
|||
} |
|||
if string(followBody) != string(payload) { |
|||
t.Fatalf("redirect-follow body mismatch: got %q want %q", string(followBody), string(payload)) |
|||
} |
|||
} |
|||
|
|||
func TestReadModeLocalMissingLocalVolumeReturnsNotFound(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P1() |
|||
profile.ReadMode = "local" |
|||
clusterHarness := framework.StartDualVolumeCluster(t, profile) |
|||
|
|||
conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) |
|||
defer conn0.Close() |
|||
|
|||
const volumeID = uint32(103) |
|||
framework.AllocateVolume(t, grpc0, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 120003, 0x0102BEEF) |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, []byte("local-read-mode-content")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
readResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(1), fid) |
|||
_ = framework.ReadAllAndClose(t, readResp) |
|||
if readResp.StatusCode != http.StatusNotFound { |
|||
t.Fatalf("local read mode expected 404 on non-owning server, got %d", readResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestReadDeletedProxyModeOnMissingLocalVolume(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P1() |
|||
profile.ReadMode = "proxy" |
|||
clusterHarness := framework.StartDualVolumeCluster(t, profile) |
|||
|
|||
conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) |
|||
defer conn0.Close() |
|||
|
|||
const volumeID = uint32(104) |
|||
framework.AllocateVolume(t, grpc0, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 120004, 0x0102CAFE) |
|||
payload := []byte("proxy-readDeleted-missing-local-content") |
|||
|
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL(0)+"/"+fid)) |
|||
_ = framework.ReadAllAndClose(t, deleteResp) |
|||
if deleteResp.StatusCode != http.StatusAccepted { |
|||
t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode) |
|||
} |
|||
|
|||
readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?readDeleted=true" |
|||
var proxiedBody []byte |
|||
if !waitForHTTPStatus(t, client, readURL, http.StatusOK, 10*time.Second, func(resp *http.Response) { |
|||
proxiedBody = framework.ReadAllAndClose(t, resp) |
|||
}) { |
|||
t.Fatalf("proxy readDeleted path did not return 200 from non-owning volume server within deadline") |
|||
} |
|||
if string(proxiedBody) != string(payload) { |
|||
t.Fatalf("proxy readDeleted body mismatch: got %q want %q", string(proxiedBody), string(payload)) |
|||
} |
|||
} |
|||
|
|||
func TestReadDeletedRedirectModeDropsQueryParameterParity(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P1() |
|||
profile.ReadMode = "redirect" |
|||
clusterHarness := framework.StartDualVolumeCluster(t, profile) |
|||
|
|||
conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) |
|||
defer conn0.Close() |
|||
|
|||
const volumeID = uint32(105) |
|||
framework.AllocateVolume(t, grpc0, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 120005, 0x0102FACE) |
|||
payload := []byte("redirect-readDeleted-query-drop-parity") |
|||
|
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL(0)+"/"+fid)) |
|||
_ = framework.ReadAllAndClose(t, deleteResp) |
|||
if deleteResp.StatusCode != http.StatusAccepted { |
|||
t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode) |
|||
} |
|||
|
|||
noRedirectClient := &http.Client{ |
|||
Timeout: 10 * time.Second, |
|||
CheckRedirect: func(req *http.Request, via []*http.Request) error { |
|||
return http.ErrUseLastResponse |
|||
}, |
|||
} |
|||
|
|||
redirectURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?readDeleted=true" |
|||
var location string |
|||
if !waitForHTTPStatus(t, noRedirectClient, redirectURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) { |
|||
location = resp.Header.Get("Location") |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
}) { |
|||
t.Fatalf("redirect readDeleted path did not return 301 from non-owning volume server within deadline") |
|||
} |
|||
if location == "" { |
|||
t.Fatalf("redirect readDeleted response missing Location header") |
|||
} |
|||
if !strings.Contains(location, "proxied=true") { |
|||
t.Fatalf("redirect readDeleted Location should include proxied=true, got %q", location) |
|||
} |
|||
if strings.Contains(location, "readDeleted=true") { |
|||
t.Fatalf("redirect readDeleted Location should reflect current query-drop behavior, got %q", location) |
|||
} |
|||
|
|||
followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, location)) |
|||
_ = framework.ReadAllAndClose(t, followResp) |
|||
if followResp.StatusCode != http.StatusNotFound { |
|||
t.Fatalf("redirect-follow without readDeleted query expected 404 for deleted needle, got %d", followResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestReadModeRedirectPreservesCollectionQuery(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P1() |
|||
profile.ReadMode = "redirect" |
|||
clusterHarness := framework.StartDualVolumeCluster(t, profile) |
|||
|
|||
conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) |
|||
defer conn0.Close() |
|||
|
|||
const volumeID = uint32(109) |
|||
const collection = "redirect-collection" |
|||
framework.AllocateVolume(t, grpc0, volumeID, collection) |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 120006, 0x0102F00D) |
|||
payload := []byte("redirect-collection-preserve-content") |
|||
|
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
noRedirectClient := &http.Client{ |
|||
Timeout: 10 * time.Second, |
|||
CheckRedirect: func(req *http.Request, via []*http.Request) error { |
|||
return http.ErrUseLastResponse |
|||
}, |
|||
} |
|||
|
|||
redirectURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?collection=" + collection |
|||
var location string |
|||
if !waitForHTTPStatus(t, noRedirectClient, redirectURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) { |
|||
location = resp.Header.Get("Location") |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
}) { |
|||
t.Fatalf("redirect collection path did not return 301 from non-owning volume server within deadline") |
|||
} |
|||
if location == "" { |
|||
t.Fatalf("redirect collection response missing Location header") |
|||
} |
|||
if !strings.Contains(location, "proxied=true") { |
|||
t.Fatalf("redirect collection Location should include proxied=true, got %q", location) |
|||
} |
|||
if !strings.Contains(location, "collection="+collection) { |
|||
t.Fatalf("redirect collection Location should preserve collection query, got %q", location) |
|||
} |
|||
|
|||
followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, location)) |
|||
followBody := framework.ReadAllAndClose(t, followResp) |
|||
if followResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("redirect-follow expected 200, got %d", followResp.StatusCode) |
|||
} |
|||
if string(followBody) != string(payload) { |
|||
t.Fatalf("redirect-follow body mismatch: got %q want %q", string(followBody), string(payload)) |
|||
} |
|||
} |
|||
|
|||
func waitForHTTPStatus(t testing.TB, client *http.Client, url string, expectedStatus int, timeout time.Duration, onMatch func(resp *http.Response)) bool { |
|||
t.Helper() |
|||
|
|||
deadline := time.Now().Add(timeout) |
|||
for time.Now().Before(deadline) { |
|||
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, url)) |
|||
if resp.StatusCode == expectedStatus { |
|||
onMatch(resp) |
|||
return true |
|||
} |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
time.Sleep(200 * time.Millisecond) |
|||
} |
|||
|
|||
return false |
|||
} |
|||
@ -0,0 +1,191 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"fmt" |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
func TestReadPathShapesAndIfModifiedSince(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(93) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fullFileID := framework.NewFileID(volumeID, 771234, 0xBEEFCACE) |
|||
uploadPayload := []byte("read-path-shape-content") |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fullFileID, uploadPayload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
parts := strings.SplitN(fullFileID, ",", 2) |
|||
if len(parts) != 2 { |
|||
t.Fatalf("unexpected file id format: %q", fullFileID) |
|||
} |
|||
fidOnly := parts[1] |
|||
|
|||
readByVidFid := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, fmt.Sprintf("%s/%d/%s", clusterHarness.VolumeAdminURL(), volumeID, fidOnly))) |
|||
readByVidFidBody := framework.ReadAllAndClose(t, readByVidFid) |
|||
if readByVidFid.StatusCode != http.StatusOK { |
|||
t.Fatalf("GET /{vid}/{fid} expected 200, got %d", readByVidFid.StatusCode) |
|||
} |
|||
if string(readByVidFidBody) != string(uploadPayload) { |
|||
t.Fatalf("GET /{vid}/{fid} body mismatch: got %q want %q", string(readByVidFidBody), string(uploadPayload)) |
|||
} |
|||
|
|||
readWithFilename := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, fmt.Sprintf("%s/%d/%s/%s", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "named.bin"))) |
|||
readWithFilenameBody := framework.ReadAllAndClose(t, readWithFilename) |
|||
if readWithFilename.StatusCode != http.StatusOK { |
|||
t.Fatalf("GET /{vid}/{fid}/{filename} expected 200, got %d", readWithFilename.StatusCode) |
|||
} |
|||
if string(readWithFilenameBody) != string(uploadPayload) { |
|||
t.Fatalf("GET /{vid}/{fid}/{filename} body mismatch: got %q want %q", string(readWithFilenameBody), string(uploadPayload)) |
|||
} |
|||
|
|||
lastModified := readWithFilename.Header.Get("Last-Modified") |
|||
if lastModified == "" { |
|||
t.Fatalf("expected Last-Modified header on read response") |
|||
} |
|||
|
|||
ifModifiedSinceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fullFileID) |
|||
ifModifiedSinceReq.Header.Set("If-Modified-Since", lastModified) |
|||
ifModifiedSinceResp := framework.DoRequest(t, client, ifModifiedSinceReq) |
|||
_ = framework.ReadAllAndClose(t, ifModifiedSinceResp) |
|||
if ifModifiedSinceResp.StatusCode != http.StatusNotModified { |
|||
t.Fatalf("If-Modified-Since expected 304, got %d", ifModifiedSinceResp.StatusCode) |
|||
} |
|||
|
|||
headIfModifiedSinceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fullFileID) |
|||
headIfModifiedSinceReq.Header.Set("If-Modified-Since", lastModified) |
|||
headIfModifiedSinceResp := framework.DoRequest(t, client, headIfModifiedSinceReq) |
|||
headIfModifiedSinceBody := framework.ReadAllAndClose(t, headIfModifiedSinceResp) |
|||
if headIfModifiedSinceResp.StatusCode != http.StatusNotModified { |
|||
t.Fatalf("HEAD If-Modified-Since expected 304, got %d", headIfModifiedSinceResp.StatusCode) |
|||
} |
|||
if len(headIfModifiedSinceBody) != 0 { |
|||
t.Fatalf("HEAD If-Modified-Since expected empty body, got %d bytes", len(headIfModifiedSinceBody)) |
|||
} |
|||
} |
|||
|
|||
func TestMalformedVidFidPathReturnsBadRequest(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
client := framework.NewHTTPClient() |
|||
|
|||
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/not-a-vid/not-a-fid")) |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
if resp.StatusCode != http.StatusBadRequest { |
|||
t.Fatalf("malformed /{vid}/{fid} expected 400, got %d", resp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestReadWrongCookieReturnsNotFound(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(95) |
|||
const needleID = uint64(771235) |
|||
const cookie = uint32(0xBEEFCACF) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, needleID, cookie) |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("read-cookie-mismatch-content")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
wrongCookieFid := framework.NewFileID(volumeID, needleID, cookie+1) |
|||
getResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), wrongCookieFid) |
|||
_ = framework.ReadAllAndClose(t, getResp) |
|||
if getResp.StatusCode != http.StatusNotFound { |
|||
t.Fatalf("GET with wrong cookie expected 404, got %d", getResp.StatusCode) |
|||
} |
|||
|
|||
headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+wrongCookieFid)) |
|||
headBody := framework.ReadAllAndClose(t, headResp) |
|||
if headResp.StatusCode != http.StatusNotFound { |
|||
t.Fatalf("HEAD with wrong cookie expected 404, got %d", headResp.StatusCode) |
|||
} |
|||
if len(headBody) != 0 { |
|||
t.Fatalf("HEAD wrong-cookie response body should be empty, got %d bytes", len(headBody)) |
|||
} |
|||
} |
|||
|
|||
func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(99) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 772002, 0x2B3C4D5E) |
|||
payload := []byte("conditional-precedence-content") |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
baselineResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) |
|||
_ = framework.ReadAllAndClose(t, baselineResp) |
|||
if baselineResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("baseline read expected 200, got %d", baselineResp.StatusCode) |
|||
} |
|||
lastModified := baselineResp.Header.Get("Last-Modified") |
|||
if lastModified == "" { |
|||
t.Fatalf("baseline read expected Last-Modified header") |
|||
} |
|||
|
|||
precedenceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
precedenceReq.Header.Set("If-Modified-Since", lastModified) |
|||
precedenceReq.Header.Set("If-None-Match", "\"definitely-different-etag\"") |
|||
precedenceResp := framework.DoRequest(t, client, precedenceReq) |
|||
precedenceBody := framework.ReadAllAndClose(t, precedenceResp) |
|||
if precedenceResp.StatusCode != http.StatusNotModified { |
|||
t.Fatalf("conditional precedence expected 304, got %d", precedenceResp.StatusCode) |
|||
} |
|||
if len(precedenceBody) != 0 { |
|||
t.Fatalf("conditional precedence expected empty body, got %d bytes", len(precedenceBody)) |
|||
} |
|||
|
|||
invalidIMSReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid) |
|||
invalidIMSReq.Header.Set("If-Modified-Since", "not-a-valid-http-date") |
|||
invalidIMSReq.Header.Set("If-None-Match", "\"definitely-different-etag\"") |
|||
invalidIMSResp := framework.DoRequest(t, client, invalidIMSReq) |
|||
invalidIMSBody := framework.ReadAllAndClose(t, invalidIMSResp) |
|||
if invalidIMSResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("invalid If-Modified-Since with mismatched etag expected 200, got %d", invalidIMSResp.StatusCode) |
|||
} |
|||
if string(invalidIMSBody) != string(payload) { |
|||
t.Fatalf("invalid If-Modified-Since fallback body mismatch: got %q want %q", string(invalidIMSBody), string(payload)) |
|||
} |
|||
} |
|||
@ -0,0 +1,123 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"net/http" |
|||
"strconv" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
func TestUploadReadRangeHeadDeleteRoundTrip(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(7) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
fid := framework.NewFileID(volumeID, 123456, 0xA1B2C3D4) |
|||
data := []byte("hello-volume-server-integration") |
|||
client := framework.NewHTTPClient() |
|||
|
|||
uploadResp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(), fid, data) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid) |
|||
getBody := framework.ReadAllAndClose(t, getResp) |
|||
if getResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("get status: expected 200, got %d", getResp.StatusCode) |
|||
} |
|||
if string(getBody) != string(data) { |
|||
t.Fatalf("get body mismatch: got %q want %q", string(getBody), string(data)) |
|||
} |
|||
etag := getResp.Header.Get("ETag") |
|||
if etag == "" { |
|||
t.Fatalf("expected ETag header from GET response") |
|||
} |
|||
|
|||
notModifiedReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid) |
|||
notModifiedReq.Header.Set("If-None-Match", etag) |
|||
notModifiedResp := framework.DoRequest(t, client, notModifiedReq) |
|||
_ = framework.ReadAllAndClose(t, notModifiedResp) |
|||
if notModifiedResp.StatusCode != http.StatusNotModified { |
|||
t.Fatalf("if-none-match expected 304, got %d", notModifiedResp.StatusCode) |
|||
} |
|||
|
|||
rangeReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid) |
|||
rangeReq.Header.Set("Range", "bytes=0-4") |
|||
rangeResp := framework.DoRequest(t, client, rangeReq) |
|||
rangeBody := framework.ReadAllAndClose(t, rangeResp) |
|||
if rangeResp.StatusCode != http.StatusPartialContent { |
|||
t.Fatalf("range status: expected 206, got %d", rangeResp.StatusCode) |
|||
} |
|||
if got, want := string(rangeBody), "hello"; got != want { |
|||
t.Fatalf("range body mismatch: got %q want %q", got, want) |
|||
} |
|||
|
|||
invalidRangeReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid) |
|||
invalidRangeReq.Header.Set("Range", "bytes=9999-10000") |
|||
invalidRangeResp := framework.DoRequest(t, client, invalidRangeReq) |
|||
_ = framework.ReadAllAndClose(t, invalidRangeResp) |
|||
if invalidRangeResp.StatusCode != http.StatusRequestedRangeNotSatisfiable { |
|||
t.Fatalf("invalid range expected 416, got %d", invalidRangeResp.StatusCode) |
|||
} |
|||
|
|||
headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, cluster.VolumeAdminURL()+"/"+fid)) |
|||
headBody := framework.ReadAllAndClose(t, headResp) |
|||
if headResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("head status: expected 200, got %d", headResp.StatusCode) |
|||
} |
|||
if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(data)) { |
|||
t.Fatalf("head content-length mismatch: got %q want %d", got, len(data)) |
|||
} |
|||
if len(headBody) != 0 { |
|||
t.Fatalf("head body should be empty, got %d bytes", len(headBody)) |
|||
} |
|||
|
|||
headNotModifiedReq := mustNewRequest(t, http.MethodHead, cluster.VolumeAdminURL()+"/"+fid) |
|||
headNotModifiedReq.Header.Set("If-None-Match", etag) |
|||
headNotModifiedResp := framework.DoRequest(t, client, headNotModifiedReq) |
|||
headNotModifiedBody := framework.ReadAllAndClose(t, headNotModifiedResp) |
|||
if headNotModifiedResp.StatusCode != http.StatusNotModified { |
|||
t.Fatalf("head if-none-match expected 304, got %d", headNotModifiedResp.StatusCode) |
|||
} |
|||
if len(headNotModifiedBody) != 0 { |
|||
t.Fatalf("head if-none-match body should be empty, got %d bytes", len(headNotModifiedBody)) |
|||
} |
|||
|
|||
deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, cluster.VolumeAdminURL()+"/"+fid)) |
|||
_ = framework.ReadAllAndClose(t, deleteResp) |
|||
if deleteResp.StatusCode != http.StatusAccepted { |
|||
t.Fatalf("delete status: expected 202, got %d", deleteResp.StatusCode) |
|||
} |
|||
|
|||
notFoundResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid) |
|||
_ = framework.ReadAllAndClose(t, notFoundResp) |
|||
if notFoundResp.StatusCode != http.StatusNotFound { |
|||
t.Fatalf("read after delete: expected 404, got %d", notFoundResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestInvalidReadPathReturnsBadRequest(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
client := framework.NewHTTPClient() |
|||
|
|||
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/invalid,needle")) |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
if resp.StatusCode != http.StatusBadRequest { |
|||
t.Fatalf("invalid read expected 400, got %d", resp.StatusCode) |
|||
} |
|||
} |
|||
@ -0,0 +1,730 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"bytes" |
|||
"context" |
|||
"io" |
|||
"net/http" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/needle" |
|||
) |
|||
|
|||
type pausableReader struct { |
|||
remaining int64 |
|||
pauseAfter int64 |
|||
paused bool |
|||
unblock <-chan struct{} |
|||
} |
|||
|
|||
func (r *pausableReader) Read(p []byte) (int, error) { |
|||
if r.remaining <= 0 { |
|||
return 0, io.EOF |
|||
} |
|||
if !r.paused && r.pauseAfter > 0 { |
|||
n := int64(len(p)) |
|||
if n > r.pauseAfter { |
|||
n = r.pauseAfter |
|||
} |
|||
for i := int64(0); i < n; i++ { |
|||
p[i] = 'a' |
|||
} |
|||
r.remaining -= n |
|||
r.pauseAfter -= n |
|||
if r.pauseAfter == 0 { |
|||
r.paused = true |
|||
} |
|||
return int(n), nil |
|||
} |
|||
if r.paused { |
|||
<-r.unblock |
|||
r.paused = false |
|||
} |
|||
n := int64(len(p)) |
|||
if n > r.remaining { |
|||
n = r.remaining |
|||
} |
|||
for i := int64(0); i < n; i++ { |
|||
p[i] = 'b' |
|||
} |
|||
r.remaining -= n |
|||
return int(n), nil |
|||
} |
|||
|
|||
func TestUploadLimitTimeoutAndReplicateBypass(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(98) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
const blockedUploadSize = 2 * 1024 * 1024 // over 1MB P8 upload limit
|
|||
|
|||
unblockFirstUpload := make(chan struct{}) |
|||
firstUploadDone := make(chan error, 1) |
|||
firstFID := framework.NewFileID(volumeID, 880001, 0x1A2B3C4D) |
|||
go func() { |
|||
req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{ |
|||
remaining: blockedUploadSize, |
|||
pauseAfter: 1, |
|||
unblock: unblockFirstUpload, |
|||
}) |
|||
if err != nil { |
|||
firstUploadDone <- err |
|||
return |
|||
} |
|||
req.Header.Set("Content-Type", "application/octet-stream") |
|||
req.ContentLength = blockedUploadSize |
|||
|
|||
resp, err := (&http.Client{}).Do(req) |
|||
if resp != nil { |
|||
_, _ = io.Copy(io.Discard, resp.Body) |
|||
_ = resp.Body.Close() |
|||
} |
|||
firstUploadDone <- err |
|||
}() |
|||
|
|||
// Give the first upload time to pass limit checks and block in body processing.
|
|||
time.Sleep(300 * time.Millisecond) |
|||
|
|||
replicateFID := framework.NewFileID(volumeID, 880002, 0x5E6F7A8B) |
|||
replicateReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+replicateFID+"?type=replicate", bytes.NewReader([]byte("replicate"))) |
|||
if err != nil { |
|||
t.Fatalf("create replicate request: %v", err) |
|||
} |
|||
replicateReq.Header.Set("Content-Type", "application/octet-stream") |
|||
replicateReq.ContentLength = int64(len("replicate")) |
|||
replicateResp, err := framework.NewHTTPClient().Do(replicateReq) |
|||
if err != nil { |
|||
t.Fatalf("replicate request failed: %v", err) |
|||
} |
|||
_ = framework.ReadAllAndClose(t, replicateResp) |
|||
if replicateResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("replicate request expected 201 bypassing limit, got %d", replicateResp.StatusCode) |
|||
} |
|||
|
|||
normalFID := framework.NewFileID(volumeID, 880003, 0x9C0D1E2F) |
|||
normalReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+normalFID, bytes.NewReader([]byte("normal"))) |
|||
if err != nil { |
|||
t.Fatalf("create normal request: %v", err) |
|||
} |
|||
normalReq.Header.Set("Content-Type", "application/octet-stream") |
|||
normalReq.ContentLength = int64(len("normal")) |
|||
|
|||
timeoutClient := &http.Client{Timeout: 10 * time.Second} |
|||
normalResp, err := timeoutClient.Do(normalReq) |
|||
if err != nil { |
|||
t.Fatalf("normal upload request failed: %v", err) |
|||
} |
|||
_ = framework.ReadAllAndClose(t, normalResp) |
|||
if normalResp.StatusCode != http.StatusTooManyRequests { |
|||
t.Fatalf("normal upload expected 429 while limit blocked, got %d", normalResp.StatusCode) |
|||
} |
|||
|
|||
close(unblockFirstUpload) |
|||
select { |
|||
case <-firstUploadDone: |
|||
case <-time.After(5 * time.Second): |
|||
t.Fatalf("timed out waiting for blocked upload to finish") |
|||
} |
|||
} |
|||
|
|||
func TestUploadLimitWaitThenProceed(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(111) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
const blockedUploadSize = 2 * 1024 * 1024 |
|||
|
|||
unblockFirstUpload := make(chan struct{}) |
|||
firstUploadDone := make(chan error, 1) |
|||
firstFID := framework.NewFileID(volumeID, 880601, 0x6A2B3C4D) |
|||
go func() { |
|||
req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{ |
|||
remaining: blockedUploadSize, |
|||
pauseAfter: 1, |
|||
unblock: unblockFirstUpload, |
|||
}) |
|||
if err != nil { |
|||
firstUploadDone <- err |
|||
return |
|||
} |
|||
req.Header.Set("Content-Type", "application/octet-stream") |
|||
req.ContentLength = blockedUploadSize |
|||
|
|||
resp, err := (&http.Client{}).Do(req) |
|||
if resp != nil { |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
} |
|||
firstUploadDone <- err |
|||
}() |
|||
|
|||
time.Sleep(300 * time.Millisecond) |
|||
|
|||
type uploadResult struct { |
|||
resp *http.Response |
|||
err error |
|||
} |
|||
secondUploadDone := make(chan uploadResult, 1) |
|||
secondFID := framework.NewFileID(volumeID, 880602, 0x6A2B3C4E) |
|||
go func() { |
|||
req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+secondFID, bytes.NewReader([]byte("wait-then-proceed"))) |
|||
if err != nil { |
|||
secondUploadDone <- uploadResult{err: err} |
|||
return |
|||
} |
|||
req.Header.Set("Content-Type", "application/octet-stream") |
|||
req.ContentLength = int64(len("wait-then-proceed")) |
|||
resp, err := (&http.Client{Timeout: 10 * time.Second}).Do(req) |
|||
secondUploadDone <- uploadResult{resp: resp, err: err} |
|||
}() |
|||
|
|||
time.Sleep(500 * time.Millisecond) |
|||
close(unblockFirstUpload) |
|||
|
|||
select { |
|||
case firstErr := <-firstUploadDone: |
|||
if firstErr != nil { |
|||
t.Fatalf("first blocked upload failed: %v", firstErr) |
|||
} |
|||
case <-time.After(5 * time.Second): |
|||
t.Fatalf("timed out waiting for first upload completion") |
|||
} |
|||
|
|||
select { |
|||
case result := <-secondUploadDone: |
|||
if result.err != nil { |
|||
t.Fatalf("second upload failed: %v", result.err) |
|||
} |
|||
_ = framework.ReadAllAndClose(t, result.resp) |
|||
if result.resp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("second upload expected 201 after waiting for slot, got %d", result.resp.StatusCode) |
|||
} |
|||
case <-time.After(5 * time.Second): |
|||
t.Fatalf("timed out waiting for second upload completion") |
|||
} |
|||
} |
|||
|
|||
func TestUploadLimitTimeoutThenRecovery(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(113) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
const blockedUploadSize = 2 * 1024 * 1024 |
|||
|
|||
unblockFirstUpload := make(chan struct{}) |
|||
firstUploadDone := make(chan error, 1) |
|||
firstFID := framework.NewFileID(volumeID, 880801, 0x7A2B3C4D) |
|||
go func() { |
|||
req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{ |
|||
remaining: blockedUploadSize, |
|||
pauseAfter: 1, |
|||
unblock: unblockFirstUpload, |
|||
}) |
|||
if err != nil { |
|||
firstUploadDone <- err |
|||
return |
|||
} |
|||
req.Header.Set("Content-Type", "application/octet-stream") |
|||
req.ContentLength = blockedUploadSize |
|||
resp, err := (&http.Client{}).Do(req) |
|||
if resp != nil { |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
} |
|||
firstUploadDone <- err |
|||
}() |
|||
|
|||
time.Sleep(300 * time.Millisecond) |
|||
|
|||
timeoutFID := framework.NewFileID(volumeID, 880802, 0x7A2B3C4E) |
|||
timeoutResp := framework.UploadBytes(t, &http.Client{Timeout: 10 * time.Second}, clusterHarness.VolumeAdminURL(), timeoutFID, []byte("should-timeout")) |
|||
_ = framework.ReadAllAndClose(t, timeoutResp) |
|||
if timeoutResp.StatusCode != http.StatusTooManyRequests { |
|||
t.Fatalf("second upload under blocked pressure expected 429, got %d", timeoutResp.StatusCode) |
|||
} |
|||
|
|||
close(unblockFirstUpload) |
|||
select { |
|||
case firstErr := <-firstUploadDone: |
|||
if firstErr != nil { |
|||
t.Fatalf("first blocked upload failed: %v", firstErr) |
|||
} |
|||
case <-time.After(5 * time.Second): |
|||
t.Fatalf("timed out waiting for first upload completion") |
|||
} |
|||
|
|||
recoveryFID := framework.NewFileID(volumeID, 880803, 0x7A2B3C4F) |
|||
recoveryResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), recoveryFID, []byte("recovered-upload")) |
|||
_ = framework.ReadAllAndClose(t, recoveryResp) |
|||
if recoveryResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("recovery upload expected 201, got %d", recoveryResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestDownloadLimitTimeoutReturnsTooManyRequests(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(99) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
largePayload := make([]byte, 12*1024*1024) // over 1MB P8 download limit
|
|||
for i := range largePayload { |
|||
largePayload[i] = byte(i % 251) |
|||
} |
|||
downloadFID := framework.NewFileID(volumeID, 880101, 0x10203040) |
|||
uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), downloadFID, largePayload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+downloadFID)) |
|||
if err != nil { |
|||
t.Fatalf("first GET failed: %v", err) |
|||
} |
|||
if firstResp.StatusCode != http.StatusOK { |
|||
_ = framework.ReadAllAndClose(t, firstResp) |
|||
t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) |
|||
} |
|||
defer firstResp.Body.Close() |
|||
|
|||
// Keep first response body unread so server write path stays in-flight.
|
|||
time.Sleep(300 * time.Millisecond) |
|||
|
|||
secondClient := &http.Client{Timeout: 10 * time.Second} |
|||
secondResp, err := secondClient.Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+downloadFID)) |
|||
if err != nil { |
|||
t.Fatalf("second GET failed: %v", err) |
|||
} |
|||
_ = framework.ReadAllAndClose(t, secondResp) |
|||
if secondResp.StatusCode != http.StatusTooManyRequests { |
|||
t.Fatalf("second GET expected 429 while first download holds limit, got %d", secondResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestDownloadLimitWaitThenProceedWithoutReplica(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(112) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
largePayload := make([]byte, 12*1024*1024) |
|||
for i := range largePayload { |
|||
largePayload[i] = byte(i % 251) |
|||
} |
|||
fid := framework.NewFileID(volumeID, 880701, 0x60708090) |
|||
uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) |
|||
if err != nil { |
|||
t.Fatalf("first GET failed: %v", err) |
|||
} |
|||
if firstResp.StatusCode != http.StatusOK { |
|||
_ = framework.ReadAllAndClose(t, firstResp) |
|||
t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) |
|||
} |
|||
|
|||
type readResult struct { |
|||
resp *http.Response |
|||
err error |
|||
} |
|||
secondReadDone := make(chan readResult, 1) |
|||
go func() { |
|||
resp, readErr := (&http.Client{Timeout: 10 * time.Second}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) |
|||
secondReadDone <- readResult{resp: resp, err: readErr} |
|||
}() |
|||
|
|||
time.Sleep(500 * time.Millisecond) |
|||
_ = firstResp.Body.Close() |
|||
|
|||
select { |
|||
case result := <-secondReadDone: |
|||
if result.err != nil { |
|||
t.Fatalf("second GET failed: %v", result.err) |
|||
} |
|||
secondBody := framework.ReadAllAndClose(t, result.resp) |
|||
if result.resp.StatusCode != http.StatusOK { |
|||
t.Fatalf("second GET expected 200 after waiting for slot, got %d", result.resp.StatusCode) |
|||
} |
|||
if len(secondBody) != len(largePayload) { |
|||
t.Fatalf("second GET body size mismatch: got %d want %d", len(secondBody), len(largePayload)) |
|||
} |
|||
case <-time.After(5 * time.Second): |
|||
t.Fatalf("timed out waiting for second GET completion") |
|||
} |
|||
} |
|||
|
|||
func TestDownloadLimitTimeoutThenRecovery(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(114) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
largePayload := make([]byte, 12*1024*1024) |
|||
for i := range largePayload { |
|||
largePayload[i] = byte(i % 251) |
|||
} |
|||
fid := framework.NewFileID(volumeID, 880901, 0x708090A0) |
|||
uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) |
|||
if err != nil { |
|||
t.Fatalf("first GET failed: %v", err) |
|||
} |
|||
if firstResp.StatusCode != http.StatusOK { |
|||
_ = framework.ReadAllAndClose(t, firstResp) |
|||
t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) |
|||
} |
|||
|
|||
time.Sleep(300 * time.Millisecond) |
|||
|
|||
timeoutResp := framework.ReadBytes(t, &http.Client{Timeout: 10 * time.Second}, clusterHarness.VolumeAdminURL(), fid) |
|||
_ = framework.ReadAllAndClose(t, timeoutResp) |
|||
if timeoutResp.StatusCode != http.StatusTooManyRequests { |
|||
t.Fatalf("second GET under blocked pressure expected 429, got %d", timeoutResp.StatusCode) |
|||
} |
|||
|
|||
_ = firstResp.Body.Close() |
|||
|
|||
recoveryResp := framework.ReadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid) |
|||
recoveryBody := framework.ReadAllAndClose(t, recoveryResp) |
|||
if recoveryResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("recovery GET expected 200, got %d", recoveryResp.StatusCode) |
|||
} |
|||
if len(recoveryBody) != len(largePayload) { |
|||
t.Fatalf("recovery GET body size mismatch: got %d want %d", len(recoveryBody), len(largePayload)) |
|||
} |
|||
} |
|||
|
|||
func TestDownloadLimitOverageProxiesToReplica(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P8() |
|||
profile.ReadMode = "proxy" |
|||
clusterHarness := framework.StartDualVolumeCluster(t, profile) |
|||
|
|||
conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) |
|||
defer conn0.Close() |
|||
conn1, grpc1 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) |
|||
defer conn1.Close() |
|||
|
|||
const volumeID = uint32(100) |
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
req := &volume_server_pb.AllocateVolumeRequest{ |
|||
VolumeId: volumeID, |
|||
Replication: "001", |
|||
Version: uint32(needle.GetCurrentVersion()), |
|||
} |
|||
if _, err := grpc0.AllocateVolume(ctx, req); err != nil { |
|||
t.Fatalf("allocate replicated volume on node0: %v", err) |
|||
} |
|||
if _, err := grpc1.AllocateVolume(ctx, req); err != nil { |
|||
t.Fatalf("allocate replicated volume on node1: %v", err) |
|||
} |
|||
|
|||
largePayload := make([]byte, 12*1024*1024) |
|||
for i := range largePayload { |
|||
largePayload[i] = byte(i % 251) |
|||
} |
|||
fid := framework.NewFileID(volumeID, 880201, 0x0A0B0C0D) |
|||
uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(0), fid, largePayload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("replicated large upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
replicaReadURL := clusterHarness.VolumeAdminURL(1) + "/" + fid |
|||
if !waitForHTTPStatus(t, framework.NewHTTPClient(), replicaReadURL, http.StatusOK, 10*time.Second, func(resp *http.Response) { |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
}) { |
|||
t.Fatalf("replica did not become readable within deadline") |
|||
} |
|||
|
|||
firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid)) |
|||
if err != nil { |
|||
t.Fatalf("first GET failed: %v", err) |
|||
} |
|||
if firstResp.StatusCode != http.StatusOK { |
|||
_ = framework.ReadAllAndClose(t, firstResp) |
|||
t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) |
|||
} |
|||
defer firstResp.Body.Close() |
|||
|
|||
time.Sleep(300 * time.Millisecond) |
|||
|
|||
secondResp, err := framework.NewHTTPClient().Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid)) |
|||
if err != nil { |
|||
t.Fatalf("second GET failed: %v", err) |
|||
} |
|||
secondBody := framework.ReadAllAndClose(t, secondResp) |
|||
if secondResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("second GET expected 200 via replica proxy fallback, got %d", secondResp.StatusCode) |
|||
} |
|||
if len(secondBody) != len(largePayload) { |
|||
t.Fatalf("second GET proxied body size mismatch: got %d want %d", len(secondBody), len(largePayload)) |
|||
} |
|||
} |
|||
|
|||
func TestDownloadLimitProxiedRequestSkipsReplicaFallbackAndTimesOut(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
profile := matrix.P8() |
|||
profile.ReadMode = "proxy" |
|||
clusterHarness := framework.StartDualVolumeCluster(t, profile) |
|||
|
|||
conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0)) |
|||
defer conn0.Close() |
|||
conn1, grpc1 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1)) |
|||
defer conn1.Close() |
|||
|
|||
const volumeID = uint32(106) |
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
|||
defer cancel() |
|||
req := &volume_server_pb.AllocateVolumeRequest{ |
|||
VolumeId: volumeID, |
|||
Replication: "001", |
|||
Version: uint32(needle.GetCurrentVersion()), |
|||
} |
|||
if _, err := grpc0.AllocateVolume(ctx, req); err != nil { |
|||
t.Fatalf("allocate replicated volume on node0: %v", err) |
|||
} |
|||
if _, err := grpc1.AllocateVolume(ctx, req); err != nil { |
|||
t.Fatalf("allocate replicated volume on node1: %v", err) |
|||
} |
|||
|
|||
largePayload := make([]byte, 12*1024*1024) |
|||
for i := range largePayload { |
|||
largePayload[i] = byte(i % 251) |
|||
} |
|||
fid := framework.NewFileID(volumeID, 880202, 0x0A0B0D0E) |
|||
uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(0), fid, largePayload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("replicated large upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
// Ensure replica path is actually available, so a non-proxied request would proxy.
|
|||
replicaReadURL := clusterHarness.VolumeAdminURL(1) + "/" + fid |
|||
if !waitForHTTPStatus(t, framework.NewHTTPClient(), replicaReadURL, http.StatusOK, 10*time.Second, func(resp *http.Response) { |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
}) { |
|||
t.Fatalf("replica did not become readable within deadline") |
|||
} |
|||
|
|||
firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid)) |
|||
if err != nil { |
|||
t.Fatalf("first GET failed: %v", err) |
|||
} |
|||
if firstResp.StatusCode != http.StatusOK { |
|||
_ = framework.ReadAllAndClose(t, firstResp) |
|||
t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) |
|||
} |
|||
defer firstResp.Body.Close() |
|||
|
|||
time.Sleep(300 * time.Millisecond) |
|||
|
|||
// proxied=true should bypass replica fallback and hit wait/timeout branch.
|
|||
secondResp, err := framework.NewHTTPClient().Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid+"?proxied=true")) |
|||
if err != nil { |
|||
t.Fatalf("second GET failed: %v", err) |
|||
} |
|||
_ = framework.ReadAllAndClose(t, secondResp) |
|||
if secondResp.StatusCode != http.StatusTooManyRequests { |
|||
t.Fatalf("second GET with proxied=true expected 429 timeout path, got %d", secondResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestUploadLimitDisabledAllowsConcurrentUploads(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(107) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
const blockedUploadSize = 2 * 1024 * 1024 |
|||
unblockFirstUpload := make(chan struct{}) |
|||
firstUploadDone := make(chan error, 1) |
|||
firstFID := framework.NewFileID(volumeID, 880301, 0x1A2B3C5D) |
|||
go func() { |
|||
req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{ |
|||
remaining: blockedUploadSize, |
|||
pauseAfter: 1, |
|||
unblock: unblockFirstUpload, |
|||
}) |
|||
if err != nil { |
|||
firstUploadDone <- err |
|||
return |
|||
} |
|||
req.Header.Set("Content-Type", "application/octet-stream") |
|||
req.ContentLength = blockedUploadSize |
|||
|
|||
resp, err := (&http.Client{}).Do(req) |
|||
if resp != nil { |
|||
_ = framework.ReadAllAndClose(t, resp) |
|||
} |
|||
firstUploadDone <- err |
|||
}() |
|||
|
|||
time.Sleep(300 * time.Millisecond) |
|||
|
|||
secondFID := framework.NewFileID(volumeID, 880302, 0x1A2B3C5E) |
|||
secondResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), secondFID, []byte("no-limit-second-upload")) |
|||
_ = framework.ReadAllAndClose(t, secondResp) |
|||
if secondResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("second upload with disabled limit expected 201, got %d", secondResp.StatusCode) |
|||
} |
|||
|
|||
close(unblockFirstUpload) |
|||
select { |
|||
case <-firstUploadDone: |
|||
case <-time.After(5 * time.Second): |
|||
t.Fatalf("timed out waiting for first upload completion") |
|||
} |
|||
} |
|||
|
|||
func TestDownloadLimitDisabledAllowsConcurrentDownloads(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(108) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
largePayload := make([]byte, 12*1024*1024) |
|||
for i := range largePayload { |
|||
largePayload[i] = byte(i % 251) |
|||
} |
|||
fid := framework.NewFileID(volumeID, 880401, 0x20304050) |
|||
uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) |
|||
if err != nil { |
|||
t.Fatalf("first GET failed: %v", err) |
|||
} |
|||
if firstResp.StatusCode != http.StatusOK { |
|||
_ = framework.ReadAllAndClose(t, firstResp) |
|||
t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) |
|||
} |
|||
defer firstResp.Body.Close() |
|||
|
|||
time.Sleep(300 * time.Millisecond) |
|||
|
|||
secondResp := framework.ReadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid) |
|||
secondBody := framework.ReadAllAndClose(t, secondResp) |
|||
if secondResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("second GET with disabled limit expected 200, got %d", secondResp.StatusCode) |
|||
} |
|||
if len(secondBody) != len(largePayload) { |
|||
t.Fatalf("second GET body size mismatch: got %d want %d", len(secondBody), len(largePayload)) |
|||
} |
|||
} |
|||
|
|||
func TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(110) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
largePayload := make([]byte, 12*1024*1024) |
|||
for i := range largePayload { |
|||
largePayload[i] = byte(i % 251) |
|||
} |
|||
fid := framework.NewFileID(volumeID, 880501, 0x50607080) |
|||
uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)) |
|||
if err != nil { |
|||
t.Fatalf("first GET failed: %v", err) |
|||
} |
|||
if firstResp.StatusCode != http.StatusOK { |
|||
_ = framework.ReadAllAndClose(t, firstResp) |
|||
t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode) |
|||
} |
|||
defer firstResp.Body.Close() |
|||
|
|||
time.Sleep(300 * time.Millisecond) |
|||
|
|||
invalidReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/not-a-vid,1234567890ab") |
|||
invalidResp := framework.DoRequest(t, framework.NewHTTPClient(), invalidReq) |
|||
_ = framework.ReadAllAndClose(t, invalidResp) |
|||
if invalidResp.StatusCode != http.StatusBadRequest { |
|||
t.Fatalf("invalid vid while over limit expected 400, got %d", invalidResp.StatusCode) |
|||
} |
|||
} |
|||
@ -0,0 +1,118 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
"net/http" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
func TestWriteUnchangedAndDeleteEdgeVariants(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(87) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
const key = uint64(999001) |
|||
const cookie = uint32(0xDEADBEEF) |
|||
fid := framework.NewFileID(volumeID, key, cookie) |
|||
client := framework.NewHTTPClient() |
|||
payload := []byte("unchanged-write-content") |
|||
|
|||
firstUpload := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
firstUploadResp := framework.DoRequest(t, client, firstUpload) |
|||
_ = framework.ReadAllAndClose(t, firstUploadResp) |
|||
if firstUploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("first upload expected 201, got %d", firstUploadResp.StatusCode) |
|||
} |
|||
|
|||
secondUpload := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload) |
|||
secondUploadResp := framework.DoRequest(t, client, secondUpload) |
|||
_ = framework.ReadAllAndClose(t, secondUploadResp) |
|||
if secondUploadResp.StatusCode != http.StatusNoContent { |
|||
t.Fatalf("second unchanged upload expected 204, got %d", secondUploadResp.StatusCode) |
|||
} |
|||
if secondUploadResp.Header.Get("ETag") == "" { |
|||
t.Fatalf("second unchanged upload expected ETag header") |
|||
} |
|||
|
|||
wrongCookieFid := framework.NewFileID(volumeID, key, cookie+1) |
|||
wrongCookieDelete := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+wrongCookieFid)) |
|||
_ = framework.ReadAllAndClose(t, wrongCookieDelete) |
|||
if wrongCookieDelete.StatusCode != http.StatusBadRequest { |
|||
t.Fatalf("delete with mismatched cookie expected 400, got %d", wrongCookieDelete.StatusCode) |
|||
} |
|||
|
|||
missingDelete := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+framework.NewFileID(volumeID, key+1, cookie))) |
|||
missingDeleteBody := framework.ReadAllAndClose(t, missingDelete) |
|||
if missingDelete.StatusCode != http.StatusNotFound { |
|||
t.Fatalf("delete missing needle expected 404, got %d", missingDelete.StatusCode) |
|||
} |
|||
|
|||
var payloadMap map[string]int64 |
|||
if err := json.Unmarshal(missingDeleteBody, &payloadMap); err != nil { |
|||
t.Fatalf("decode delete missing response: %v", err) |
|||
} |
|||
if payloadMap["size"] != 0 { |
|||
t.Fatalf("delete missing needle expected size=0, got %d", payloadMap["size"]) |
|||
} |
|||
} |
|||
|
|||
func TestDeleteTimestampOverrideKeepsReadDeletedLastModifiedParity(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(88) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 999002, 0xABCD1234) |
|||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("delete-ts-override")) |
|||
_ = framework.ReadAllAndClose(t, uploadResp) |
|||
if uploadResp.StatusCode != http.StatusCreated { |
|||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) |
|||
} |
|||
|
|||
beforeDeleteResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid) |
|||
_ = framework.ReadAllAndClose(t, beforeDeleteResp) |
|||
if beforeDeleteResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("pre-delete read expected 200, got %d", beforeDeleteResp.StatusCode) |
|||
} |
|||
lastModifiedBeforeDelete := beforeDeleteResp.Header.Get("Last-Modified") |
|||
if lastModifiedBeforeDelete == "" { |
|||
t.Fatalf("expected Last-Modified before delete") |
|||
} |
|||
|
|||
deleteReq := mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid+"?ts=1700000000") |
|||
deleteResp := framework.DoRequest(t, client, deleteReq) |
|||
_ = framework.ReadAllAndClose(t, deleteResp) |
|||
if deleteResp.StatusCode != http.StatusAccepted { |
|||
t.Fatalf("delete with ts override expected 202, got %d", deleteResp.StatusCode) |
|||
} |
|||
|
|||
readDeletedResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true")) |
|||
_ = framework.ReadAllAndClose(t, readDeletedResp) |
|||
if readDeletedResp.StatusCode != http.StatusOK { |
|||
t.Fatalf("readDeleted after ts override expected 200, got %d", readDeletedResp.StatusCode) |
|||
} |
|||
lastModified := readDeletedResp.Header.Get("Last-Modified") |
|||
if lastModified == "" { |
|||
t.Fatalf("expected Last-Modified header on readDeleted response") |
|||
} |
|||
if lastModified != lastModifiedBeforeDelete { |
|||
t.Fatalf("expected readDeleted Last-Modified parity with pre-delete header, got %q want %q", lastModified, lastModifiedBeforeDelete) |
|||
} |
|||
} |
|||
@ -0,0 +1,74 @@ |
|||
package volume_server_http_test |
|||
|
|||
import ( |
|||
"net/http" |
|||
"strings" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework" |
|||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix" |
|||
) |
|||
|
|||
func TestWriteInvalidVidAndFidReturnBadRequest(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
client := framework.NewHTTPClient() |
|||
|
|||
invalidVidReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/invalid,12345678", []byte("x")) |
|||
invalidVidResp := framework.DoRequest(t, client, invalidVidReq) |
|||
_ = framework.ReadAllAndClose(t, invalidVidResp) |
|||
if invalidVidResp.StatusCode != http.StatusBadRequest { |
|||
t.Fatalf("write with invalid vid expected 400, got %d", invalidVidResp.StatusCode) |
|||
} |
|||
|
|||
invalidFidReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/1,bad", []byte("x")) |
|||
invalidFidResp := framework.DoRequest(t, client, invalidFidReq) |
|||
_ = framework.ReadAllAndClose(t, invalidFidResp) |
|||
if invalidFidResp.StatusCode != http.StatusBadRequest { |
|||
t.Fatalf("write with invalid fid expected 400, got %d", invalidFidResp.StatusCode) |
|||
} |
|||
} |
|||
|
|||
func TestWriteMalformedMultipartAndMD5Mismatch(t *testing.T) { |
|||
if testing.Short() { |
|||
t.Skip("skipping integration test in short mode") |
|||
} |
|||
|
|||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1()) |
|||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) |
|||
defer conn.Close() |
|||
|
|||
const volumeID = uint32(98) |
|||
framework.AllocateVolume(t, grpcClient, volumeID, "") |
|||
|
|||
client := framework.NewHTTPClient() |
|||
fid := framework.NewFileID(volumeID, 772001, 0x1A2B3C4D) |
|||
|
|||
malformedMultipartReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+fid, strings.NewReader("not-a-valid-multipart-body")) |
|||
if err != nil { |
|||
t.Fatalf("create malformed multipart request: %v", err) |
|||
} |
|||
malformedMultipartReq.Header.Set("Content-Type", "multipart/form-data") |
|||
malformedMultipartResp := framework.DoRequest(t, client, malformedMultipartReq) |
|||
malformedMultipartBody := framework.ReadAllAndClose(t, malformedMultipartResp) |
|||
if malformedMultipartResp.StatusCode != http.StatusBadRequest { |
|||
t.Fatalf("malformed multipart write expected 400, got %d", malformedMultipartResp.StatusCode) |
|||
} |
|||
if !strings.Contains(strings.ToLower(string(malformedMultipartBody)), "boundary") { |
|||
t.Fatalf("malformed multipart response should mention boundary parse failure, got %q", string(malformedMultipartBody)) |
|||
} |
|||
|
|||
md5MismatchReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, []byte("content-md5-mismatch-body")) |
|||
md5MismatchReq.Header.Set("Content-MD5", "AAAAAAAAAAAAAAAAAAAAAA==") |
|||
md5MismatchResp := framework.DoRequest(t, client, md5MismatchReq) |
|||
md5MismatchBody := framework.ReadAllAndClose(t, md5MismatchResp) |
|||
if md5MismatchResp.StatusCode != http.StatusBadRequest { |
|||
t.Fatalf("content-md5 mismatch write expected 400, got %d", md5MismatchResp.StatusCode) |
|||
} |
|||
if !strings.Contains(string(md5MismatchBody), "Content-MD5") { |
|||
t.Fatalf("content-md5 mismatch response should mention Content-MD5, got %q", string(md5MismatchBody)) |
|||
} |
|||
} |
|||
@ -0,0 +1,63 @@ |
|||
package matrix |
|||
|
|||
import "time" |
|||
|
|||
// Profile describes one runtime test matrix configuration.
|
|||
type Profile struct { |
|||
Name string |
|||
|
|||
ReadMode string |
|||
SplitPublicPort bool |
|||
|
|||
EnableJWT bool |
|||
JWTSigningKey string |
|||
JWTReadKey string |
|||
EnableMaintain bool |
|||
|
|||
ConcurrentUploadLimitMB int |
|||
ConcurrentDownloadLimitMB int |
|||
InflightUploadTimeout time.Duration |
|||
InflightDownloadTimeout time.Duration |
|||
|
|||
ReplicatedLayout bool |
|||
HasErasureCoding bool |
|||
HasRemoteTier bool |
|||
} |
|||
|
|||
// P1 is the baseline profile: one volume server, no JWT, proxy read mode.
|
|||
func P1() Profile { |
|||
return Profile{ |
|||
Name: "P1", |
|||
ReadMode: "proxy", |
|||
SplitPublicPort: false, |
|||
} |
|||
} |
|||
|
|||
// P2 uses split public/admin ports to verify public read-only behavior.
|
|||
func P2() Profile { |
|||
p := P1() |
|||
p.Name = "P2" |
|||
p.SplitPublicPort = true |
|||
return p |
|||
} |
|||
|
|||
// P3 enables JWT verification for read/write flows.
|
|||
func P3() Profile { |
|||
p := P1() |
|||
p.Name = "P3" |
|||
p.EnableJWT = true |
|||
p.JWTSigningKey = "volume-server-write-key" |
|||
p.JWTReadKey = "volume-server-read-key" |
|||
return p |
|||
} |
|||
|
|||
// P8 enables upload/download throttling branches.
|
|||
func P8() Profile { |
|||
p := P1() |
|||
p.Name = "P8" |
|||
p.ConcurrentUploadLimitMB = 1 |
|||
p.ConcurrentDownloadLimitMB = 1 |
|||
p.InflightUploadTimeout = 2 * time.Second |
|||
p.InflightDownloadTimeout = 2 * time.Second |
|||
return p |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue