From e0d74fffea14cec0e23160c08dbe64dbf4a94531 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 13:01:35 -0800 Subject: [PATCH] Improve TUS integration test setup Add comprehensive Makefile for TUS tests with targets: - test-with-server: Run tests with automatic server management - test-basic/chunked/resume/errors: Specific test categories - manual-start/stop: For development testing - debug-logs/status: For debugging - ci-test: For CI/CD pipelines Update README.md with: - Detailed TUS protocol documentation - All endpoint descriptions with headers - Usage examples with curl commands - Architecture diagram - Comparison with S3 multipart uploads Follows the pattern established by other tests in test/ folder. --- test/tus/Makefile | 226 ++++++++++++++++++++++++++++++++++++++++++ test/tus/README.md | 241 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 427 insertions(+), 40 deletions(-) create mode 100644 test/tus/Makefile diff --git a/test/tus/Makefile b/test/tus/Makefile new file mode 100644 index 000000000..fd0b6d6b7 --- /dev/null +++ b/test/tus/Makefile @@ -0,0 +1,226 @@ +# Makefile for TUS Protocol Integration Tests +# This Makefile provides targets for running TUS (resumable upload) integration tests + +# Default values +SEAWEEDFS_BINARY ?= weed +FILER_PORT ?= 18888 +VOLUME_PORT ?= 18080 +MASTER_PORT ?= 19333 +TEST_TIMEOUT ?= 10m +VOLUME_MAX_SIZE_MB ?= 50 +VOLUME_MAX_COUNT ?= 100 + +# Test directory +TEST_DIR := $(shell pwd) +SEAWEEDFS_ROOT := $(shell cd ../.. && pwd) + +# Colors for output +RED := \033[0;31m +GREEN := \033[0;32m +YELLOW := \033[1;33m +NC := \033[0m # No Color + +.PHONY: all test clean start-seaweedfs stop-seaweedfs check-binary build-weed help test-basic test-chunked test-resume test-errors test-with-server + +all: test + +# Build SeaweedFS binary +build-weed: + @echo "Building SeaweedFS binary..." + @cd $(SEAWEEDFS_ROOT)/weed && go build -o weed + @echo "$(GREEN)SeaweedFS binary built successfully$(NC)" + +help: + @echo "SeaweedFS TUS Protocol Integration Tests" + @echo "" + @echo "Available targets:" + @echo " test - Run all TUS integration tests" + @echo " test-basic - Run basic TUS upload tests" + @echo " test-chunked - Run chunked upload tests" + @echo " test-resume - Run upload resume tests" + @echo " test-errors - Run error handling tests" + @echo " test-with-server - Run tests with automatic server management" + @echo " start-seaweedfs - Start SeaweedFS server for testing" + @echo " stop-seaweedfs - Stop SeaweedFS server" + @echo " clean - Clean up test artifacts" + @echo " check-binary - Check if SeaweedFS binary exists" + @echo " build-weed - Build SeaweedFS binary" + @echo "" + @echo "Configuration:" + @echo " SEAWEEDFS_BINARY=$(SEAWEEDFS_BINARY)" + @echo " FILER_PORT=$(FILER_PORT)" + @echo " VOLUME_PORT=$(VOLUME_PORT)" + @echo " MASTER_PORT=$(MASTER_PORT)" + @echo " TEST_TIMEOUT=$(TEST_TIMEOUT)" + +check-binary: + @if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1 && [ ! -f "$(SEAWEEDFS_ROOT)/weed/weed" ]; then \ + echo "$(RED)Error: SeaweedFS binary not found$(NC)"; \ + echo "Please build SeaweedFS first: make build-weed"; \ + exit 1; \ + fi + @echo "$(GREEN)SeaweedFS binary found$(NC)" + +start-seaweedfs: check-binary + @echo "$(YELLOW)Starting SeaweedFS server for TUS testing...$(NC)" + @# Clean up any existing processes on our test ports + @lsof -ti :$(MASTER_PORT) | xargs -r kill -TERM 2>/dev/null || true + @lsof -ti :$(VOLUME_PORT) | xargs -r kill -TERM 2>/dev/null || true + @lsof -ti :$(FILER_PORT) | xargs -r kill -TERM 2>/dev/null || true + @sleep 2 + + # Create necessary directories + @mkdir -p /tmp/seaweedfs-test-tus-master + @mkdir -p /tmp/seaweedfs-test-tus-volume + @mkdir -p /tmp/seaweedfs-test-tus-filer + + # Determine binary path + $(eval WEED_BIN := $(shell command -v $(SEAWEEDFS_BINARY) 2>/dev/null || echo "$(SEAWEEDFS_ROOT)/weed/weed")) + + # Start master server + @echo "Starting master server..." + @nohup $(WEED_BIN) master \ + -port=$(MASTER_PORT) \ + -mdir=/tmp/seaweedfs-test-tus-master \ + -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) \ + -ip=127.0.0.1 \ + > /tmp/seaweedfs-tus-master.log 2>&1 & + @sleep 3 + + # Start volume server + @echo "Starting volume server..." + @nohup $(WEED_BIN) volume \ + -port=$(VOLUME_PORT) \ + -mserver=127.0.0.1:$(MASTER_PORT) \ + -dir=/tmp/seaweedfs-test-tus-volume \ + -max=$(VOLUME_MAX_COUNT) \ + -ip=127.0.0.1 \ + > /tmp/seaweedfs-tus-volume.log 2>&1 & + @sleep 3 + + # Start filer server + @echo "Starting filer server..." + @nohup $(WEED_BIN) filer \ + -port=$(FILER_PORT) \ + -master=127.0.0.1:$(MASTER_PORT) \ + -ip=127.0.0.1 \ + > /tmp/seaweedfs-tus-filer.log 2>&1 & + @sleep 5 + + # Wait for filer to be ready + @echo "$(YELLOW)Waiting for filer to be ready...$(NC)" + @for i in $$(seq 1 30); do \ + if curl -s -f http://127.0.0.1:$(FILER_PORT)/ > /dev/null 2>&1; then \ + echo "$(GREEN)Filer is ready$(NC)"; \ + break; \ + fi; \ + if [ $$i -eq 30 ]; then \ + echo "$(RED)Filer failed to start within 30 seconds$(NC)"; \ + $(MAKE) debug-logs; \ + exit 1; \ + fi; \ + echo "Waiting for filer... ($$i/30)"; \ + sleep 1; \ + done + + @echo "$(GREEN)SeaweedFS server started successfully for TUS testing$(NC)" + @echo "Master: http://localhost:$(MASTER_PORT)" + @echo "Volume: http://localhost:$(VOLUME_PORT)" + @echo "Filer: http://localhost:$(FILER_PORT)" + @echo "TUS Endpoint: http://localhost:$(FILER_PORT)/.tus/" + +stop-seaweedfs: + @echo "$(YELLOW)Stopping SeaweedFS server...$(NC)" + @lsof -ti :$(MASTER_PORT) | xargs -r kill -TERM 2>/dev/null || true + @lsof -ti :$(VOLUME_PORT) | xargs -r kill -TERM 2>/dev/null || true + @lsof -ti :$(FILER_PORT) | xargs -r kill -TERM 2>/dev/null || true + @sleep 2 + @echo "$(GREEN)SeaweedFS server stopped$(NC)" + +clean: + @echo "$(YELLOW)Cleaning up TUS test artifacts...$(NC)" + @rm -rf /tmp/seaweedfs-test-tus-* + @rm -f /tmp/seaweedfs-tus-*.log + @echo "$(GREEN)TUS test cleanup completed$(NC)" + +# Run all tests +test: check-binary + @echo "$(YELLOW)Running all TUS integration tests...$(NC)" + @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/tus/... + @echo "$(GREEN)All TUS tests completed$(NC)" + +# Run basic upload tests +test-basic: check-binary + @echo "$(YELLOW)Running basic TUS upload tests...$(NC)" + @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusBasicUpload|TestTusOptionsHandler" ./test/tus/... + @echo "$(GREEN)Basic TUS tests completed$(NC)" + +# Run chunked upload tests +test-chunked: check-binary + @echo "$(YELLOW)Running chunked TUS upload tests...$(NC)" + @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusChunkedUpload" ./test/tus/... + @echo "$(GREEN)Chunked TUS tests completed$(NC)" + +# Run resume tests +test-resume: check-binary + @echo "$(YELLOW)Running TUS upload resume tests...$(NC)" + @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusResumeAfterInterruption|TestTusHeadRequest" ./test/tus/... + @echo "$(GREEN)TUS resume tests completed$(NC)" + +# Run error handling tests +test-errors: check-binary + @echo "$(YELLOW)Running TUS error handling tests...$(NC)" + @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusInvalidOffset|TestTusUploadNotFound|TestTusDeleteUpload" ./test/tus/... + @echo "$(GREEN)TUS error tests completed$(NC)" + +# Run tests with automatic server management +test-with-server: build-weed + @echo "$(YELLOW)Running TUS tests with automatic server management...$(NC)" + @trap '$(MAKE) stop-seaweedfs clean || true' EXIT; \ + $(MAKE) start-seaweedfs && \ + sleep 3 && \ + cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/tus/... && \ + echo "$(GREEN)All TUS tests passed!$(NC)" + +# Debug targets +debug-logs: + @echo "$(YELLOW)=== Master Log ===$(NC)" + @tail -n 50 /tmp/seaweedfs-tus-master.log 2>/dev/null || echo "No master log found" + @echo "$(YELLOW)=== Volume Log ===$(NC)" + @tail -n 50 /tmp/seaweedfs-tus-volume.log 2>/dev/null || echo "No volume log found" + @echo "$(YELLOW)=== Filer Log ===$(NC)" + @tail -n 50 /tmp/seaweedfs-tus-filer.log 2>/dev/null || echo "No filer log found" + +debug-status: + @echo "$(YELLOW)=== Process Status ===$(NC)" + @ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found" + @echo "$(YELLOW)=== Port Status ===$(NC)" + @lsof -i :$(MASTER_PORT) -i :$(VOLUME_PORT) -i :$(FILER_PORT) 2>/dev/null || echo "No ports in use" + +# Manual testing targets +manual-start: start-seaweedfs + @echo "$(GREEN)SeaweedFS is now running for manual TUS testing$(NC)" + @echo "" + @echo "TUS Endpoints:" + @echo " OPTIONS /.tus/ - Capability discovery" + @echo " POST /.tus/{path} - Create upload" + @echo " HEAD /.tus/.uploads/{id} - Get offset" + @echo " PATCH /.tus/.uploads/{id} - Upload data" + @echo " DELETE /.tus/.uploads/{id} - Cancel upload" + @echo "" + @echo "Example curl commands:" + @echo " curl -X OPTIONS http://localhost:$(FILER_PORT)/.tus/ -H 'Tus-Resumable: 1.0.0'" + @echo "" + @echo "Run 'make manual-stop' when finished" + +manual-stop: stop-seaweedfs clean + +# CI targets +ci-test: test-with-server + +# Skip integration tests (short mode) +test-short: + @echo "$(YELLOW)Running TUS tests in short mode (skipping integration tests)...$(NC)" + @cd $(SEAWEEDFS_ROOT) && go test -v -short ./test/tus/... + @echo "$(GREEN)Short tests completed$(NC)" + diff --git a/test/tus/README.md b/test/tus/README.md index 7477168d0..e8913d82f 100644 --- a/test/tus/README.md +++ b/test/tus/README.md @@ -2,64 +2,225 @@ This directory contains integration tests for the TUS (resumable upload) protocol support in SeaweedFS Filer. -## TUS Protocol Overview +## Overview -TUS is an open protocol for resumable file uploads. It allows clients to upload files in chunks and resume uploads after network failures. +TUS is an open protocol for resumable file uploads over HTTP. It allows clients to upload files in chunks and resume uploads after network failures or interruptions. -Key endpoints: -- `OPTIONS /tus/` - Server capability discovery -- `POST /tus/{path}` - Create new upload session -- `HEAD /tus/{upload-id}` - Get current upload offset -- `PATCH /tus/{upload-id}` - Upload data at offset -- `DELETE /tus/{upload-id}` - Cancel upload +### Why TUS? -## Prerequisites +- **Resumable uploads**: Resume interrupted uploads without re-sending data +- **Chunked uploads**: Upload large files in smaller pieces +- **Simple protocol**: Standard HTTP methods with custom headers +- **Wide client support**: Libraries available for JavaScript, Python, Go, and more -1. Build the weed binary: +## TUS Protocol Endpoints + +| Method | Path | Description | +|--------|------|-------------| +| `OPTIONS` | `/.tus/` | Server capability discovery | +| `POST` | `/.tus/{path}` | Create new upload session | +| `HEAD` | `/.tus/.uploads/{id}` | Get current upload offset | +| `PATCH` | `/.tus/.uploads/{id}` | Upload data at offset | +| `DELETE` | `/.tus/.uploads/{id}` | Cancel upload | + +### TUS Headers + +**Request Headers:** +- `Tus-Resumable: 1.0.0` - Protocol version (required) +- `Upload-Length` - Total file size in bytes (required on POST) +- `Upload-Offset` - Current byte offset (required on PATCH) +- `Upload-Metadata` - Base64-encoded key-value pairs (optional) +- `Content-Type: application/offset+octet-stream` (required on PATCH) + +**Response Headers:** +- `Tus-Resumable` - Protocol version +- `Tus-Version` - Supported versions +- `Tus-Extension` - Supported extensions +- `Tus-Max-Size` - Maximum upload size +- `Upload-Offset` - Current byte offset +- `Location` - Upload URL (on POST) + +## Test Structure + +### Integration Tests + +The tests cover: + +1. **Basic Functionality** + - `TestTusOptionsHandler` - Capability discovery + - `TestTusBasicUpload` - Simple complete upload + - `TestTusCreationWithUpload` - Creation-with-upload extension + +2. **Chunked Uploads** + - `TestTusChunkedUpload` - Upload in multiple chunks + +3. **Resumable Uploads** + - `TestTusHeadRequest` - Offset tracking + - `TestTusResumeAfterInterruption` - Resume after failure + +4. **Error Handling** + - `TestTusInvalidOffset` - Offset mismatch (409 Conflict) + - `TestTusUploadNotFound` - Missing upload (404 Not Found) + - `TestTusDeleteUpload` - Upload cancellation + +## Running Tests + +### Prerequisites + +1. **Build SeaweedFS**: ```bash -cd ../../weed -go build +make build-weed +# or +cd ../../weed && go build -o weed ``` -2. The tests will automatically start required servers (master, volume, filer). +### Using Makefile -## Running Tests +```bash +# Show available targets +make help + +# Run all tests with automatic server management +make test-with-server + +# Run all tests (requires running server) +make test + +# Run specific test categories +make test-basic # Basic upload tests +make test-chunked # Chunked upload tests +make test-resume # Resume/HEAD tests +make test-errors # Error handling tests + +# Manual testing +make manual-start # Start SeaweedFS for manual testing +make manual-stop # Stop and cleanup +``` + +### Using Go Test Directly -### Run all TUS tests: ```bash +# Run all TUS tests go test -v ./test/tus/... + +# Run specific test +go test -v ./test/tus -run TestTusBasicUpload + +# Skip integration tests (short mode) +go test -v -short ./test/tus/... ``` -### Run specific test: +### Debug + ```bash -go test -v ./test/tus -run TestTusBasicUpload +# View server logs +make debug-logs + +# Check process and port status +make debug-status ``` -### Skip integration tests (short mode): +## Test Environment + +Each test run: +1. Starts a SeaweedFS cluster (master, volume, filer) +2. Creates uploads using TUS protocol +3. Verifies files are stored correctly +4. Cleans up test data + +### Default Ports + +| Service | Port | +|---------|------| +| Master | 19333 | +| Volume | 18080 | +| Filer | 18888 | + +### Configuration + +Override defaults via environment or Makefile variables: ```bash -go test -v -short ./test/tus/... +FILER_PORT=8889 MASTER_PORT=9334 make test ``` -## Test Coverage +## Example Usage -The tests cover: -- Basic upload creation and completion -- Chunked/resumable uploads -- Upload offset tracking (HEAD requests) -- Upload cancellation (DELETE requests) -- Error handling (invalid offsets, missing uploads) -- Large file uploads with multiple chunks -- Concurrent uploads -- Metadata handling - -## TUS Protocol Headers - -Required headers for TUS requests: -- `Tus-Resumable: 1.0.0` - Protocol version -- `Upload-Length` - Total file size (on creation) -- `Upload-Offset` - Current byte offset (on PATCH) -- `Content-Type: application/offset+octet-stream` (on PATCH) - -Optional headers: -- `Upload-Metadata` - Base64-encoded key-value pairs +### Create Upload + +```bash +curl -X POST http://localhost:18888/.tus/mydir/file.txt \ + -H "Tus-Resumable: 1.0.0" \ + -H "Upload-Length: 1000" \ + -H "Upload-Metadata: filename dGVzdC50eHQ=" +``` + +### Upload Data + +```bash +curl -X PATCH http://localhost:18888/.tus/.uploads/{upload-id} \ + -H "Tus-Resumable: 1.0.0" \ + -H "Upload-Offset: 0" \ + -H "Content-Type: application/offset+octet-stream" \ + --data-binary @file.txt +``` + +### Check Offset + +```bash +curl -I http://localhost:18888/.tus/.uploads/{upload-id} \ + -H "Tus-Resumable: 1.0.0" +``` + +### Cancel Upload + +```bash +curl -X DELETE http://localhost:18888/.tus/.uploads/{upload-id} \ + -H "Tus-Resumable: 1.0.0" +``` + +## TUS Extensions Supported + +- **creation**: Create new uploads with POST +- **creation-with-upload**: Send data in creation request +- **termination**: Cancel uploads with DELETE + +## Architecture + +``` +Client Filer Volume Servers + | | | + |-- POST /.tus/path/file.mp4 ->| | + | |-- Create session dir ------->| + |<-- 201 Location: /.../{id} --| | + | | | + |-- PATCH /.tus/.uploads/{id} >| | + | Upload-Offset: 0 |-- Assign volume ------------>| + | [chunk data] |-- Upload chunk ------------->| + |<-- 204 Upload-Offset: N -----| | + | | | + | (network failure) | | + | | | + |-- HEAD /.tus/.uploads/{id} ->| | + |<-- Upload-Offset: N ---------| | + | | | + |-- PATCH (resume) ----------->|-- Upload remaining -------->| + |<-- 204 (complete) -----------|-- Assemble final file ----->| +``` + +## Comparison with S3 Multipart + +| Feature | TUS | S3 Multipart | +|---------|-----|--------------| +| Protocol | Custom HTTP headers | S3 API | +| Session Init | POST with Upload-Length | CreateMultipartUpload | +| Upload Data | PATCH with offset | UploadPart with partNumber | +| Resume | HEAD to get offset | ListParts | +| Complete | Automatic at final offset | CompleteMultipartUpload | +| Ordering | Sequential (offset-based) | Parallel (part numbers) | + +## Related Resources +- [TUS Protocol Specification](https://tus.io/protocols/resumable-upload) +- [tus-js-client](https://github.com/tus/tus-js-client) - JavaScript client +- [go-tus](https://github.com/eventials/go-tus) - Go client +- [SeaweedFS S3 API](../../weed/s3api) - Alternative multipart upload