From 5b4fede08e939c30330f4c307dfd39f0eab4f0eb Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 12:53:20 -0800 Subject: [PATCH 01/16] Add TUS protocol integration tests This commit adds integration tests for the TUS (resumable upload) protocol in preparation for implementing TUS support in the filer. Test coverage includes: - OPTIONS handler for capability discovery - Basic single-request upload - Chunked/resumable uploads - HEAD requests for offset tracking - DELETE for upload cancellation - Error handling (invalid offsets, missing uploads) - Creation-with-upload extension - Resume after interruption simulation Tests are skipped in short mode and require a running SeaweedFS cluster. --- test/tus/README.md | 65 +++ test/tus/tus_integration_test.go | 739 +++++++++++++++++++++++++++++++ 2 files changed, 804 insertions(+) create mode 100644 test/tus/README.md create mode 100644 test/tus/tus_integration_test.go diff --git a/test/tus/README.md b/test/tus/README.md new file mode 100644 index 000000000..7477168d0 --- /dev/null +++ b/test/tus/README.md @@ -0,0 +1,65 @@ +# TUS Protocol Integration Tests + +This directory contains integration tests for the TUS (resumable upload) protocol support in SeaweedFS Filer. + +## TUS Protocol Overview + +TUS is an open protocol for resumable file uploads. It allows clients to upload files in chunks and resume uploads after network failures. + +Key endpoints: +- `OPTIONS /tus/` - Server capability discovery +- `POST /tus/{path}` - Create new upload session +- `HEAD /tus/{upload-id}` - Get current upload offset +- `PATCH /tus/{upload-id}` - Upload data at offset +- `DELETE /tus/{upload-id}` - Cancel upload + +## Prerequisites + +1. Build the weed binary: +```bash +cd ../../weed +go build +``` + +2. The tests will automatically start required servers (master, volume, filer). + +## Running Tests + +### Run all TUS tests: +```bash +go test -v ./test/tus/... +``` + +### Run specific test: +```bash +go test -v ./test/tus -run TestTusBasicUpload +``` + +### Skip integration tests (short mode): +```bash +go test -v -short ./test/tus/... +``` + +## Test Coverage + +The tests cover: +- Basic upload creation and completion +- Chunked/resumable uploads +- Upload offset tracking (HEAD requests) +- Upload cancellation (DELETE requests) +- Error handling (invalid offsets, missing uploads) +- Large file uploads with multiple chunks +- Concurrent uploads +- Metadata handling + +## TUS Protocol Headers + +Required headers for TUS requests: +- `Tus-Resumable: 1.0.0` - Protocol version +- `Upload-Length` - Total file size (on creation) +- `Upload-Offset` - Current byte offset (on PATCH) +- `Content-Type: application/offset+octet-stream` (on PATCH) + +Optional headers: +- `Upload-Metadata` - Base64-encoded key-value pairs + diff --git a/test/tus/tus_integration_test.go b/test/tus/tus_integration_test.go new file mode 100644 index 000000000..ac5348087 --- /dev/null +++ b/test/tus/tus_integration_test.go @@ -0,0 +1,739 @@ +package tus + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + TusVersion = "1.0.0" + testFilerPort = "18888" + testMasterPort = "19333" + testVolumePort = "18080" +) + +// TestCluster represents a running SeaweedFS cluster for testing +type TestCluster struct { + masterCmd *exec.Cmd + volumeCmd *exec.Cmd + filerCmd *exec.Cmd + dataDir string +} + +func (c *TestCluster) Stop() { + if c.filerCmd != nil && c.filerCmd.Process != nil { + c.filerCmd.Process.Kill() + c.filerCmd.Wait() + } + if c.volumeCmd != nil && c.volumeCmd.Process != nil { + c.volumeCmd.Process.Kill() + c.volumeCmd.Wait() + } + if c.masterCmd != nil && c.masterCmd.Process != nil { + c.masterCmd.Process.Kill() + c.masterCmd.Wait() + } +} + +func (c *TestCluster) FilerURL() string { + return fmt.Sprintf("http://127.0.0.1:%s", testFilerPort) +} + +func (c *TestCluster) TusURL() string { + return fmt.Sprintf("%s/.tus", c.FilerURL()) +} + +// startTestCluster starts a SeaweedFS cluster for testing +func startTestCluster(t *testing.T, ctx context.Context) (*TestCluster, error) { + weedBinary := findWeedBinary() + if weedBinary == "" { + return nil, fmt.Errorf("weed binary not found - please build it first: cd weed && go build") + } + + dataDir, err := os.MkdirTemp("", "seaweedfs_tus_test_") + if err != nil { + return nil, err + } + + cluster := &TestCluster{dataDir: dataDir} + + // Create subdirectories + masterDir := filepath.Join(dataDir, "master") + volumeDir := filepath.Join(dataDir, "volume") + filerDir := filepath.Join(dataDir, "filer") + os.MkdirAll(masterDir, 0755) + os.MkdirAll(volumeDir, 0755) + os.MkdirAll(filerDir, 0755) + + // Start master + masterCmd := exec.CommandContext(ctx, weedBinary, "master", + "-port", testMasterPort, + "-mdir", masterDir, + "-ip", "127.0.0.1", + ) + masterLogFile, _ := os.Create(filepath.Join(masterDir, "master.log")) + masterCmd.Stdout = masterLogFile + masterCmd.Stderr = masterLogFile + if err := masterCmd.Start(); err != nil { + os.RemoveAll(dataDir) + return nil, fmt.Errorf("failed to start master: %v", err) + } + cluster.masterCmd = masterCmd + + // Wait for master + time.Sleep(2 * time.Second) + + // Start volume server + volumeCmd := exec.CommandContext(ctx, weedBinary, "volume", + "-port", testVolumePort, + "-dir", volumeDir, + "-mserver", "127.0.0.1:"+testMasterPort, + "-ip", "127.0.0.1", + ) + volumeLogFile, _ := os.Create(filepath.Join(volumeDir, "volume.log")) + volumeCmd.Stdout = volumeLogFile + volumeCmd.Stderr = volumeLogFile + if err := volumeCmd.Start(); err != nil { + cluster.Stop() + os.RemoveAll(dataDir) + return nil, fmt.Errorf("failed to start volume server: %v", err) + } + cluster.volumeCmd = volumeCmd + + // Wait for volume server + time.Sleep(2 * time.Second) + + // Start filer with TUS enabled + filerCmd := exec.CommandContext(ctx, weedBinary, "filer", + "-port", testFilerPort, + "-master", "127.0.0.1:"+testMasterPort, + "-ip", "127.0.0.1", + "-dataCenter", "dc1", + ) + filerLogFile, _ := os.Create(filepath.Join(filerDir, "filer.log")) + filerCmd.Stdout = filerLogFile + filerCmd.Stderr = filerLogFile + if err := filerCmd.Start(); err != nil { + cluster.Stop() + os.RemoveAll(dataDir) + return nil, fmt.Errorf("failed to start filer: %v", err) + } + cluster.filerCmd = filerCmd + + // Wait for filer + if err := waitForHTTPServer("http://127.0.0.1:"+testFilerPort+"/", 30*time.Second); err != nil { + cluster.Stop() + os.RemoveAll(dataDir) + return nil, fmt.Errorf("filer not ready: %v", err) + } + + return cluster, nil +} + +func findWeedBinary() string { + candidates := []string{ + "../../weed/weed", + "../weed/weed", + "./weed/weed", + "weed", + } + for _, candidate := range candidates { + if _, err := os.Stat(candidate); err == nil { + return candidate + } + } + if path, err := exec.LookPath("weed"); err == nil { + return path + } + return "" +} + +func waitForHTTPServer(url string, timeout time.Duration) error { + start := time.Now() + client := &http.Client{Timeout: 1 * time.Second} + for time.Since(start) < timeout { + resp, err := client.Get(url) + if err == nil { + resp.Body.Close() + return nil + } + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("timeout waiting for %s", url) +} + +// encodeTusMetadata encodes key-value pairs for Upload-Metadata header +func encodeTusMetadata(metadata map[string]string) string { + var parts []string + for k, v := range metadata { + encoded := base64.StdEncoding.EncodeToString([]byte(v)) + parts = append(parts, fmt.Sprintf("%s %s", k, encoded)) + } + return strings.Join(parts, ",") +} + +// TestTusOptionsHandler tests the OPTIONS endpoint for capability discovery +func TestTusOptionsHandler(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + cluster, err := startTestCluster(t, ctx) + require.NoError(t, err) + defer func() { + cluster.Stop() + os.RemoveAll(cluster.dataDir) + }() + + // Test OPTIONS request + req, err := http.NewRequest(http.MethodOptions, cluster.TusURL()+"/", nil) + require.NoError(t, err) + req.Header.Set("Tus-Resumable", TusVersion) + + client := &http.Client{} + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + // Verify TUS headers + assert.Equal(t, http.StatusOK, resp.StatusCode, "OPTIONS should return 200 OK") + assert.Equal(t, TusVersion, resp.Header.Get("Tus-Resumable"), "Should return Tus-Resumable header") + assert.NotEmpty(t, resp.Header.Get("Tus-Version"), "Should return Tus-Version header") + assert.NotEmpty(t, resp.Header.Get("Tus-Extension"), "Should return Tus-Extension header") + assert.NotEmpty(t, resp.Header.Get("Tus-Max-Size"), "Should return Tus-Max-Size header") +} + +// TestTusBasicUpload tests a simple complete upload +func TestTusBasicUpload(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + cluster, err := startTestCluster(t, ctx) + require.NoError(t, err) + defer func() { + cluster.Stop() + os.RemoveAll(cluster.dataDir) + }() + + testData := []byte("Hello, TUS Protocol! This is a test file.") + targetPath := "/testdir/testfile.txt" + + // Step 1: Create upload (POST) + createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil) + require.NoError(t, err) + createReq.Header.Set("Tus-Resumable", TusVersion) + createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData))) + createReq.Header.Set("Upload-Metadata", encodeTusMetadata(map[string]string{ + "filename": "testfile.txt", + "content-type": "text/plain", + })) + + client := &http.Client{} + createResp, err := client.Do(createReq) + require.NoError(t, err) + defer createResp.Body.Close() + + assert.Equal(t, http.StatusCreated, createResp.StatusCode, "POST should return 201 Created") + uploadLocation := createResp.Header.Get("Location") + assert.NotEmpty(t, uploadLocation, "Should return Location header with upload URL") + t.Logf("Upload location: %s", uploadLocation) + + // Step 2: Upload data (PATCH) + patchReq, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(testData)) + require.NoError(t, err) + patchReq.Header.Set("Tus-Resumable", TusVersion) + patchReq.Header.Set("Upload-Offset", "0") + patchReq.Header.Set("Content-Type", "application/offset+octet-stream") + patchReq.Header.Set("Content-Length", strconv.Itoa(len(testData))) + + patchResp, err := client.Do(patchReq) + require.NoError(t, err) + defer patchResp.Body.Close() + + assert.Equal(t, http.StatusNoContent, patchResp.StatusCode, "PATCH should return 204 No Content") + newOffset := patchResp.Header.Get("Upload-Offset") + assert.Equal(t, strconv.Itoa(len(testData)), newOffset, "Upload-Offset should equal total file size") + + // Step 3: Verify the file was created + getResp, err := client.Get(cluster.FilerURL() + targetPath) + require.NoError(t, err) + defer getResp.Body.Close() + + assert.Equal(t, http.StatusOK, getResp.StatusCode, "GET should return 200 OK") + body, err := io.ReadAll(getResp.Body) + require.NoError(t, err) + assert.Equal(t, testData, body, "File content should match uploaded data") +} + +// TestTusChunkedUpload tests uploading a file in multiple chunks +func TestTusChunkedUpload(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + cluster, err := startTestCluster(t, ctx) + require.NoError(t, err) + defer func() { + cluster.Stop() + os.RemoveAll(cluster.dataDir) + }() + + // Create test data (100KB) + testData := make([]byte, 100*1024) + for i := range testData { + testData[i] = byte(i % 256) + } + chunkSize := 32 * 1024 // 32KB chunks + targetPath := "/chunked/largefile.bin" + + client := &http.Client{} + + // Step 1: Create upload + createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil) + require.NoError(t, err) + createReq.Header.Set("Tus-Resumable", TusVersion) + createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData))) + + createResp, err := client.Do(createReq) + require.NoError(t, err) + defer createResp.Body.Close() + + require.Equal(t, http.StatusCreated, createResp.StatusCode) + uploadLocation := createResp.Header.Get("Location") + require.NotEmpty(t, uploadLocation) + t.Logf("Upload location: %s", uploadLocation) + + // Step 2: Upload in chunks + offset := 0 + for offset < len(testData) { + end := offset + chunkSize + if end > len(testData) { + end = len(testData) + } + chunk := testData[offset:end] + + patchReq, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(chunk)) + require.NoError(t, err) + patchReq.Header.Set("Tus-Resumable", TusVersion) + patchReq.Header.Set("Upload-Offset", strconv.Itoa(offset)) + patchReq.Header.Set("Content-Type", "application/offset+octet-stream") + patchReq.Header.Set("Content-Length", strconv.Itoa(len(chunk))) + + patchResp, err := client.Do(patchReq) + require.NoError(t, err) + patchResp.Body.Close() + + require.Equal(t, http.StatusNoContent, patchResp.StatusCode, + "PATCH chunk at offset %d should return 204", offset) + newOffset, _ := strconv.Atoi(patchResp.Header.Get("Upload-Offset")) + require.Equal(t, end, newOffset, "New offset should be %d", end) + + t.Logf("Uploaded chunk: offset=%d, size=%d, newOffset=%d", offset, len(chunk), newOffset) + offset = end + } + + // Step 3: Verify the complete file + getResp, err := client.Get(cluster.FilerURL() + targetPath) + require.NoError(t, err) + defer getResp.Body.Close() + + assert.Equal(t, http.StatusOK, getResp.StatusCode) + body, err := io.ReadAll(getResp.Body) + require.NoError(t, err) + assert.Equal(t, testData, body, "File content should match uploaded data") +} + +// TestTusHeadRequest tests the HEAD endpoint to get upload offset +func TestTusHeadRequest(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + cluster, err := startTestCluster(t, ctx) + require.NoError(t, err) + defer func() { + cluster.Stop() + os.RemoveAll(cluster.dataDir) + }() + + testData := []byte("Test data for HEAD request verification") + targetPath := "/headtest/file.txt" + client := &http.Client{} + + // Create upload + createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil) + require.NoError(t, err) + createReq.Header.Set("Tus-Resumable", TusVersion) + createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData))) + + createResp, err := client.Do(createReq) + require.NoError(t, err) + defer createResp.Body.Close() + require.Equal(t, http.StatusCreated, createResp.StatusCode) + uploadLocation := createResp.Header.Get("Location") + + // HEAD before any data uploaded - offset should be 0 + headReq1, err := http.NewRequest(http.MethodHead, uploadLocation, nil) + require.NoError(t, err) + headReq1.Header.Set("Tus-Resumable", TusVersion) + + headResp1, err := client.Do(headReq1) + require.NoError(t, err) + defer headResp1.Body.Close() + + assert.Equal(t, http.StatusOK, headResp1.StatusCode) + assert.Equal(t, "0", headResp1.Header.Get("Upload-Offset"), "Initial offset should be 0") + assert.Equal(t, strconv.Itoa(len(testData)), headResp1.Header.Get("Upload-Length")) + + // Upload half the data + halfLen := len(testData) / 2 + patchReq, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(testData[:halfLen])) + require.NoError(t, err) + patchReq.Header.Set("Tus-Resumable", TusVersion) + patchReq.Header.Set("Upload-Offset", "0") + patchReq.Header.Set("Content-Type", "application/offset+octet-stream") + + patchResp, err := client.Do(patchReq) + require.NoError(t, err) + patchResp.Body.Close() + require.Equal(t, http.StatusNoContent, patchResp.StatusCode) + + // HEAD after partial upload - offset should be halfLen + headReq2, err := http.NewRequest(http.MethodHead, uploadLocation, nil) + require.NoError(t, err) + headReq2.Header.Set("Tus-Resumable", TusVersion) + + headResp2, err := client.Do(headReq2) + require.NoError(t, err) + defer headResp2.Body.Close() + + assert.Equal(t, http.StatusOK, headResp2.StatusCode) + assert.Equal(t, strconv.Itoa(halfLen), headResp2.Header.Get("Upload-Offset"), + "Offset should be %d after partial upload", halfLen) +} + +// TestTusDeleteUpload tests canceling an in-progress upload +func TestTusDeleteUpload(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + cluster, err := startTestCluster(t, ctx) + require.NoError(t, err) + defer func() { + cluster.Stop() + os.RemoveAll(cluster.dataDir) + }() + + testData := []byte("Data to be deleted") + targetPath := "/deletetest/file.txt" + client := &http.Client{} + + // Create upload + createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil) + require.NoError(t, err) + createReq.Header.Set("Tus-Resumable", TusVersion) + createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData))) + + createResp, err := client.Do(createReq) + require.NoError(t, err) + defer createResp.Body.Close() + require.Equal(t, http.StatusCreated, createResp.StatusCode) + uploadLocation := createResp.Header.Get("Location") + + // Upload some data + patchReq, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(testData[:10])) + require.NoError(t, err) + patchReq.Header.Set("Tus-Resumable", TusVersion) + patchReq.Header.Set("Upload-Offset", "0") + patchReq.Header.Set("Content-Type", "application/offset+octet-stream") + + patchResp, err := client.Do(patchReq) + require.NoError(t, err) + patchResp.Body.Close() + + // Delete the upload + deleteReq, err := http.NewRequest(http.MethodDelete, uploadLocation, nil) + require.NoError(t, err) + deleteReq.Header.Set("Tus-Resumable", TusVersion) + + deleteResp, err := client.Do(deleteReq) + require.NoError(t, err) + defer deleteResp.Body.Close() + + assert.Equal(t, http.StatusNoContent, deleteResp.StatusCode, "DELETE should return 204") + + // Verify upload is gone - HEAD should return 404 + headReq, err := http.NewRequest(http.MethodHead, uploadLocation, nil) + require.NoError(t, err) + headReq.Header.Set("Tus-Resumable", TusVersion) + + headResp, err := client.Do(headReq) + require.NoError(t, err) + defer headResp.Body.Close() + + assert.Equal(t, http.StatusNotFound, headResp.StatusCode, "HEAD after DELETE should return 404") +} + +// TestTusInvalidOffset tests error handling for mismatched offsets +func TestTusInvalidOffset(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + cluster, err := startTestCluster(t, ctx) + require.NoError(t, err) + defer func() { + cluster.Stop() + os.RemoveAll(cluster.dataDir) + }() + + testData := []byte("Test data for offset validation") + targetPath := "/offsettest/file.txt" + client := &http.Client{} + + // Create upload + createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil) + require.NoError(t, err) + createReq.Header.Set("Tus-Resumable", TusVersion) + createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData))) + + createResp, err := client.Do(createReq) + require.NoError(t, err) + defer createResp.Body.Close() + require.Equal(t, http.StatusCreated, createResp.StatusCode) + uploadLocation := createResp.Header.Get("Location") + + // Try to upload with wrong offset (should be 0, but we send 100) + patchReq, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(testData)) + require.NoError(t, err) + patchReq.Header.Set("Tus-Resumable", TusVersion) + patchReq.Header.Set("Upload-Offset", "100") // Wrong offset! + patchReq.Header.Set("Content-Type", "application/offset+octet-stream") + + patchResp, err := client.Do(patchReq) + require.NoError(t, err) + defer patchResp.Body.Close() + + assert.Equal(t, http.StatusConflict, patchResp.StatusCode, + "PATCH with wrong offset should return 409 Conflict") +} + +// TestTusUploadNotFound tests accessing a non-existent upload +func TestTusUploadNotFound(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + cluster, err := startTestCluster(t, ctx) + require.NoError(t, err) + defer func() { + cluster.Stop() + os.RemoveAll(cluster.dataDir) + }() + + client := &http.Client{} + fakeUploadURL := cluster.TusURL() + "/.uploads/nonexistent-upload-id" + + // HEAD on non-existent upload + headReq, err := http.NewRequest(http.MethodHead, fakeUploadURL, nil) + require.NoError(t, err) + headReq.Header.Set("Tus-Resumable", TusVersion) + + headResp, err := client.Do(headReq) + require.NoError(t, err) + defer headResp.Body.Close() + + assert.Equal(t, http.StatusNotFound, headResp.StatusCode, + "HEAD on non-existent upload should return 404") + + // PATCH on non-existent upload + patchReq, err := http.NewRequest(http.MethodPatch, fakeUploadURL, bytes.NewReader([]byte("data"))) + require.NoError(t, err) + patchReq.Header.Set("Tus-Resumable", TusVersion) + patchReq.Header.Set("Upload-Offset", "0") + patchReq.Header.Set("Content-Type", "application/offset+octet-stream") + + patchResp, err := client.Do(patchReq) + require.NoError(t, err) + defer patchResp.Body.Close() + + assert.Equal(t, http.StatusNotFound, patchResp.StatusCode, + "PATCH on non-existent upload should return 404") +} + +// TestTusCreationWithUpload tests the creation-with-upload extension +func TestTusCreationWithUpload(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + cluster, err := startTestCluster(t, ctx) + require.NoError(t, err) + defer func() { + cluster.Stop() + os.RemoveAll(cluster.dataDir) + }() + + testData := []byte("Small file uploaded in creation request") + targetPath := "/creationwithupload/smallfile.txt" + client := &http.Client{} + + // Create upload with data in the same request + createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, bytes.NewReader(testData)) + require.NoError(t, err) + createReq.Header.Set("Tus-Resumable", TusVersion) + createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData))) + createReq.Header.Set("Content-Type", "application/offset+octet-stream") + + createResp, err := client.Do(createReq) + require.NoError(t, err) + defer createResp.Body.Close() + + assert.Equal(t, http.StatusCreated, createResp.StatusCode) + uploadLocation := createResp.Header.Get("Location") + assert.NotEmpty(t, uploadLocation) + + // Check Upload-Offset header - should indicate all data was received + uploadOffset := createResp.Header.Get("Upload-Offset") + assert.Equal(t, strconv.Itoa(len(testData)), uploadOffset, + "Upload-Offset should equal file size for complete upload") + + // Verify the file + getResp, err := client.Get(cluster.FilerURL() + targetPath) + require.NoError(t, err) + defer getResp.Body.Close() + + assert.Equal(t, http.StatusOK, getResp.StatusCode) + body, err := io.ReadAll(getResp.Body) + require.NoError(t, err) + assert.Equal(t, testData, body) +} + +// TestTusResumeAfterInterruption simulates resuming an upload after failure +func TestTusResumeAfterInterruption(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + cluster, err := startTestCluster(t, ctx) + require.NoError(t, err) + defer func() { + cluster.Stop() + os.RemoveAll(cluster.dataDir) + }() + + // 50KB test data + testData := make([]byte, 50*1024) + for i := range testData { + testData[i] = byte(i % 256) + } + targetPath := "/resume/interrupted.bin" + client := &http.Client{} + + // Create upload + createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil) + require.NoError(t, err) + createReq.Header.Set("Tus-Resumable", TusVersion) + createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData))) + + createResp, err := client.Do(createReq) + require.NoError(t, err) + defer createResp.Body.Close() + require.Equal(t, http.StatusCreated, createResp.StatusCode) + uploadLocation := createResp.Header.Get("Location") + + // Upload first 20KB + firstChunkSize := 20 * 1024 + patchReq1, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(testData[:firstChunkSize])) + require.NoError(t, err) + patchReq1.Header.Set("Tus-Resumable", TusVersion) + patchReq1.Header.Set("Upload-Offset", "0") + patchReq1.Header.Set("Content-Type", "application/offset+octet-stream") + + patchResp1, err := client.Do(patchReq1) + require.NoError(t, err) + patchResp1.Body.Close() + require.Equal(t, http.StatusNoContent, patchResp1.StatusCode) + + t.Log("Simulating network interruption...") + + // Simulate resumption: Query current offset with HEAD + headReq, err := http.NewRequest(http.MethodHead, uploadLocation, nil) + require.NoError(t, err) + headReq.Header.Set("Tus-Resumable", TusVersion) + + headResp, err := client.Do(headReq) + require.NoError(t, err) + defer headResp.Body.Close() + + require.Equal(t, http.StatusOK, headResp.StatusCode) + currentOffset, _ := strconv.Atoi(headResp.Header.Get("Upload-Offset")) + t.Logf("Resumed upload at offset: %d", currentOffset) + require.Equal(t, firstChunkSize, currentOffset) + + // Resume upload from current offset + patchReq2, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(testData[currentOffset:])) + require.NoError(t, err) + patchReq2.Header.Set("Tus-Resumable", TusVersion) + patchReq2.Header.Set("Upload-Offset", strconv.Itoa(currentOffset)) + patchReq2.Header.Set("Content-Type", "application/offset+octet-stream") + + patchResp2, err := client.Do(patchReq2) + require.NoError(t, err) + patchResp2.Body.Close() + require.Equal(t, http.StatusNoContent, patchResp2.StatusCode) + + // Verify complete file + getResp, err := client.Get(cluster.FilerURL() + targetPath) + require.NoError(t, err) + defer getResp.Body.Close() + + assert.Equal(t, http.StatusOK, getResp.StatusCode) + body, err := io.ReadAll(getResp.Body) + require.NoError(t, err) + assert.Equal(t, testData, body, "Resumed upload should produce complete file") +} + From 3675d9db016c486cffcd77e67eaf7802aefd3f18 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 12:55:08 -0800 Subject: [PATCH 02/16] Add TUS session storage types and utilities Implements TUS upload session management: - TusSession struct for tracking upload state - Session creation with directory-based storage - Session persistence using filer entries - Session retrieval and offset updates - Session deletion with chunk cleanup - Upload completion with chunk assembly into final file Session data is stored in /.uploads.tus/{upload-id}/ directory, following the pattern used by S3 multipart uploads. --- weed/server/filer_server_tus_session.go | 255 ++++++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100644 weed/server/filer_server_tus_session.go diff --git a/weed/server/filer_server_tus_session.go b/weed/server/filer_server_tus_session.go new file mode 100644 index 000000000..e0e811a8c --- /dev/null +++ b/weed/server/filer_server_tus_session.go @@ -0,0 +1,255 @@ +package weed_server + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +const ( + TusVersion = "1.0.0" + TusMaxSize = int64(5 * 1024 * 1024 * 1024) // 5GB default max size + TusUploadsFolder = ".uploads.tus" + TusInfoFileName = ".info" + TusChunkExt = ".chunk" + TusExtensions = "creation,creation-with-upload,termination" +) + +// TusSession represents an in-progress TUS upload session +type TusSession struct { + ID string `json:"id"` + TargetPath string `json:"target_path"` + Size int64 `json:"size"` + Offset int64 `json:"offset"` + Metadata map[string]string `json:"metadata,omitempty"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt time.Time `json:"expires_at,omitempty"` + Chunks []*TusChunkInfo `json:"chunks,omitempty"` +} + +// TusChunkInfo tracks individual chunk uploads within a session +type TusChunkInfo struct { + Offset int64 `json:"offset"` + Size int64 `json:"size"` + FileId string `json:"file_id"` + UploadAt int64 `json:"upload_at"` +} + +// tusSessionDir returns the directory path for storing TUS upload sessions +func (fs *FilerServer) tusSessionDir() string { + return "/" + TusUploadsFolder +} + +// tusSessionPath returns the path to a specific upload session directory +func (fs *FilerServer) tusSessionPath(uploadID string) string { + return fmt.Sprintf("/%s/%s", TusUploadsFolder, uploadID) +} + +// tusSessionInfoPath returns the path to the session info file +func (fs *FilerServer) tusSessionInfoPath(uploadID string) string { + return fmt.Sprintf("/%s/%s/%s", TusUploadsFolder, uploadID, TusInfoFileName) +} + +// createTusSession creates a new TUS upload session +func (fs *FilerServer) createTusSession(ctx context.Context, uploadID, targetPath string, size int64, metadata map[string]string) (*TusSession, error) { + session := &TusSession{ + ID: uploadID, + TargetPath: targetPath, + Size: size, + Offset: 0, + Metadata: metadata, + CreatedAt: time.Now(), + ExpiresAt: time.Now().Add(7 * 24 * time.Hour), // 7 days default expiration + Chunks: []*TusChunkInfo{}, + } + + // Create session directory + sessionDirPath := util.FullPath(fs.tusSessionPath(uploadID)) + if err := fs.filer.CreateEntry(ctx, &filer.Entry{ + FullPath: sessionDirPath, + Attr: filer.Attr{ + Mode: os.ModeDir | 0755, + Crtime: time.Now(), + Mtime: time.Now(), + Uid: OS_UID, + Gid: OS_GID, + }, + }, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil { + return nil, fmt.Errorf("create session directory: %w", err) + } + + // Save session info + if err := fs.saveTusSession(ctx, session); err != nil { + // Cleanup the directory on failure + fs.filer.DeleteEntryMetaAndData(ctx, sessionDirPath, true, true, false, false, nil, 0) + return nil, fmt.Errorf("save session info: %w", err) + } + + glog.V(2).Infof("Created TUS session %s for %s, size=%d", uploadID, targetPath, size) + return session, nil +} + +// saveTusSession saves the session info to the filer +func (fs *FilerServer) saveTusSession(ctx context.Context, session *TusSession) error { + sessionData, err := json.Marshal(session) + if err != nil { + return fmt.Errorf("marshal session: %w", err) + } + + infoPath := util.FullPath(fs.tusSessionInfoPath(session.ID)) + entry := &filer.Entry{ + FullPath: infoPath, + Attr: filer.Attr{ + Mode: 0644, + Crtime: session.CreatedAt, + Mtime: time.Now(), + Uid: OS_UID, + Gid: OS_GID, + }, + Content: sessionData, + } + + if err := fs.filer.CreateEntry(ctx, entry, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil { + return fmt.Errorf("save session info entry: %w", err) + } + + return nil +} + +// getTusSession retrieves a TUS session by upload ID +func (fs *FilerServer) getTusSession(ctx context.Context, uploadID string) (*TusSession, error) { + infoPath := util.FullPath(fs.tusSessionInfoPath(uploadID)) + entry, err := fs.filer.FindEntry(ctx, infoPath) + if err != nil { + if err == filer_pb.ErrNotFound { + return nil, fmt.Errorf("session not found: %s", uploadID) + } + return nil, fmt.Errorf("find session: %w", err) + } + + var session TusSession + if err := json.Unmarshal(entry.Content, &session); err != nil { + return nil, fmt.Errorf("unmarshal session: %w", err) + } + + return &session, nil +} + +// updateTusSessionOffset updates the session offset after a successful chunk upload +func (fs *FilerServer) updateTusSessionOffset(ctx context.Context, uploadID string, newOffset int64, chunk *TusChunkInfo) error { + session, err := fs.getTusSession(ctx, uploadID) + if err != nil { + return err + } + + session.Offset = newOffset + if chunk != nil { + session.Chunks = append(session.Chunks, chunk) + } + + return fs.saveTusSession(ctx, session) +} + +// deleteTusSession removes a TUS upload session and all its data +func (fs *FilerServer) deleteTusSession(ctx context.Context, uploadID string) error { + session, err := fs.getTusSession(ctx, uploadID) + if err != nil { + // Session might already be deleted or never existed + glog.V(1).Infof("TUS session %s not found for deletion: %v", uploadID, err) + return nil + } + + // Delete any uploaded chunks from volume servers + for _, chunk := range session.Chunks { + if chunk.FileId != "" { + fs.filer.DeleteChunks(ctx, util.FullPath(session.TargetPath), []*filer_pb.FileChunk{ + {FileId: chunk.FileId}, + }) + } + } + + // Delete the session directory + sessionDirPath := util.FullPath(fs.tusSessionPath(uploadID)) + if err := fs.filer.DeleteEntryMetaAndData(ctx, sessionDirPath, true, true, false, false, nil, 0); err != nil { + return fmt.Errorf("delete session directory: %w", err) + } + + glog.V(2).Infof("Deleted TUS session %s", uploadID) + return nil +} + +// completeTusUpload assembles all chunks and creates the final file +func (fs *FilerServer) completeTusUpload(ctx context.Context, session *TusSession) error { + if session.Offset != session.Size { + return fmt.Errorf("upload incomplete: offset=%d, expected=%d", session.Offset, session.Size) + } + + // Assemble file chunks in order + var fileChunks []*filer_pb.FileChunk + var offset int64 = 0 + + for _, chunk := range session.Chunks { + fid, fidErr := filer_pb.ToFileIdObject(chunk.FileId) + if fidErr != nil { + glog.Warningf("Invalid file ID %s: %v", chunk.FileId, fidErr) + continue + } + + fileChunk := &filer_pb.FileChunk{ + FileId: chunk.FileId, + Offset: offset, + Size: uint64(chunk.Size), + ModifiedTsNs: chunk.UploadAt, + Fid: fid, + } + fileChunks = append(fileChunks, fileChunk) + offset += chunk.Size + } + + // Determine content type from metadata + contentType := "" + if session.Metadata != nil { + if ct, ok := session.Metadata["content-type"]; ok { + contentType = ct + } + } + + // Create the final file entry + targetPath := util.FullPath(session.TargetPath) + entry := &filer.Entry{ + FullPath: targetPath, + Attr: filer.Attr{ + Mode: 0644, + Crtime: session.CreatedAt, + Mtime: time.Now(), + Uid: OS_UID, + Gid: OS_GID, + Mime: contentType, + }, + Chunks: fileChunks, + } + + // Ensure parent directory exists + if err := fs.filer.CreateEntry(ctx, entry, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil { + return fmt.Errorf("create final file entry: %w", err) + } + + // Delete the session (but keep the chunks since they're now part of the final file) + sessionDirPath := util.FullPath(fs.tusSessionPath(session.ID)) + if err := fs.filer.DeleteEntryMetaAndData(ctx, sessionDirPath, true, false, false, false, nil, 0); err != nil { + glog.V(1).Infof("Failed to cleanup TUS session directory %s: %v", session.ID, err) + } + + glog.V(2).Infof("Completed TUS upload %s -> %s, size=%d, chunks=%d", + session.ID, session.TargetPath, session.Size, len(fileChunks)) + + return nil +} + From 44d0925d7bd932e8836e8884e751191231b6a298 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 12:56:42 -0800 Subject: [PATCH 03/16] Add TUS HTTP handlers Implements TUS protocol HTTP handlers: - tusHandler: Main entry point routing requests - tusOptionsHandler: Capability discovery (OPTIONS) - tusCreateHandler: Create new upload (POST) - tusHeadHandler: Get upload offset (HEAD) - tusPatchHandler: Upload data at offset (PATCH) - tusDeleteHandler: Cancel upload (DELETE) - tusWriteData: Upload data to volume servers Features: - Supports creation-with-upload extension - Validates TUS protocol headers - Offset conflict detection - Automatic upload completion when size is reached - Metadata parsing from Upload-Metadata header --- weed/server/filer_server_tus_handlers.go | 352 +++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 weed/server/filer_server_tus_handlers.go diff --git a/weed/server/filer_server_tus_handlers.go b/weed/server/filer_server_tus_handlers.go new file mode 100644 index 000000000..8a426941e --- /dev/null +++ b/weed/server/filer_server_tus_handlers.go @@ -0,0 +1,352 @@ +package weed_server + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// tusHandler is the main entry point for TUS protocol requests +func (fs *FilerServer) tusHandler(w http.ResponseWriter, r *http.Request) { + // Set common TUS response headers + w.Header().Set("Tus-Resumable", TusVersion) + + // Check Tus-Resumable header for non-OPTIONS requests + if r.Method != http.MethodOptions { + tusVersion := r.Header.Get("Tus-Resumable") + if tusVersion != TusVersion { + http.Error(w, "Unsupported TUS version", http.StatusPreconditionFailed) + return + } + } + + // Route based on method and path + path := r.URL.Path + tusPrefix := "/.tus" + + // Check if this is an upload location (contains upload ID after /.tus/.uploads/) + if strings.HasPrefix(path, tusPrefix+"/.uploads/") { + uploadID := strings.TrimPrefix(path, tusPrefix+"/.uploads/") + uploadID = strings.Split(uploadID, "/")[0] // Get just the ID, not any trailing path + + switch r.Method { + case http.MethodHead: + fs.tusHeadHandler(w, r, uploadID) + case http.MethodPatch: + fs.tusPatchHandler(w, r, uploadID) + case http.MethodDelete: + fs.tusDeleteHandler(w, r, uploadID) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + return + } + + // Handle creation endpoints (POST to /.tus/{path}) + switch r.Method { + case http.MethodOptions: + fs.tusOptionsHandler(w, r) + case http.MethodPost: + fs.tusCreateHandler(w, r) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } +} + +// tusOptionsHandler handles OPTIONS requests for capability discovery +func (fs *FilerServer) tusOptionsHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Tus-Version", TusVersion) + w.Header().Set("Tus-Extension", TusExtensions) + w.Header().Set("Tus-Max-Size", strconv.FormatInt(TusMaxSize, 10)) + w.WriteHeader(http.StatusOK) +} + +// tusCreateHandler handles POST requests to create new uploads +func (fs *FilerServer) tusCreateHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Parse Upload-Length header (required) + uploadLengthStr := r.Header.Get("Upload-Length") + if uploadLengthStr == "" { + http.Error(w, "Upload-Length header required", http.StatusBadRequest) + return + } + uploadLength, err := strconv.ParseInt(uploadLengthStr, 10, 64) + if err != nil || uploadLength < 0 { + http.Error(w, "Invalid Upload-Length", http.StatusBadRequest) + return + } + if uploadLength > TusMaxSize { + http.Error(w, "Upload-Length exceeds maximum", http.StatusRequestEntityTooLarge) + return + } + + // Parse Upload-Metadata header (optional) + metadata := parseTusMetadata(r.Header.Get("Upload-Metadata")) + + // Determine target path from request URL + targetPath := strings.TrimPrefix(r.URL.Path, "/.tus") + if targetPath == "" || targetPath == "/" { + http.Error(w, "Target path required", http.StatusBadRequest) + return + } + + // Generate upload ID + uploadID := uuid.New().String() + + // Create upload session + session, err := fs.createTusSession(ctx, uploadID, targetPath, uploadLength, metadata) + if err != nil { + glog.Errorf("Failed to create TUS session: %v", err) + http.Error(w, "Failed to create upload", http.StatusInternalServerError) + return + } + + // Build upload location URL + uploadLocation := fmt.Sprintf("/.tus/.uploads/%s", uploadID) + + // Handle creation-with-upload extension + if r.ContentLength > 0 && r.Header.Get("Content-Type") == "application/offset+octet-stream" { + // Upload data in the creation request + bytesWritten, uploadErr := fs.tusWriteData(ctx, session, 0, r.Body, r.ContentLength) + if uploadErr != nil { + // Cleanup session on failure + fs.deleteTusSession(ctx, uploadID) + glog.Errorf("Failed to write initial TUS data: %v", uploadErr) + http.Error(w, "Failed to write data", http.StatusInternalServerError) + return + } + + // Update offset in response header + w.Header().Set("Upload-Offset", strconv.FormatInt(bytesWritten, 10)) + + // Check if upload is complete + if bytesWritten == session.Size { + if err := fs.completeTusUpload(ctx, session); err != nil { + glog.Errorf("Failed to complete TUS upload: %v", err) + http.Error(w, "Failed to complete upload", http.StatusInternalServerError) + return + } + } + } + + w.Header().Set("Location", uploadLocation) + w.WriteHeader(http.StatusCreated) +} + +// tusHeadHandler handles HEAD requests to get current upload offset +func (fs *FilerServer) tusHeadHandler(w http.ResponseWriter, r *http.Request, uploadID string) { + ctx := r.Context() + + session, err := fs.getTusSession(ctx, uploadID) + if err != nil { + http.Error(w, "Upload not found", http.StatusNotFound) + return + } + + w.Header().Set("Upload-Offset", strconv.FormatInt(session.Offset, 10)) + w.Header().Set("Upload-Length", strconv.FormatInt(session.Size, 10)) + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) +} + +// tusPatchHandler handles PATCH requests to upload data +func (fs *FilerServer) tusPatchHandler(w http.ResponseWriter, r *http.Request, uploadID string) { + ctx := r.Context() + + // Validate Content-Type + contentType := r.Header.Get("Content-Type") + if contentType != "application/offset+octet-stream" { + http.Error(w, "Content-Type must be application/offset+octet-stream", http.StatusUnsupportedMediaType) + return + } + + // Get current session + session, err := fs.getTusSession(ctx, uploadID) + if err != nil { + http.Error(w, "Upload not found", http.StatusNotFound) + return + } + + // Validate Upload-Offset header + uploadOffsetStr := r.Header.Get("Upload-Offset") + if uploadOffsetStr == "" { + http.Error(w, "Upload-Offset header required", http.StatusBadRequest) + return + } + uploadOffset, err := strconv.ParseInt(uploadOffsetStr, 10, 64) + if err != nil || uploadOffset < 0 { + http.Error(w, "Invalid Upload-Offset", http.StatusBadRequest) + return + } + + // Check offset matches current position + if uploadOffset != session.Offset { + http.Error(w, fmt.Sprintf("Offset mismatch: expected %d, got %d", session.Offset, uploadOffset), http.StatusConflict) + return + } + + // Write data + bytesWritten, err := fs.tusWriteData(ctx, session, uploadOffset, r.Body, r.ContentLength) + if err != nil { + glog.Errorf("Failed to write TUS data: %v", err) + http.Error(w, "Failed to write data", http.StatusInternalServerError) + return + } + + newOffset := uploadOffset + bytesWritten + + // Check if upload is complete + if newOffset == session.Size { + // Refresh session to get updated chunks + session, err = fs.getTusSession(ctx, uploadID) + if err != nil { + glog.Errorf("Failed to get updated TUS session: %v", err) + http.Error(w, "Failed to complete upload", http.StatusInternalServerError) + return + } + + if err := fs.completeTusUpload(ctx, session); err != nil { + glog.Errorf("Failed to complete TUS upload: %v", err) + http.Error(w, "Failed to complete upload", http.StatusInternalServerError) + return + } + } + + w.Header().Set("Upload-Offset", strconv.FormatInt(newOffset, 10)) + w.WriteHeader(http.StatusNoContent) +} + +// tusDeleteHandler handles DELETE requests to cancel uploads +func (fs *FilerServer) tusDeleteHandler(w http.ResponseWriter, r *http.Request, uploadID string) { + ctx := r.Context() + + if err := fs.deleteTusSession(ctx, uploadID); err != nil { + glog.Errorf("Failed to delete TUS session: %v", err) + http.Error(w, "Failed to delete upload", http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +// tusWriteData uploads data to volume servers and updates session +func (fs *FilerServer) tusWriteData(ctx context.Context, session *TusSession, offset int64, reader io.Reader, contentLength int64) (int64, error) { + if contentLength == 0 { + return 0, nil + } + + // Limit content length to remaining size + remaining := session.Size - offset + if contentLength > remaining { + contentLength = remaining + } + if contentLength <= 0 { + return 0, nil + } + + // Read data into buffer + buf := new(bytes.Buffer) + n, err := io.CopyN(buf, reader, contentLength) + if err != nil && err != io.EOF { + return 0, fmt.Errorf("read data: %w", err) + } + if n == 0 { + return 0, nil + } + + // Determine storage options based on target path + so, err := fs.detectStorageOption0(ctx, session.TargetPath, "", "", "", "", "", "", "", "", "") + if err != nil { + return 0, fmt.Errorf("detect storage option: %w", err) + } + + // Assign file ID from master + fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(ctx, so) + if assignErr != nil { + return 0, fmt.Errorf("assign volume: %w", assignErr) + } + + // Upload to volume server + uploader, uploaderErr := operation.NewUploader() + if uploaderErr != nil { + return 0, fmt.Errorf("create uploader: %w", uploaderErr) + } + + uploadResult, uploadErr, _ := uploader.Upload(ctx, bytes.NewReader(buf.Bytes()), &operation.UploadOption{ + UploadUrl: urlLocation, + Filename: "", + Cipher: false, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: auth, + }) + if uploadErr != nil { + return 0, fmt.Errorf("upload data: %w", uploadErr) + } + + // Create chunk info + chunk := &TusChunkInfo{ + Offset: offset, + Size: int64(uploadResult.Size), + FileId: fileId, + UploadAt: time.Now().UnixNano(), + } + + // Update session + if err := fs.updateTusSessionOffset(ctx, session.ID, offset+int64(uploadResult.Size), chunk); err != nil { + // Try to clean up the uploaded chunk + fs.filer.DeleteChunks(ctx, util.FullPath(session.TargetPath), []*filer_pb.FileChunk{ + {FileId: fileId}, + }) + return 0, fmt.Errorf("update session: %w", err) + } + + stats.FilerHandlerCounter.WithLabelValues("tusUploadChunk").Inc() + + return int64(uploadResult.Size), nil +} + +// parseTusMetadata parses the Upload-Metadata header +// Format: key1 base64value1,key2 base64value2,... +func parseTusMetadata(header string) map[string]string { + metadata := make(map[string]string) + if header == "" { + return metadata + } + + pairs := strings.Split(header, ",") + for _, pair := range pairs { + pair = strings.TrimSpace(pair) + parts := strings.SplitN(pair, " ", 2) + if len(parts) != 2 { + continue + } + key := strings.TrimSpace(parts[0]) + encodedValue := strings.TrimSpace(parts[1]) + + value, err := base64.StdEncoding.DecodeString(encodedValue) + if err != nil { + glog.V(1).Infof("Failed to decode TUS metadata value for key %s: %v", key, err) + continue + } + metadata[key] = string(value) + } + + return metadata +} + From 2ff0cc80a7495dd75ef7c5ae2942b0e8dddfb1dc Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 12:57:25 -0800 Subject: [PATCH 04/16] Wire up TUS protocol routes in filer server Add TUS handler route (/.tus/) to the filer HTTP server. The TUS route is registered before the catch-all route to ensure proper routing of TUS protocol requests. TUS protocol is now accessible at: - OPTIONS /.tus/ - Capability discovery - POST /.tus/{path} - Create upload - HEAD /.tus/.uploads/{id} - Get offset - PATCH /.tus/.uploads/{id} - Upload data - DELETE /.tus/.uploads/{id} - Cancel upload --- weed/server/filer_server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 95d344af4..a76994b89 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -195,6 +195,8 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) handleStaticResources(defaultMux) if !option.DisableHttp { defaultMux.HandleFunc("/healthz", requestIDMiddleware(fs.filerHealthzHandler)) + // TUS resumable upload protocol handler + defaultMux.HandleFunc("/.tus/", fs.filerGuard.WhiteList(requestIDMiddleware(fs.tusHandler))) defaultMux.HandleFunc("/", fs.filerGuard.WhiteList(requestIDMiddleware(fs.filerHandler))) } if defaultMux != readonlyMux { From e0d74fffea14cec0e23160c08dbe64dbf4a94531 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 13:01:35 -0800 Subject: [PATCH 05/16] Improve TUS integration test setup Add comprehensive Makefile for TUS tests with targets: - test-with-server: Run tests with automatic server management - test-basic/chunked/resume/errors: Specific test categories - manual-start/stop: For development testing - debug-logs/status: For debugging - ci-test: For CI/CD pipelines Update README.md with: - Detailed TUS protocol documentation - All endpoint descriptions with headers - Usage examples with curl commands - Architecture diagram - Comparison with S3 multipart uploads Follows the pattern established by other tests in test/ folder. --- test/tus/Makefile | 226 ++++++++++++++++++++++++++++++++++++++++++ test/tus/README.md | 241 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 427 insertions(+), 40 deletions(-) create mode 100644 test/tus/Makefile diff --git a/test/tus/Makefile b/test/tus/Makefile new file mode 100644 index 000000000..fd0b6d6b7 --- /dev/null +++ b/test/tus/Makefile @@ -0,0 +1,226 @@ +# Makefile for TUS Protocol Integration Tests +# This Makefile provides targets for running TUS (resumable upload) integration tests + +# Default values +SEAWEEDFS_BINARY ?= weed +FILER_PORT ?= 18888 +VOLUME_PORT ?= 18080 +MASTER_PORT ?= 19333 +TEST_TIMEOUT ?= 10m +VOLUME_MAX_SIZE_MB ?= 50 +VOLUME_MAX_COUNT ?= 100 + +# Test directory +TEST_DIR := $(shell pwd) +SEAWEEDFS_ROOT := $(shell cd ../.. && pwd) + +# Colors for output +RED := \033[0;31m +GREEN := \033[0;32m +YELLOW := \033[1;33m +NC := \033[0m # No Color + +.PHONY: all test clean start-seaweedfs stop-seaweedfs check-binary build-weed help test-basic test-chunked test-resume test-errors test-with-server + +all: test + +# Build SeaweedFS binary +build-weed: + @echo "Building SeaweedFS binary..." + @cd $(SEAWEEDFS_ROOT)/weed && go build -o weed + @echo "$(GREEN)SeaweedFS binary built successfully$(NC)" + +help: + @echo "SeaweedFS TUS Protocol Integration Tests" + @echo "" + @echo "Available targets:" + @echo " test - Run all TUS integration tests" + @echo " test-basic - Run basic TUS upload tests" + @echo " test-chunked - Run chunked upload tests" + @echo " test-resume - Run upload resume tests" + @echo " test-errors - Run error handling tests" + @echo " test-with-server - Run tests with automatic server management" + @echo " start-seaweedfs - Start SeaweedFS server for testing" + @echo " stop-seaweedfs - Stop SeaweedFS server" + @echo " clean - Clean up test artifacts" + @echo " check-binary - Check if SeaweedFS binary exists" + @echo " build-weed - Build SeaweedFS binary" + @echo "" + @echo "Configuration:" + @echo " SEAWEEDFS_BINARY=$(SEAWEEDFS_BINARY)" + @echo " FILER_PORT=$(FILER_PORT)" + @echo " VOLUME_PORT=$(VOLUME_PORT)" + @echo " MASTER_PORT=$(MASTER_PORT)" + @echo " TEST_TIMEOUT=$(TEST_TIMEOUT)" + +check-binary: + @if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1 && [ ! -f "$(SEAWEEDFS_ROOT)/weed/weed" ]; then \ + echo "$(RED)Error: SeaweedFS binary not found$(NC)"; \ + echo "Please build SeaweedFS first: make build-weed"; \ + exit 1; \ + fi + @echo "$(GREEN)SeaweedFS binary found$(NC)" + +start-seaweedfs: check-binary + @echo "$(YELLOW)Starting SeaweedFS server for TUS testing...$(NC)" + @# Clean up any existing processes on our test ports + @lsof -ti :$(MASTER_PORT) | xargs -r kill -TERM 2>/dev/null || true + @lsof -ti :$(VOLUME_PORT) | xargs -r kill -TERM 2>/dev/null || true + @lsof -ti :$(FILER_PORT) | xargs -r kill -TERM 2>/dev/null || true + @sleep 2 + + # Create necessary directories + @mkdir -p /tmp/seaweedfs-test-tus-master + @mkdir -p /tmp/seaweedfs-test-tus-volume + @mkdir -p /tmp/seaweedfs-test-tus-filer + + # Determine binary path + $(eval WEED_BIN := $(shell command -v $(SEAWEEDFS_BINARY) 2>/dev/null || echo "$(SEAWEEDFS_ROOT)/weed/weed")) + + # Start master server + @echo "Starting master server..." + @nohup $(WEED_BIN) master \ + -port=$(MASTER_PORT) \ + -mdir=/tmp/seaweedfs-test-tus-master \ + -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) \ + -ip=127.0.0.1 \ + > /tmp/seaweedfs-tus-master.log 2>&1 & + @sleep 3 + + # Start volume server + @echo "Starting volume server..." + @nohup $(WEED_BIN) volume \ + -port=$(VOLUME_PORT) \ + -mserver=127.0.0.1:$(MASTER_PORT) \ + -dir=/tmp/seaweedfs-test-tus-volume \ + -max=$(VOLUME_MAX_COUNT) \ + -ip=127.0.0.1 \ + > /tmp/seaweedfs-tus-volume.log 2>&1 & + @sleep 3 + + # Start filer server + @echo "Starting filer server..." + @nohup $(WEED_BIN) filer \ + -port=$(FILER_PORT) \ + -master=127.0.0.1:$(MASTER_PORT) \ + -ip=127.0.0.1 \ + > /tmp/seaweedfs-tus-filer.log 2>&1 & + @sleep 5 + + # Wait for filer to be ready + @echo "$(YELLOW)Waiting for filer to be ready...$(NC)" + @for i in $$(seq 1 30); do \ + if curl -s -f http://127.0.0.1:$(FILER_PORT)/ > /dev/null 2>&1; then \ + echo "$(GREEN)Filer is ready$(NC)"; \ + break; \ + fi; \ + if [ $$i -eq 30 ]; then \ + echo "$(RED)Filer failed to start within 30 seconds$(NC)"; \ + $(MAKE) debug-logs; \ + exit 1; \ + fi; \ + echo "Waiting for filer... ($$i/30)"; \ + sleep 1; \ + done + + @echo "$(GREEN)SeaweedFS server started successfully for TUS testing$(NC)" + @echo "Master: http://localhost:$(MASTER_PORT)" + @echo "Volume: http://localhost:$(VOLUME_PORT)" + @echo "Filer: http://localhost:$(FILER_PORT)" + @echo "TUS Endpoint: http://localhost:$(FILER_PORT)/.tus/" + +stop-seaweedfs: + @echo "$(YELLOW)Stopping SeaweedFS server...$(NC)" + @lsof -ti :$(MASTER_PORT) | xargs -r kill -TERM 2>/dev/null || true + @lsof -ti :$(VOLUME_PORT) | xargs -r kill -TERM 2>/dev/null || true + @lsof -ti :$(FILER_PORT) | xargs -r kill -TERM 2>/dev/null || true + @sleep 2 + @echo "$(GREEN)SeaweedFS server stopped$(NC)" + +clean: + @echo "$(YELLOW)Cleaning up TUS test artifacts...$(NC)" + @rm -rf /tmp/seaweedfs-test-tus-* + @rm -f /tmp/seaweedfs-tus-*.log + @echo "$(GREEN)TUS test cleanup completed$(NC)" + +# Run all tests +test: check-binary + @echo "$(YELLOW)Running all TUS integration tests...$(NC)" + @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/tus/... + @echo "$(GREEN)All TUS tests completed$(NC)" + +# Run basic upload tests +test-basic: check-binary + @echo "$(YELLOW)Running basic TUS upload tests...$(NC)" + @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusBasicUpload|TestTusOptionsHandler" ./test/tus/... + @echo "$(GREEN)Basic TUS tests completed$(NC)" + +# Run chunked upload tests +test-chunked: check-binary + @echo "$(YELLOW)Running chunked TUS upload tests...$(NC)" + @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusChunkedUpload" ./test/tus/... + @echo "$(GREEN)Chunked TUS tests completed$(NC)" + +# Run resume tests +test-resume: check-binary + @echo "$(YELLOW)Running TUS upload resume tests...$(NC)" + @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusResumeAfterInterruption|TestTusHeadRequest" ./test/tus/... + @echo "$(GREEN)TUS resume tests completed$(NC)" + +# Run error handling tests +test-errors: check-binary + @echo "$(YELLOW)Running TUS error handling tests...$(NC)" + @cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusInvalidOffset|TestTusUploadNotFound|TestTusDeleteUpload" ./test/tus/... + @echo "$(GREEN)TUS error tests completed$(NC)" + +# Run tests with automatic server management +test-with-server: build-weed + @echo "$(YELLOW)Running TUS tests with automatic server management...$(NC)" + @trap '$(MAKE) stop-seaweedfs clean || true' EXIT; \ + $(MAKE) start-seaweedfs && \ + sleep 3 && \ + cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/tus/... && \ + echo "$(GREEN)All TUS tests passed!$(NC)" + +# Debug targets +debug-logs: + @echo "$(YELLOW)=== Master Log ===$(NC)" + @tail -n 50 /tmp/seaweedfs-tus-master.log 2>/dev/null || echo "No master log found" + @echo "$(YELLOW)=== Volume Log ===$(NC)" + @tail -n 50 /tmp/seaweedfs-tus-volume.log 2>/dev/null || echo "No volume log found" + @echo "$(YELLOW)=== Filer Log ===$(NC)" + @tail -n 50 /tmp/seaweedfs-tus-filer.log 2>/dev/null || echo "No filer log found" + +debug-status: + @echo "$(YELLOW)=== Process Status ===$(NC)" + @ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found" + @echo "$(YELLOW)=== Port Status ===$(NC)" + @lsof -i :$(MASTER_PORT) -i :$(VOLUME_PORT) -i :$(FILER_PORT) 2>/dev/null || echo "No ports in use" + +# Manual testing targets +manual-start: start-seaweedfs + @echo "$(GREEN)SeaweedFS is now running for manual TUS testing$(NC)" + @echo "" + @echo "TUS Endpoints:" + @echo " OPTIONS /.tus/ - Capability discovery" + @echo " POST /.tus/{path} - Create upload" + @echo " HEAD /.tus/.uploads/{id} - Get offset" + @echo " PATCH /.tus/.uploads/{id} - Upload data" + @echo " DELETE /.tus/.uploads/{id} - Cancel upload" + @echo "" + @echo "Example curl commands:" + @echo " curl -X OPTIONS http://localhost:$(FILER_PORT)/.tus/ -H 'Tus-Resumable: 1.0.0'" + @echo "" + @echo "Run 'make manual-stop' when finished" + +manual-stop: stop-seaweedfs clean + +# CI targets +ci-test: test-with-server + +# Skip integration tests (short mode) +test-short: + @echo "$(YELLOW)Running TUS tests in short mode (skipping integration tests)...$(NC)" + @cd $(SEAWEEDFS_ROOT) && go test -v -short ./test/tus/... + @echo "$(GREEN)Short tests completed$(NC)" + diff --git a/test/tus/README.md b/test/tus/README.md index 7477168d0..e8913d82f 100644 --- a/test/tus/README.md +++ b/test/tus/README.md @@ -2,64 +2,225 @@ This directory contains integration tests for the TUS (resumable upload) protocol support in SeaweedFS Filer. -## TUS Protocol Overview +## Overview -TUS is an open protocol for resumable file uploads. It allows clients to upload files in chunks and resume uploads after network failures. +TUS is an open protocol for resumable file uploads over HTTP. It allows clients to upload files in chunks and resume uploads after network failures or interruptions. -Key endpoints: -- `OPTIONS /tus/` - Server capability discovery -- `POST /tus/{path}` - Create new upload session -- `HEAD /tus/{upload-id}` - Get current upload offset -- `PATCH /tus/{upload-id}` - Upload data at offset -- `DELETE /tus/{upload-id}` - Cancel upload +### Why TUS? -## Prerequisites +- **Resumable uploads**: Resume interrupted uploads without re-sending data +- **Chunked uploads**: Upload large files in smaller pieces +- **Simple protocol**: Standard HTTP methods with custom headers +- **Wide client support**: Libraries available for JavaScript, Python, Go, and more -1. Build the weed binary: +## TUS Protocol Endpoints + +| Method | Path | Description | +|--------|------|-------------| +| `OPTIONS` | `/.tus/` | Server capability discovery | +| `POST` | `/.tus/{path}` | Create new upload session | +| `HEAD` | `/.tus/.uploads/{id}` | Get current upload offset | +| `PATCH` | `/.tus/.uploads/{id}` | Upload data at offset | +| `DELETE` | `/.tus/.uploads/{id}` | Cancel upload | + +### TUS Headers + +**Request Headers:** +- `Tus-Resumable: 1.0.0` - Protocol version (required) +- `Upload-Length` - Total file size in bytes (required on POST) +- `Upload-Offset` - Current byte offset (required on PATCH) +- `Upload-Metadata` - Base64-encoded key-value pairs (optional) +- `Content-Type: application/offset+octet-stream` (required on PATCH) + +**Response Headers:** +- `Tus-Resumable` - Protocol version +- `Tus-Version` - Supported versions +- `Tus-Extension` - Supported extensions +- `Tus-Max-Size` - Maximum upload size +- `Upload-Offset` - Current byte offset +- `Location` - Upload URL (on POST) + +## Test Structure + +### Integration Tests + +The tests cover: + +1. **Basic Functionality** + - `TestTusOptionsHandler` - Capability discovery + - `TestTusBasicUpload` - Simple complete upload + - `TestTusCreationWithUpload` - Creation-with-upload extension + +2. **Chunked Uploads** + - `TestTusChunkedUpload` - Upload in multiple chunks + +3. **Resumable Uploads** + - `TestTusHeadRequest` - Offset tracking + - `TestTusResumeAfterInterruption` - Resume after failure + +4. **Error Handling** + - `TestTusInvalidOffset` - Offset mismatch (409 Conflict) + - `TestTusUploadNotFound` - Missing upload (404 Not Found) + - `TestTusDeleteUpload` - Upload cancellation + +## Running Tests + +### Prerequisites + +1. **Build SeaweedFS**: ```bash -cd ../../weed -go build +make build-weed +# or +cd ../../weed && go build -o weed ``` -2. The tests will automatically start required servers (master, volume, filer). +### Using Makefile -## Running Tests +```bash +# Show available targets +make help + +# Run all tests with automatic server management +make test-with-server + +# Run all tests (requires running server) +make test + +# Run specific test categories +make test-basic # Basic upload tests +make test-chunked # Chunked upload tests +make test-resume # Resume/HEAD tests +make test-errors # Error handling tests + +# Manual testing +make manual-start # Start SeaweedFS for manual testing +make manual-stop # Stop and cleanup +``` + +### Using Go Test Directly -### Run all TUS tests: ```bash +# Run all TUS tests go test -v ./test/tus/... + +# Run specific test +go test -v ./test/tus -run TestTusBasicUpload + +# Skip integration tests (short mode) +go test -v -short ./test/tus/... ``` -### Run specific test: +### Debug + ```bash -go test -v ./test/tus -run TestTusBasicUpload +# View server logs +make debug-logs + +# Check process and port status +make debug-status ``` -### Skip integration tests (short mode): +## Test Environment + +Each test run: +1. Starts a SeaweedFS cluster (master, volume, filer) +2. Creates uploads using TUS protocol +3. Verifies files are stored correctly +4. Cleans up test data + +### Default Ports + +| Service | Port | +|---------|------| +| Master | 19333 | +| Volume | 18080 | +| Filer | 18888 | + +### Configuration + +Override defaults via environment or Makefile variables: ```bash -go test -v -short ./test/tus/... +FILER_PORT=8889 MASTER_PORT=9334 make test ``` -## Test Coverage +## Example Usage -The tests cover: -- Basic upload creation and completion -- Chunked/resumable uploads -- Upload offset tracking (HEAD requests) -- Upload cancellation (DELETE requests) -- Error handling (invalid offsets, missing uploads) -- Large file uploads with multiple chunks -- Concurrent uploads -- Metadata handling - -## TUS Protocol Headers - -Required headers for TUS requests: -- `Tus-Resumable: 1.0.0` - Protocol version -- `Upload-Length` - Total file size (on creation) -- `Upload-Offset` - Current byte offset (on PATCH) -- `Content-Type: application/offset+octet-stream` (on PATCH) - -Optional headers: -- `Upload-Metadata` - Base64-encoded key-value pairs +### Create Upload + +```bash +curl -X POST http://localhost:18888/.tus/mydir/file.txt \ + -H "Tus-Resumable: 1.0.0" \ + -H "Upload-Length: 1000" \ + -H "Upload-Metadata: filename dGVzdC50eHQ=" +``` + +### Upload Data + +```bash +curl -X PATCH http://localhost:18888/.tus/.uploads/{upload-id} \ + -H "Tus-Resumable: 1.0.0" \ + -H "Upload-Offset: 0" \ + -H "Content-Type: application/offset+octet-stream" \ + --data-binary @file.txt +``` + +### Check Offset + +```bash +curl -I http://localhost:18888/.tus/.uploads/{upload-id} \ + -H "Tus-Resumable: 1.0.0" +``` + +### Cancel Upload + +```bash +curl -X DELETE http://localhost:18888/.tus/.uploads/{upload-id} \ + -H "Tus-Resumable: 1.0.0" +``` + +## TUS Extensions Supported + +- **creation**: Create new uploads with POST +- **creation-with-upload**: Send data in creation request +- **termination**: Cancel uploads with DELETE + +## Architecture + +``` +Client Filer Volume Servers + | | | + |-- POST /.tus/path/file.mp4 ->| | + | |-- Create session dir ------->| + |<-- 201 Location: /.../{id} --| | + | | | + |-- PATCH /.tus/.uploads/{id} >| | + | Upload-Offset: 0 |-- Assign volume ------------>| + | [chunk data] |-- Upload chunk ------------->| + |<-- 204 Upload-Offset: N -----| | + | | | + | (network failure) | | + | | | + |-- HEAD /.tus/.uploads/{id} ->| | + |<-- Upload-Offset: N ---------| | + | | | + |-- PATCH (resume) ----------->|-- Upload remaining -------->| + |<-- 204 (complete) -----------|-- Assemble final file ----->| +``` + +## Comparison with S3 Multipart + +| Feature | TUS | S3 Multipart | +|---------|-----|--------------| +| Protocol | Custom HTTP headers | S3 API | +| Session Init | POST with Upload-Length | CreateMultipartUpload | +| Upload Data | PATCH with offset | UploadPart with partNumber | +| Resume | HEAD to get offset | ListParts | +| Complete | Automatic at final offset | CompleteMultipartUpload | +| Ordering | Sequential (offset-based) | Parallel (part numbers) | + +## Related Resources +- [TUS Protocol Specification](https://tus.io/protocols/resumable-upload) +- [tus-js-client](https://github.com/tus/tus-js-client) - JavaScript client +- [go-tus](https://github.com/eventials/go-tus) - Go client +- [SeaweedFS S3 API](../../weed/s3api) - Alternative multipart upload From 45b32ad0c7d46dd6adfa8a338c57b6ba787e97d8 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 13:18:02 -0800 Subject: [PATCH 06/16] Fix TUS integration tests and creation-with-upload - Fix test URLs to use full URLs instead of relative paths - Fix creation-with-upload to refresh session before completing - Fix Makefile to properly handle test cleanup - Add FullURL helper function to TestCluster --- .gitignore | 1 + test/tus/Makefile | 28 ++++++++++----------- test/tus/tus_integration_test.go | 32 +++++++++++++++--------- weed/server/filer_server_tus_handlers.go | 7 ++++++ 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index cd240ab6d..81b4b107d 100644 --- a/.gitignore +++ b/.gitignore @@ -124,3 +124,4 @@ ADVANCED_IAM_DEVELOPMENT_PLAN.md *.log weed-iam test/kafka/kafka-client-loadtest/weed-linux-arm64 +/test/tus/filerldb2 diff --git a/test/tus/Makefile b/test/tus/Makefile index fd0b6d6b7..d07d6c2b6 100644 --- a/test/tus/Makefile +++ b/test/tus/Makefile @@ -64,9 +64,9 @@ check-binary: start-seaweedfs: check-binary @echo "$(YELLOW)Starting SeaweedFS server for TUS testing...$(NC)" @# Clean up any existing processes on our test ports - @lsof -ti :$(MASTER_PORT) | xargs -r kill -TERM 2>/dev/null || true - @lsof -ti :$(VOLUME_PORT) | xargs -r kill -TERM 2>/dev/null || true - @lsof -ti :$(FILER_PORT) | xargs -r kill -TERM 2>/dev/null || true + @lsof -ti :$(MASTER_PORT) | xargs kill -TERM 2>/dev/null || true + @lsof -ti :$(VOLUME_PORT) | xargs kill -TERM 2>/dev/null || true + @lsof -ti :$(FILER_PORT) | xargs kill -TERM 2>/dev/null || true @sleep 2 # Create necessary directories @@ -74,12 +74,9 @@ start-seaweedfs: check-binary @mkdir -p /tmp/seaweedfs-test-tus-volume @mkdir -p /tmp/seaweedfs-test-tus-filer - # Determine binary path - $(eval WEED_BIN := $(shell command -v $(SEAWEEDFS_BINARY) 2>/dev/null || echo "$(SEAWEEDFS_ROOT)/weed/weed")) - - # Start master server + # Start master server (use freshly built binary) @echo "Starting master server..." - @nohup $(WEED_BIN) master \ + @nohup $(SEAWEEDFS_ROOT)/weed/weed master \ -port=$(MASTER_PORT) \ -mdir=/tmp/seaweedfs-test-tus-master \ -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) \ @@ -89,7 +86,7 @@ start-seaweedfs: check-binary # Start volume server @echo "Starting volume server..." - @nohup $(WEED_BIN) volume \ + @nohup $(SEAWEEDFS_ROOT)/weed/weed volume \ -port=$(VOLUME_PORT) \ -mserver=127.0.0.1:$(MASTER_PORT) \ -dir=/tmp/seaweedfs-test-tus-volume \ @@ -100,7 +97,7 @@ start-seaweedfs: check-binary # Start filer server @echo "Starting filer server..." - @nohup $(WEED_BIN) filer \ + @nohup $(SEAWEEDFS_ROOT)/weed/weed filer \ -port=$(FILER_PORT) \ -master=127.0.0.1:$(MASTER_PORT) \ -ip=127.0.0.1 \ @@ -176,11 +173,14 @@ test-errors: check-binary # Run tests with automatic server management test-with-server: build-weed @echo "$(YELLOW)Running TUS tests with automatic server management...$(NC)" - @trap '$(MAKE) stop-seaweedfs clean || true' EXIT; \ - $(MAKE) start-seaweedfs && \ + @$(MAKE) -C $(TEST_DIR) start-seaweedfs && \ sleep 3 && \ - cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/tus/... && \ - echo "$(GREEN)All TUS tests passed!$(NC)" + cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/tus/...; \ + TEST_RESULT=$$?; \ + $(MAKE) -C $(TEST_DIR) stop-seaweedfs; \ + $(MAKE) -C $(TEST_DIR) clean; \ + if [ $$TEST_RESULT -eq 0 ]; then echo "$(GREEN)All TUS tests passed!$(NC)"; fi; \ + exit $$TEST_RESULT # Debug targets debug-logs: diff --git a/test/tus/tus_integration_test.go b/test/tus/tus_integration_test.go index ac5348087..b95d36345 100644 --- a/test/tus/tus_integration_test.go +++ b/test/tus/tus_integration_test.go @@ -57,6 +57,14 @@ func (c *TestCluster) TusURL() string { return fmt.Sprintf("%s/.tus", c.FilerURL()) } +// FullURL converts a relative path to a full URL +func (c *TestCluster) FullURL(path string) string { + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + return path + } + return fmt.Sprintf("http://127.0.0.1:%s%s", testFilerPort, path) +} + // startTestCluster starts a SeaweedFS cluster for testing func startTestCluster(t *testing.T, ctx context.Context) (*TestCluster, error) { weedBinary := findWeedBinary() @@ -260,7 +268,7 @@ func TestTusBasicUpload(t *testing.T) { t.Logf("Upload location: %s", uploadLocation) // Step 2: Upload data (PATCH) - patchReq, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(testData)) + patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData)) require.NoError(t, err) patchReq.Header.Set("Tus-Resumable", TusVersion) patchReq.Header.Set("Upload-Offset", "0") @@ -336,7 +344,7 @@ func TestTusChunkedUpload(t *testing.T) { } chunk := testData[offset:end] - patchReq, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(chunk)) + patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(chunk)) require.NoError(t, err) patchReq.Header.Set("Tus-Resumable", TusVersion) patchReq.Header.Set("Upload-Offset", strconv.Itoa(offset)) @@ -400,7 +408,7 @@ func TestTusHeadRequest(t *testing.T) { uploadLocation := createResp.Header.Get("Location") // HEAD before any data uploaded - offset should be 0 - headReq1, err := http.NewRequest(http.MethodHead, uploadLocation, nil) + headReq1, err := http.NewRequest(http.MethodHead, cluster.FullURL(uploadLocation), nil) require.NoError(t, err) headReq1.Header.Set("Tus-Resumable", TusVersion) @@ -414,7 +422,7 @@ func TestTusHeadRequest(t *testing.T) { // Upload half the data halfLen := len(testData) / 2 - patchReq, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(testData[:halfLen])) + patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData[:halfLen])) require.NoError(t, err) patchReq.Header.Set("Tus-Resumable", TusVersion) patchReq.Header.Set("Upload-Offset", "0") @@ -426,7 +434,7 @@ func TestTusHeadRequest(t *testing.T) { require.Equal(t, http.StatusNoContent, patchResp.StatusCode) // HEAD after partial upload - offset should be halfLen - headReq2, err := http.NewRequest(http.MethodHead, uploadLocation, nil) + headReq2, err := http.NewRequest(http.MethodHead, cluster.FullURL(uploadLocation), nil) require.NoError(t, err) headReq2.Header.Set("Tus-Resumable", TusVersion) @@ -472,7 +480,7 @@ func TestTusDeleteUpload(t *testing.T) { uploadLocation := createResp.Header.Get("Location") // Upload some data - patchReq, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(testData[:10])) + patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData[:10])) require.NoError(t, err) patchReq.Header.Set("Tus-Resumable", TusVersion) patchReq.Header.Set("Upload-Offset", "0") @@ -483,7 +491,7 @@ func TestTusDeleteUpload(t *testing.T) { patchResp.Body.Close() // Delete the upload - deleteReq, err := http.NewRequest(http.MethodDelete, uploadLocation, nil) + deleteReq, err := http.NewRequest(http.MethodDelete, cluster.FullURL(uploadLocation), nil) require.NoError(t, err) deleteReq.Header.Set("Tus-Resumable", TusVersion) @@ -494,7 +502,7 @@ func TestTusDeleteUpload(t *testing.T) { assert.Equal(t, http.StatusNoContent, deleteResp.StatusCode, "DELETE should return 204") // Verify upload is gone - HEAD should return 404 - headReq, err := http.NewRequest(http.MethodHead, uploadLocation, nil) + headReq, err := http.NewRequest(http.MethodHead, cluster.FullURL(uploadLocation), nil) require.NoError(t, err) headReq.Header.Set("Tus-Resumable", TusVersion) @@ -538,7 +546,7 @@ func TestTusInvalidOffset(t *testing.T) { uploadLocation := createResp.Header.Get("Location") // Try to upload with wrong offset (should be 0, but we send 100) - patchReq, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(testData)) + patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData)) require.NoError(t, err) patchReq.Header.Set("Tus-Resumable", TusVersion) patchReq.Header.Set("Upload-Offset", "100") // Wrong offset! @@ -687,7 +695,7 @@ func TestTusResumeAfterInterruption(t *testing.T) { // Upload first 20KB firstChunkSize := 20 * 1024 - patchReq1, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(testData[:firstChunkSize])) + patchReq1, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData[:firstChunkSize])) require.NoError(t, err) patchReq1.Header.Set("Tus-Resumable", TusVersion) patchReq1.Header.Set("Upload-Offset", "0") @@ -701,7 +709,7 @@ func TestTusResumeAfterInterruption(t *testing.T) { t.Log("Simulating network interruption...") // Simulate resumption: Query current offset with HEAD - headReq, err := http.NewRequest(http.MethodHead, uploadLocation, nil) + headReq, err := http.NewRequest(http.MethodHead, cluster.FullURL(uploadLocation), nil) require.NoError(t, err) headReq.Header.Set("Tus-Resumable", TusVersion) @@ -715,7 +723,7 @@ func TestTusResumeAfterInterruption(t *testing.T) { require.Equal(t, firstChunkSize, currentOffset) // Resume upload from current offset - patchReq2, err := http.NewRequest(http.MethodPatch, uploadLocation, bytes.NewReader(testData[currentOffset:])) + patchReq2, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData[currentOffset:])) require.NoError(t, err) patchReq2.Header.Set("Tus-Resumable", TusVersion) patchReq2.Header.Set("Upload-Offset", strconv.Itoa(currentOffset)) diff --git a/weed/server/filer_server_tus_handlers.go b/weed/server/filer_server_tus_handlers.go index 8a426941e..5a95b249f 100644 --- a/weed/server/filer_server_tus_handlers.go +++ b/weed/server/filer_server_tus_handlers.go @@ -135,6 +135,13 @@ func (fs *FilerServer) tusCreateHandler(w http.ResponseWriter, r *http.Request) // Check if upload is complete if bytesWritten == session.Size { + // Refresh session to get updated chunks + session, err = fs.getTusSession(ctx, uploadID) + if err != nil { + glog.Errorf("Failed to get updated TUS session: %v", err) + http.Error(w, "Failed to complete upload", http.StatusInternalServerError) + return + } if err := fs.completeTusUpload(ctx, session); err != nil { glog.Errorf("Failed to complete TUS upload: %v", err) http.Error(w, "Failed to complete upload", http.StatusInternalServerError) From a91159259dd97062d5445023995fff8caecb36fb Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 13:20:28 -0800 Subject: [PATCH 07/16] Add TUS protocol tests to GitHub Actions CI - Add tus-tests.yml workflow that runs on PRs and pushes - Runs when TUS-related files are modified - Automatic server management for integration testing - Upload logs on failure for debugging --- .github/workflows/tus-tests.yml | 114 ++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 .github/workflows/tus-tests.yml diff --git a/.github/workflows/tus-tests.yml b/.github/workflows/tus-tests.yml new file mode 100644 index 000000000..5e6041f48 --- /dev/null +++ b/.github/workflows/tus-tests.yml @@ -0,0 +1,114 @@ +name: "TUS Protocol Tests" + +on: + pull_request: + paths: + - 'weed/server/filer_server_tus*.go' + - 'weed/server/filer_server.go' + - 'test/tus/**' + - '.github/workflows/tus-tests.yml' + push: + branches: [ master, main ] + paths: + - 'weed/server/filer_server_tus*.go' + - 'weed/server/filer_server.go' + - 'test/tus/**' + +concurrency: + group: ${{ github.head_ref }}/tus-tests + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: weed + +jobs: + tus-integration-tests: + name: TUS Protocol Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run TUS Integration Tests + timeout-minutes: 15 + working-directory: test/tus + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting TUS Tests ===" + + # Run tests with automatic server management + make test-with-server || { + echo "TUS integration tests failed, checking logs..." + if [ -f /tmp/seaweedfs-tus-filer.log ]; then + echo "=== Filer logs ===" + tail -100 /tmp/seaweedfs-tus-filer.log + fi + if [ -f /tmp/seaweedfs-tus-master.log ]; then + echo "=== Master logs ===" + tail -50 /tmp/seaweedfs-tus-master.log + fi + if [ -f /tmp/seaweedfs-tus-volume.log ]; then + echo "=== Volume logs ===" + tail -50 /tmp/seaweedfs-tus-volume.log + fi + exit 1 + } + + - name: Show server logs on failure + if: failure() + working-directory: test/tus + run: | + echo "=== Filer Server Logs ===" + if [ -f /tmp/seaweedfs-tus-filer.log ]; then + echo "Last 100 lines of filer logs:" + tail -100 /tmp/seaweedfs-tus-filer.log + else + echo "No filer log file found" + fi + + echo "=== Master Server Logs ===" + if [ -f /tmp/seaweedfs-tus-master.log ]; then + tail -50 /tmp/seaweedfs-tus-master.log + else + echo "No master log file found" + fi + + echo "=== Volume Server Logs ===" + if [ -f /tmp/seaweedfs-tus-volume.log ]; then + tail -50 /tmp/seaweedfs-tus-volume.log + else + echo "No volume log file found" + fi + + echo "=== Test Environment ===" + ps aux | grep -E "(weed|test)" || true + netstat -tlnp 2>/dev/null | grep -E "(18888|19333|18080)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v5 + with: + name: tus-test-logs + path: | + /tmp/seaweedfs-tus-*.log + retention-days: 3 From 31214b2874d7c247a70de17dc12551697c784ca7 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 13:27:37 -0800 Subject: [PATCH 08/16] Make TUS base path configurable via CLI - Add -tus.path CLI flag to filer command - TUS is disabled by default (empty path) - Example: -tus.path=/.tus to enable at /.tus endpoint - Update test Makefile to use -tus.path flag - Update README with TUS enabling instructions --- test/tus/Makefile | 3 ++- test/tus/README.md | 14 ++++++++++++++ weed/command/filer.go | 3 +++ weed/server/filer_server.go | 9 ++++++++- weed/server/filer_server_tus_handlers.go | 22 ++++++++++++++++------ 5 files changed, 43 insertions(+), 8 deletions(-) diff --git a/test/tus/Makefile b/test/tus/Makefile index d07d6c2b6..1a944a4f0 100644 --- a/test/tus/Makefile +++ b/test/tus/Makefile @@ -95,12 +95,13 @@ start-seaweedfs: check-binary > /tmp/seaweedfs-tus-volume.log 2>&1 & @sleep 3 - # Start filer server + # Start filer server with TUS enabled @echo "Starting filer server..." @nohup $(SEAWEEDFS_ROOT)/weed/weed filer \ -port=$(FILER_PORT) \ -master=127.0.0.1:$(MASTER_PORT) \ -ip=127.0.0.1 \ + -tus.path=/.tus \ > /tmp/seaweedfs-tus-filer.log 2>&1 & @sleep 5 diff --git a/test/tus/README.md b/test/tus/README.md index e8913d82f..e948c06d1 100644 --- a/test/tus/README.md +++ b/test/tus/README.md @@ -40,6 +40,20 @@ TUS is an open protocol for resumable file uploads over HTTP. It allows clients - `Upload-Offset` - Current byte offset - `Location` - Upload URL (on POST) +## Enabling TUS + +TUS protocol support must be explicitly enabled when starting the filer server using the `-tus.path` flag: + +```bash +# Start filer with TUS enabled at /.tus path +weed filer -master=localhost:9333 -tus.path=/.tus + +# Or use a custom path +weed filer -master=localhost:9333 -tus.path=/uploads/tus +``` + +If `-tus.path` is not specified, TUS endpoints are disabled. + ## Test Structure ### Integration Tests diff --git a/weed/command/filer.go b/weed/command/filer.go index 86991a181..d60312f5e 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -74,6 +74,7 @@ type FilerOptions struct { diskType *string allowedOrigins *string exposeDirectoryData *bool + tusPath *string certProvider certprovider.Provider } @@ -109,6 +110,7 @@ func init() { f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins") f.exposeDirectoryData = cmdFiler.Flag.Bool("exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI") + f.tusPath = cmdFiler.Flag.String("tus.path", "", "TUS resumable upload endpoint path, e.g., /.tus (disabled if empty)") // start s3 on filer filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway") @@ -334,6 +336,7 @@ func (fo *FilerOptions) startFiler() { DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024, DiskType: *fo.diskType, AllowedOrigins: strings.Split(*fo.allowedOrigins, ","), + TusPath: *fo.tusPath, }) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index a76994b89..d35485ad2 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -79,6 +79,7 @@ type FilerOption struct { DiskType string AllowedOrigins []string ExposeDirectoryData bool + TusPath string } type FilerServer struct { @@ -196,7 +197,13 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) if !option.DisableHttp { defaultMux.HandleFunc("/healthz", requestIDMiddleware(fs.filerHealthzHandler)) // TUS resumable upload protocol handler - defaultMux.HandleFunc("/.tus/", fs.filerGuard.WhiteList(requestIDMiddleware(fs.tusHandler))) + if option.TusPath != "" { + tusPath := option.TusPath + if !strings.HasSuffix(tusPath, "/") { + tusPath += "/" + } + defaultMux.HandleFunc(tusPath, fs.filerGuard.WhiteList(requestIDMiddleware(fs.tusHandler))) + } defaultMux.HandleFunc("/", fs.filerGuard.WhiteList(requestIDMiddleware(fs.filerHandler))) } if defaultMux != readonlyMux { diff --git a/weed/server/filer_server_tus_handlers.go b/weed/server/filer_server_tus_handlers.go index 5a95b249f..8fe93a6e7 100644 --- a/weed/server/filer_server_tus_handlers.go +++ b/weed/server/filer_server_tus_handlers.go @@ -35,11 +35,15 @@ func (fs *FilerServer) tusHandler(w http.ResponseWriter, r *http.Request) { // Route based on method and path path := r.URL.Path - tusPrefix := "/.tus" + tusPrefix := fs.option.TusPath + if tusPrefix == "" { + tusPrefix = "/.tus" + } - // Check if this is an upload location (contains upload ID after /.tus/.uploads/) - if strings.HasPrefix(path, tusPrefix+"/.uploads/") { - uploadID := strings.TrimPrefix(path, tusPrefix+"/.uploads/") + // Check if this is an upload location (contains upload ID after {tusPrefix}/.uploads/) + uploadsPrefix := tusPrefix + "/.uploads/" + if strings.HasPrefix(path, uploadsPrefix) { + uploadID := strings.TrimPrefix(path, uploadsPrefix) uploadID = strings.Split(uploadID, "/")[0] // Get just the ID, not any trailing path switch r.Method { @@ -97,8 +101,14 @@ func (fs *FilerServer) tusCreateHandler(w http.ResponseWriter, r *http.Request) // Parse Upload-Metadata header (optional) metadata := parseTusMetadata(r.Header.Get("Upload-Metadata")) + // Get TUS path prefix + tusPrefix := fs.option.TusPath + if tusPrefix == "" { + tusPrefix = "/.tus" + } + // Determine target path from request URL - targetPath := strings.TrimPrefix(r.URL.Path, "/.tus") + targetPath := strings.TrimPrefix(r.URL.Path, tusPrefix) if targetPath == "" || targetPath == "/" { http.Error(w, "Target path required", http.StatusBadRequest) return @@ -116,7 +126,7 @@ func (fs *FilerServer) tusCreateHandler(w http.ResponseWriter, r *http.Request) } // Build upload location URL - uploadLocation := fmt.Sprintf("/.tus/.uploads/%s", uploadID) + uploadLocation := fmt.Sprintf("%s/.uploads/%s", tusPrefix, uploadID) // Handle creation-with-upload extension if r.ContentLength > 0 && r.Header.Get("Content-Type") == "application/offset+octet-stream" { From 42b173fff9b8ffa84878cbf27223f7d583c26005 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 13:34:43 -0800 Subject: [PATCH 09/16] Rename -tus.path to -tusBasePath with default .tus - Rename CLI flag from -tus.path to -tusBasePath - Default to .tus (TUS enabled by default) - Add -filer.tusBasePath option to weed server command - Properly handle path prefix (prepend / if missing) --- test/tus/Makefile | 3 +-- test/tus/README.md | 15 ++++++++------- weed/command/filer.go | 2 +- weed/command/server.go | 1 + weed/server/filer_server.go | 3 +++ weed/server/filer_server_tus_handlers.go | 11 ++++++++--- 6 files changed, 22 insertions(+), 13 deletions(-) diff --git a/test/tus/Makefile b/test/tus/Makefile index 1a944a4f0..71b05e8ab 100644 --- a/test/tus/Makefile +++ b/test/tus/Makefile @@ -95,13 +95,12 @@ start-seaweedfs: check-binary > /tmp/seaweedfs-tus-volume.log 2>&1 & @sleep 3 - # Start filer server with TUS enabled + # Start filer server with TUS enabled (default tusBasePath is .tus) @echo "Starting filer server..." @nohup $(SEAWEEDFS_ROOT)/weed/weed filer \ -port=$(FILER_PORT) \ -master=127.0.0.1:$(MASTER_PORT) \ -ip=127.0.0.1 \ - -tus.path=/.tus \ > /tmp/seaweedfs-tus-filer.log 2>&1 & @sleep 5 diff --git a/test/tus/README.md b/test/tus/README.md index e948c06d1..cd014229e 100644 --- a/test/tus/README.md +++ b/test/tus/README.md @@ -42,17 +42,18 @@ TUS is an open protocol for resumable file uploads over HTTP. It allows clients ## Enabling TUS -TUS protocol support must be explicitly enabled when starting the filer server using the `-tus.path` flag: +TUS protocol support is enabled by default at `/.tus` path. You can customize the path using the `-tusBasePath` flag: ```bash -# Start filer with TUS enabled at /.tus path -weed filer -master=localhost:9333 -tus.path=/.tus +# Start filer with default TUS path (/.tus) +weed filer -master=localhost:9333 -# Or use a custom path -weed filer -master=localhost:9333 -tus.path=/uploads/tus -``` +# Use a custom path +weed filer -master=localhost:9333 -tusBasePath=uploads/tus -If `-tus.path` is not specified, TUS endpoints are disabled. +# Disable TUS by setting empty path +weed filer -master=localhost:9333 -tusBasePath= +``` ## Test Structure diff --git a/weed/command/filer.go b/weed/command/filer.go index d60312f5e..314bb83b6 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -110,7 +110,7 @@ func init() { f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins") f.exposeDirectoryData = cmdFiler.Flag.Bool("exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI") - f.tusPath = cmdFiler.Flag.String("tus.path", "", "TUS resumable upload endpoint path, e.g., /.tus (disabled if empty)") + f.tusPath = cmdFiler.Flag.String("tusBasePath", ".tus", "TUS resumable upload endpoint base path") // start s3 on filer filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway") diff --git a/weed/command/server.go b/weed/command/server.go index 47df30fc2..52f47ec32 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -129,6 +129,7 @@ func init() { filerOptions.downloadMaxMBps = cmdServer.Flag.Int("filer.downloadMaxMBps", 0, "download max speed for each download request, in MB per second") filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") filerOptions.exposeDirectoryData = cmdServer.Flag.Bool("filer.exposeDirectoryData", true, "expose directory data via filer. If false, filer UI will be innaccessible.") + filerOptions.tusPath = cmdServer.Flag.String("filer.tusBasePath", ".tus", "TUS resumable upload endpoint base path") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port") diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index d35485ad2..3d2db00ad 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -199,6 +199,9 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) // TUS resumable upload protocol handler if option.TusPath != "" { tusPath := option.TusPath + if !strings.HasPrefix(tusPath, "/") { + tusPath = "/" + tusPath + } if !strings.HasSuffix(tusPath, "/") { tusPath += "/" } diff --git a/weed/server/filer_server_tus_handlers.go b/weed/server/filer_server_tus_handlers.go index 8fe93a6e7..294204696 100644 --- a/weed/server/filer_server_tus_handlers.go +++ b/weed/server/filer_server_tus_handlers.go @@ -37,7 +37,10 @@ func (fs *FilerServer) tusHandler(w http.ResponseWriter, r *http.Request) { path := r.URL.Path tusPrefix := fs.option.TusPath if tusPrefix == "" { - tusPrefix = "/.tus" + tusPrefix = ".tus" + } + if !strings.HasPrefix(tusPrefix, "/") { + tusPrefix = "/" + tusPrefix } // Check if this is an upload location (contains upload ID after {tusPrefix}/.uploads/) @@ -104,7 +107,10 @@ func (fs *FilerServer) tusCreateHandler(w http.ResponseWriter, r *http.Request) // Get TUS path prefix tusPrefix := fs.option.TusPath if tusPrefix == "" { - tusPrefix = "/.tus" + tusPrefix = ".tus" + } + if !strings.HasPrefix(tusPrefix, "/") { + tusPrefix = "/" + tusPrefix } // Determine target path from request URL @@ -366,4 +372,3 @@ func parseTusMetadata(header string) map[string]string { return metadata } - From f45918c3ea9617ae864eac02b44a4cd4301ce163 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 13:42:39 -0800 Subject: [PATCH 10/16] Address code review comments - Sort chunks by offset before assembling final file - Use chunk.Offset directly instead of recalculating - Return error on invalid file ID instead of skipping - Require Content-Length header for PATCH requests - Use fs.option.Cipher for encryption setting - Detect MIME type from data using http.DetectContentType - Fix concurrency group for push events in workflow - Use os.Interrupt instead of Kill for graceful shutdown in tests --- .github/workflows/tus-tests.yml | 2 +- test/tus/tus_integration_test.go | 6 +++--- weed/server/filer_server_tus_handlers.go | 13 +++++++++++-- weed/server/filer_server_tus_session.go | 13 ++++++++----- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tus-tests.yml b/.github/workflows/tus-tests.yml index 5e6041f48..7fa8818ce 100644 --- a/.github/workflows/tus-tests.yml +++ b/.github/workflows/tus-tests.yml @@ -15,7 +15,7 @@ on: - 'test/tus/**' concurrency: - group: ${{ github.head_ref }}/tus-tests + group: ${{ github.head_ref || github.ref }}/tus-tests cancel-in-progress: true permissions: diff --git a/test/tus/tus_integration_test.go b/test/tus/tus_integration_test.go index b95d36345..db3576f9b 100644 --- a/test/tus/tus_integration_test.go +++ b/test/tus/tus_integration_test.go @@ -36,15 +36,15 @@ type TestCluster struct { func (c *TestCluster) Stop() { if c.filerCmd != nil && c.filerCmd.Process != nil { - c.filerCmd.Process.Kill() + c.filerCmd.Process.Signal(os.Interrupt) c.filerCmd.Wait() } if c.volumeCmd != nil && c.volumeCmd.Process != nil { - c.volumeCmd.Process.Kill() + c.volumeCmd.Process.Signal(os.Interrupt) c.volumeCmd.Wait() } if c.masterCmd != nil && c.masterCmd.Process != nil { - c.masterCmd.Process.Kill() + c.masterCmd.Process.Signal(os.Interrupt) c.masterCmd.Wait() } } diff --git a/weed/server/filer_server_tus_handlers.go b/weed/server/filer_server_tus_handlers.go index 294204696..f4fa5959e 100644 --- a/weed/server/filer_server_tus_handlers.go +++ b/weed/server/filer_server_tus_handlers.go @@ -222,6 +222,12 @@ func (fs *FilerServer) tusPatchHandler(w http.ResponseWriter, r *http.Request, u return } + // TUS requires Content-Length header for PATCH requests + if r.ContentLength < 0 { + http.Error(w, "Content-Length header required", http.StatusBadRequest) + return + } + // Write data bytesWritten, err := fs.tusWriteData(ctx, session, uploadOffset, r.Body, r.ContentLength) if err != nil { @@ -309,12 +315,15 @@ func (fs *FilerServer) tusWriteData(ctx context.Context, session *TusSession, of return 0, fmt.Errorf("create uploader: %w", uploaderErr) } + // Detect MIME type from data + mimeType := http.DetectContentType(buf.Bytes()) + uploadResult, uploadErr, _ := uploader.Upload(ctx, bytes.NewReader(buf.Bytes()), &operation.UploadOption{ UploadUrl: urlLocation, Filename: "", - Cipher: false, + Cipher: fs.option.Cipher, IsInputCompressed: false, - MimeType: "", + MimeType: mimeType, PairMap: nil, Jwt: auth, }) diff --git a/weed/server/filer_server_tus_session.go b/weed/server/filer_server_tus_session.go index e0e811a8c..b0c0885a1 100644 --- a/weed/server/filer_server_tus_session.go +++ b/weed/server/filer_server_tus_session.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "sort" "time" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -191,26 +192,28 @@ func (fs *FilerServer) completeTusUpload(ctx context.Context, session *TusSessio return fmt.Errorf("upload incomplete: offset=%d, expected=%d", session.Offset, session.Size) } + // Sort chunks by offset to ensure correct order + sort.Slice(session.Chunks, func(i, j int) bool { + return session.Chunks[i].Offset < session.Chunks[j].Offset + }) + // Assemble file chunks in order var fileChunks []*filer_pb.FileChunk - var offset int64 = 0 for _, chunk := range session.Chunks { fid, fidErr := filer_pb.ToFileIdObject(chunk.FileId) if fidErr != nil { - glog.Warningf("Invalid file ID %s: %v", chunk.FileId, fidErr) - continue + return fmt.Errorf("invalid file ID %s at offset %d: %w", chunk.FileId, chunk.Offset, fidErr) } fileChunk := &filer_pb.FileChunk{ FileId: chunk.FileId, - Offset: offset, + Offset: chunk.Offset, Size: uint64(chunk.Size), ModifiedTsNs: chunk.UploadAt, Fid: fid, } fileChunks = append(fileChunks, fileChunk) - offset += chunk.Size } // Determine content type from metadata From 35192b8bb8e4a0198593ce8fc858dc3d177b8df2 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 15:06:57 -0800 Subject: [PATCH 11/16] fmt --- test/tus/tus_integration_test.go | 1 - weed/server/filer_server_tus_session.go | 1 - 2 files changed, 2 deletions(-) diff --git a/test/tus/tus_integration_test.go b/test/tus/tus_integration_test.go index db3576f9b..b2c9bb8e8 100644 --- a/test/tus/tus_integration_test.go +++ b/test/tus/tus_integration_test.go @@ -744,4 +744,3 @@ func TestTusResumeAfterInterruption(t *testing.T) { require.NoError(t, err) assert.Equal(t, testData, body, "Resumed upload should produce complete file") } - diff --git a/weed/server/filer_server_tus_session.go b/weed/server/filer_server_tus_session.go index b0c0885a1..685488e10 100644 --- a/weed/server/filer_server_tus_session.go +++ b/weed/server/filer_server_tus_session.go @@ -255,4 +255,3 @@ func (fs *FilerServer) completeTusUpload(ctx context.Context, session *TusSessio return nil } - From 21d1a2d167f4bda4cdbe08aa31a3f6e1910fa6b6 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 15:15:25 -0800 Subject: [PATCH 12/16] Address remaining code review comments - Fix potential open redirect vulnerability by sanitizing uploadLocation path - Add language specifier to README code block - Handle os.Create errors in test setup - Use waitForHTTPServer instead of time.Sleep for master/volume readiness - Improve test reliability and debugging --- test/tus/README.md | 2 +- test/tus/tus_integration_test.go | 36 +++++++++++++++++++----- weed/server/filer_server_tus_handlers.go | 8 ++++-- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/test/tus/README.md b/test/tus/README.md index cd014229e..03c980a3d 100644 --- a/test/tus/README.md +++ b/test/tus/README.md @@ -201,7 +201,7 @@ curl -X DELETE http://localhost:18888/.tus/.uploads/{upload-id} \ ## Architecture -``` +```text Client Filer Volume Servers | | | |-- POST /.tus/path/file.mp4 ->| | diff --git a/test/tus/tus_integration_test.go b/test/tus/tus_integration_test.go index b2c9bb8e8..75763344b 100644 --- a/test/tus/tus_integration_test.go +++ b/test/tus/tus_integration_test.go @@ -93,7 +93,11 @@ func startTestCluster(t *testing.T, ctx context.Context) (*TestCluster, error) { "-mdir", masterDir, "-ip", "127.0.0.1", ) - masterLogFile, _ := os.Create(filepath.Join(masterDir, "master.log")) + masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log")) + if err != nil { + os.RemoveAll(dataDir) + return nil, fmt.Errorf("failed to create master log: %v", err) + } masterCmd.Stdout = masterLogFile masterCmd.Stderr = masterLogFile if err := masterCmd.Start(); err != nil { @@ -102,8 +106,12 @@ func startTestCluster(t *testing.T, ctx context.Context) (*TestCluster, error) { } cluster.masterCmd = masterCmd - // Wait for master - time.Sleep(2 * time.Second) + // Wait for master to be ready + if err := waitForHTTPServer("http://127.0.0.1:"+testMasterPort+"/dir/status", 30*time.Second); err != nil { + cluster.Stop() + os.RemoveAll(dataDir) + return nil, fmt.Errorf("master not ready: %v", err) + } // Start volume server volumeCmd := exec.CommandContext(ctx, weedBinary, "volume", @@ -112,7 +120,12 @@ func startTestCluster(t *testing.T, ctx context.Context) (*TestCluster, error) { "-mserver", "127.0.0.1:"+testMasterPort, "-ip", "127.0.0.1", ) - volumeLogFile, _ := os.Create(filepath.Join(volumeDir, "volume.log")) + volumeLogFile, err := os.Create(filepath.Join(volumeDir, "volume.log")) + if err != nil { + cluster.Stop() + os.RemoveAll(dataDir) + return nil, fmt.Errorf("failed to create volume log: %v", err) + } volumeCmd.Stdout = volumeLogFile volumeCmd.Stderr = volumeLogFile if err := volumeCmd.Start(); err != nil { @@ -122,8 +135,12 @@ func startTestCluster(t *testing.T, ctx context.Context) (*TestCluster, error) { } cluster.volumeCmd = volumeCmd - // Wait for volume server - time.Sleep(2 * time.Second) + // Wait for volume server to register with master + if err := waitForHTTPServer("http://127.0.0.1:"+testVolumePort+"/status", 30*time.Second); err != nil { + cluster.Stop() + os.RemoveAll(dataDir) + return nil, fmt.Errorf("volume server not ready: %v", err) + } // Start filer with TUS enabled filerCmd := exec.CommandContext(ctx, weedBinary, "filer", @@ -132,7 +149,12 @@ func startTestCluster(t *testing.T, ctx context.Context) (*TestCluster, error) { "-ip", "127.0.0.1", "-dataCenter", "dc1", ) - filerLogFile, _ := os.Create(filepath.Join(filerDir, "filer.log")) + filerLogFile, err := os.Create(filepath.Join(filerDir, "filer.log")) + if err != nil { + cluster.Stop() + os.RemoveAll(dataDir) + return nil, fmt.Errorf("failed to create filer log: %v", err) + } filerCmd.Stdout = filerLogFile filerCmd.Stderr = filerLogFile if err := filerCmd.Start(); err != nil { diff --git a/weed/server/filer_server_tus_handlers.go b/weed/server/filer_server_tus_handlers.go index f4fa5959e..7411b2527 100644 --- a/weed/server/filer_server_tus_handlers.go +++ b/weed/server/filer_server_tus_handlers.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "path" "strconv" "strings" "time" @@ -131,8 +132,11 @@ func (fs *FilerServer) tusCreateHandler(w http.ResponseWriter, r *http.Request) return } - // Build upload location URL - uploadLocation := fmt.Sprintf("%s/.uploads/%s", tusPrefix, uploadID) + // Build upload location URL (ensure it starts with single /) + uploadLocation := path.Clean(fmt.Sprintf("%s/.uploads/%s", tusPrefix, uploadID)) + if !strings.HasPrefix(uploadLocation, "/") { + uploadLocation = "/" + uploadLocation + } // Handle creation-with-upload extension if r.ContentLength > 0 && r.Header.Get("Content-Type") == "application/offset+octet-stream" { From 870e32c4ad154dac83c14fc150b6ef3a5842f1ab Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 16:15:30 -0800 Subject: [PATCH 13/16] Address critical and high-priority review comments - Add per-session locking to prevent race conditions in updateTusSessionOffset - Stream data directly to volume server instead of buffering entire chunk - Only buffer 512 bytes for MIME type detection, then stream remaining data - Clean up session locks when session is deleted --- weed/server/filer_server_tus_handlers.go | 36 +++++++++++++------- weed/server/filer_server_tus_session.go | 42 ++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 12 deletions(-) diff --git a/weed/server/filer_server_tus_handlers.go b/weed/server/filer_server_tus_handlers.go index 7411b2527..f0b47f6d0 100644 --- a/weed/server/filer_server_tus_handlers.go +++ b/weed/server/filer_server_tus_handlers.go @@ -292,15 +292,6 @@ func (fs *FilerServer) tusWriteData(ctx context.Context, session *TusSession, of } // Read data into buffer - buf := new(bytes.Buffer) - n, err := io.CopyN(buf, reader, contentLength) - if err != nil && err != io.EOF { - return 0, fmt.Errorf("read data: %w", err) - } - if n == 0 { - return 0, nil - } - // Determine storage options based on target path so, err := fs.detectStorageOption0(ctx, session.TargetPath, "", "", "", "", "", "", "", "", "") if err != nil { @@ -319,10 +310,31 @@ func (fs *FilerServer) tusWriteData(ctx context.Context, session *TusSession, of return 0, fmt.Errorf("create uploader: %w", uploaderErr) } - // Detect MIME type from data - mimeType := http.DetectContentType(buf.Bytes()) + // Read first 512 bytes for MIME type detection, then stream the rest + sniffBuf := make([]byte, 512) + sniffN, sniffErr := io.ReadFull(reader, sniffBuf) + if sniffErr != nil && sniffErr != io.EOF && sniffErr != io.ErrUnexpectedEOF { + return 0, fmt.Errorf("read data for mime detection: %w", sniffErr) + } + if sniffN == 0 { + return 0, nil + } + sniffBuf = sniffBuf[:sniffN] + + // Detect MIME type from sniffed bytes + mimeType := http.DetectContentType(sniffBuf) + + // Create a reader that combines sniffed bytes with remaining data + var dataReader io.Reader + if int64(sniffN) >= contentLength { + // All data fits in sniff buffer + dataReader = bytes.NewReader(sniffBuf) + } else { + // Combine sniffed bytes with remaining stream + dataReader = io.MultiReader(bytes.NewReader(sniffBuf), io.LimitReader(reader, contentLength-int64(sniffN))) + } - uploadResult, uploadErr, _ := uploader.Upload(ctx, bytes.NewReader(buf.Bytes()), &operation.UploadOption{ + uploadResult, uploadErr, _ := uploader.Upload(ctx, dataReader, &operation.UploadOption{ UploadUrl: urlLocation, Filename: "", Cipher: fs.option.Cipher, diff --git a/weed/server/filer_server_tus_session.go b/weed/server/filer_server_tus_session.go index 685488e10..c8ae7c1cd 100644 --- a/weed/server/filer_server_tus_session.go +++ b/weed/server/filer_server_tus_session.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "sort" + "sync" "time" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -14,6 +15,39 @@ import ( "github.com/seaweedfs/seaweedfs/weed/util" ) +// tusSessionLocks provides per-session locking to prevent race conditions +var tusSessionLocks = struct { + sync.RWMutex + locks map[string]*sync.Mutex +}{locks: make(map[string]*sync.Mutex)} + +// getTusSessionLock returns a lock for the given upload ID +func getTusSessionLock(uploadID string) *sync.Mutex { + tusSessionLocks.RLock() + lock, exists := tusSessionLocks.locks[uploadID] + tusSessionLocks.RUnlock() + if exists { + return lock + } + + tusSessionLocks.Lock() + defer tusSessionLocks.Unlock() + // Double-check after acquiring write lock + if lock, exists = tusSessionLocks.locks[uploadID]; exists { + return lock + } + lock = &sync.Mutex{} + tusSessionLocks.locks[uploadID] = lock + return lock +} + +// removeTusSessionLock removes the lock for the given upload ID +func removeTusSessionLock(uploadID string) { + tusSessionLocks.Lock() + defer tusSessionLocks.Unlock() + delete(tusSessionLocks.locks, uploadID) +} + const ( TusVersion = "1.0.0" TusMaxSize = int64(5 * 1024 * 1024 * 1024) // 5GB default max size @@ -145,6 +179,11 @@ func (fs *FilerServer) getTusSession(ctx context.Context, uploadID string) (*Tus // updateTusSessionOffset updates the session offset after a successful chunk upload func (fs *FilerServer) updateTusSessionOffset(ctx context.Context, uploadID string, newOffset int64, chunk *TusChunkInfo) error { + // Lock the session to prevent concurrent modifications + lock := getTusSessionLock(uploadID) + lock.Lock() + defer lock.Unlock() + session, err := fs.getTusSession(ctx, uploadID) if err != nil { return err @@ -160,6 +199,9 @@ func (fs *FilerServer) updateTusSessionOffset(ctx context.Context, uploadID stri // deleteTusSession removes a TUS upload session and all its data func (fs *FilerServer) deleteTusSession(ctx context.Context, uploadID string) error { + // Clean up the session lock + defer removeTusSessionLock(uploadID) + session, err := fs.getTusSession(ctx, uploadID) if err != nil { // Session might already be deleted or never existed From b49ee254814d6ecea7937147f30862c154f25667 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 16:21:29 -0800 Subject: [PATCH 14/16] Fix race condition to work across multiple filer instances - Store each chunk as a separate file entry instead of updating session JSON - Chunk file names encode offset, size, and fileId for atomic storage - getTusSession loads chunks from directory listing (atomic read) - Eliminates read-modify-write race condition across multiple filers - Remove in-memory mutex that only worked for single filer instance --- weed/server/filer_server_tus_session.go | 138 +++++++++++++++--------- 1 file changed, 90 insertions(+), 48 deletions(-) diff --git a/weed/server/filer_server_tus_session.go b/weed/server/filer_server_tus_session.go index c8ae7c1cd..550574058 100644 --- a/weed/server/filer_server_tus_session.go +++ b/weed/server/filer_server_tus_session.go @@ -6,7 +6,8 @@ import ( "fmt" "os" "sort" - "sync" + "strconv" + "strings" "time" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -15,39 +16,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/util" ) -// tusSessionLocks provides per-session locking to prevent race conditions -var tusSessionLocks = struct { - sync.RWMutex - locks map[string]*sync.Mutex -}{locks: make(map[string]*sync.Mutex)} - -// getTusSessionLock returns a lock for the given upload ID -func getTusSessionLock(uploadID string) *sync.Mutex { - tusSessionLocks.RLock() - lock, exists := tusSessionLocks.locks[uploadID] - tusSessionLocks.RUnlock() - if exists { - return lock - } - - tusSessionLocks.Lock() - defer tusSessionLocks.Unlock() - // Double-check after acquiring write lock - if lock, exists = tusSessionLocks.locks[uploadID]; exists { - return lock - } - lock = &sync.Mutex{} - tusSessionLocks.locks[uploadID] = lock - return lock -} - -// removeTusSessionLock removes the lock for the given upload ID -func removeTusSessionLock(uploadID string) { - tusSessionLocks.Lock() - defer tusSessionLocks.Unlock() - delete(tusSessionLocks.locks, uploadID) -} - const ( TusVersion = "1.0.0" TusMaxSize = int64(5 * 1024 * 1024 * 1024) // 5GB default max size @@ -92,6 +60,41 @@ func (fs *FilerServer) tusSessionInfoPath(uploadID string) string { return fmt.Sprintf("/%s/%s/%s", TusUploadsFolder, uploadID, TusInfoFileName) } +// tusChunkPath returns the path to store a chunk info file +// Format: /{TusUploadsFolder}/{uploadID}/chunk_{offset}_{size}_{fileId} +func (fs *FilerServer) tusChunkPath(uploadID string, offset, size int64, fileId string) string { + // Replace / in fileId with _ to make it a valid filename + safeFileId := strings.ReplaceAll(fileId, "/", "_") + return fmt.Sprintf("/%s/%s/chunk_%016d_%016d_%s", TusUploadsFolder, uploadID, offset, size, safeFileId) +} + +// parseTusChunkPath parses chunk info from a chunk file name +func parseTusChunkPath(name string) (*TusChunkInfo, error) { + if !strings.HasPrefix(name, "chunk_") { + return nil, fmt.Errorf("not a chunk file: %s", name) + } + parts := strings.SplitN(name[6:], "_", 3) // Skip "chunk_" prefix + if len(parts) < 3 { + return nil, fmt.Errorf("invalid chunk file name: %s", name) + } + offset, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid offset in chunk file: %s", name) + } + size, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid size in chunk file: %s", name) + } + // Restore / in fileId + fileId := strings.ReplaceAll(parts[2], "_", "/") + return &TusChunkInfo{ + Offset: offset, + Size: size, + FileId: fileId, + UploadAt: time.Now().UnixNano(), + }, nil +} + // createTusSession creates a new TUS upload session func (fs *FilerServer) createTusSession(ctx context.Context, uploadID, targetPath string, size int64, metadata map[string]string) (*TusSession, error) { session := &TusSession{ @@ -158,7 +161,7 @@ func (fs *FilerServer) saveTusSession(ctx context.Context, session *TusSession) return nil } -// getTusSession retrieves a TUS session by upload ID +// getTusSession retrieves a TUS session by upload ID, including chunks from directory listing func (fs *FilerServer) getTusSession(ctx context.Context, uploadID string) (*TusSession, error) { infoPath := util.FullPath(fs.tusSessionInfoPath(uploadID)) entry, err := fs.filer.FindEntry(ctx, infoPath) @@ -174,33 +177,72 @@ func (fs *FilerServer) getTusSession(ctx context.Context, uploadID string) (*Tus return nil, fmt.Errorf("unmarshal session: %w", err) } + // Load chunks from directory listing (atomic read, no race condition) + sessionDirPath := util.FullPath(fs.tusSessionPath(uploadID)) + entries, _, err := fs.filer.ListDirectoryEntries(ctx, sessionDirPath, "", false, 10000, "", "", "") + if err != nil { + return nil, fmt.Errorf("list session directory: %w", err) + } + + session.Chunks = nil + session.Offset = 0 + for _, e := range entries { + if strings.HasPrefix(e.Name(), "chunk_") { + chunk, parseErr := parseTusChunkPath(e.Name()) + if parseErr != nil { + glog.V(1).Infof("Skipping invalid chunk file %s: %v", e.Name(), parseErr) + continue + } + session.Chunks = append(session.Chunks, chunk) + } + } + + // Sort chunks by offset and compute current offset + if len(session.Chunks) > 0 { + sort.Slice(session.Chunks, func(i, j int) bool { + return session.Chunks[i].Offset < session.Chunks[j].Offset + }) + // Current offset is the end of the last chunk + lastChunk := session.Chunks[len(session.Chunks)-1] + session.Offset = lastChunk.Offset + lastChunk.Size + } + return &session, nil } -// updateTusSessionOffset updates the session offset after a successful chunk upload +// updateTusSessionOffset stores the chunk info as a separate file entry +// This avoids read-modify-write race conditions across multiple filer instances func (fs *FilerServer) updateTusSessionOffset(ctx context.Context, uploadID string, newOffset int64, chunk *TusChunkInfo) error { - // Lock the session to prevent concurrent modifications - lock := getTusSessionLock(uploadID) - lock.Lock() - defer lock.Unlock() + if chunk == nil { + return nil + } - session, err := fs.getTusSession(ctx, uploadID) + // Store chunk info as a separate file entry (atomic operation) + chunkPath := util.FullPath(fs.tusChunkPath(uploadID, chunk.Offset, chunk.Size, chunk.FileId)) + chunkData, err := json.Marshal(chunk) if err != nil { - return err + return fmt.Errorf("marshal chunk info: %w", err) } - session.Offset = newOffset - if chunk != nil { - session.Chunks = append(session.Chunks, chunk) + if err := fs.filer.CreateEntry(ctx, &filer.Entry{ + FullPath: chunkPath, + Attr: filer.Attr{ + Mode: 0644, + Crtime: time.Now(), + Mtime: time.Now(), + Uid: OS_UID, + Gid: OS_GID, + }, + Content: chunkData, + }, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil { + return fmt.Errorf("save chunk info: %w", err) } - return fs.saveTusSession(ctx, session) + return nil } // deleteTusSession removes a TUS upload session and all its data func (fs *FilerServer) deleteTusSession(ctx context.Context, uploadID string) error { - // Clean up the session lock - defer removeTusSessionLock(uploadID) session, err := fs.getTusSession(ctx, uploadID) if err != nil { From 5e5f05607a76817e3ea0871325ff0087cf722a57 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 17:00:08 -0800 Subject: [PATCH 15/16] Address code review comments: fix variable shadowing, sniff size, and test stability - Rename path variable to reqPath to avoid shadowing path package - Make sniff buffer size respect contentLength (read at most contentLength bytes) - Handle Content-Length < 0 in creation-with-upload (return error for chunked encoding) - Fix test cluster: use temp directory for filer store, add startup delay --- test/tus/tus_integration_test.go | 6 ++++- weed/server/filer_server_tus_handlers.go | 28 +++++++++++++++++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/test/tus/tus_integration_test.go b/test/tus/tus_integration_test.go index 75763344b..ff8734bda 100644 --- a/test/tus/tus_integration_test.go +++ b/test/tus/tus_integration_test.go @@ -147,7 +147,7 @@ func startTestCluster(t *testing.T, ctx context.Context) (*TestCluster, error) { "-port", testFilerPort, "-master", "127.0.0.1:"+testMasterPort, "-ip", "127.0.0.1", - "-dataCenter", "dc1", + "-defaultStoreDir", filerDir, ) filerLogFile, err := os.Create(filepath.Join(filerDir, "filer.log")) if err != nil { @@ -171,6 +171,10 @@ func startTestCluster(t *testing.T, ctx context.Context) (*TestCluster, error) { return nil, fmt.Errorf("filer not ready: %v", err) } + // Wait a bit more for the cluster to fully stabilize + // Volumes are created lazily, and we need to ensure the master topology is ready + time.Sleep(2 * time.Second) + return cluster, nil } diff --git a/weed/server/filer_server_tus_handlers.go b/weed/server/filer_server_tus_handlers.go index f0b47f6d0..abb6ecca4 100644 --- a/weed/server/filer_server_tus_handlers.go +++ b/weed/server/filer_server_tus_handlers.go @@ -35,7 +35,7 @@ func (fs *FilerServer) tusHandler(w http.ResponseWriter, r *http.Request) { } // Route based on method and path - path := r.URL.Path + reqPath := r.URL.Path tusPrefix := fs.option.TusPath if tusPrefix == "" { tusPrefix = ".tus" @@ -46,8 +46,8 @@ func (fs *FilerServer) tusHandler(w http.ResponseWriter, r *http.Request) { // Check if this is an upload location (contains upload ID after {tusPrefix}/.uploads/) uploadsPrefix := tusPrefix + "/.uploads/" - if strings.HasPrefix(path, uploadsPrefix) { - uploadID := strings.TrimPrefix(path, uploadsPrefix) + if strings.HasPrefix(reqPath, uploadsPrefix) { + uploadID := strings.TrimPrefix(reqPath, uploadsPrefix) uploadID = strings.Split(uploadID, "/")[0] // Get just the ID, not any trailing path switch r.Method { @@ -139,7 +139,17 @@ func (fs *FilerServer) tusCreateHandler(w http.ResponseWriter, r *http.Request) } // Handle creation-with-upload extension - if r.ContentLength > 0 && r.Header.Get("Content-Type") == "application/offset+octet-stream" { + // TUS requires Content-Length for uploads; reject chunked encoding + if r.Header.Get("Content-Type") == "application/offset+octet-stream" { + if r.ContentLength < 0 { + fs.deleteTusSession(ctx, uploadID) + http.Error(w, "Content-Length header required for creation-with-upload", http.StatusBadRequest) + return + } + if r.ContentLength == 0 { + // Empty body is allowed, just skip the upload + goto respond + } // Upload data in the creation request bytesWritten, uploadErr := fs.tusWriteData(ctx, session, 0, r.Body, r.ContentLength) if uploadErr != nil { @@ -170,6 +180,7 @@ func (fs *FilerServer) tusCreateHandler(w http.ResponseWriter, r *http.Request) } } +respond: w.Header().Set("Location", uploadLocation) w.WriteHeader(http.StatusCreated) } @@ -310,8 +321,13 @@ func (fs *FilerServer) tusWriteData(ctx context.Context, session *TusSession, of return 0, fmt.Errorf("create uploader: %w", uploaderErr) } - // Read first 512 bytes for MIME type detection, then stream the rest - sniffBuf := make([]byte, 512) + // Read first bytes for MIME type detection, respecting contentLength + // http.DetectContentType uses at most 512 bytes + sniffSize := int64(512) + if contentLength < sniffSize { + sniffSize = contentLength + } + sniffBuf := make([]byte, sniffSize) sniffN, sniffErr := io.ReadFull(reader, sniffBuf) if sniffErr != nil && sniffErr != io.EOF && sniffErr != io.ErrUnexpectedEOF { return 0, fmt.Errorf("read data for mime detection: %w", sniffErr) From 49ed42b367914ac4f3e2853e698e8fc05ddac24e Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 1 Dec 2025 17:28:38 -0800 Subject: [PATCH 16/16] Fix test stability: increase cluster stabilization delay to 5 seconds The tests were intermittently failing because the volume server needed more time to create volumes and register with the master. Increasing the delay from 2 to 5 seconds fixes the flaky test behavior. --- test/s3/sse/github_7562_copy_test.go | 505 +++++++++++++++++++++++ test/tus/tus_integration_test.go | 2 +- weed/s3api/s3api_object_handlers_copy.go | 277 ++++++++++++- 3 files changed, 770 insertions(+), 14 deletions(-) create mode 100644 test/s3/sse/github_7562_copy_test.go diff --git a/test/s3/sse/github_7562_copy_test.go b/test/s3/sse/github_7562_copy_test.go new file mode 100644 index 000000000..5831c0b80 --- /dev/null +++ b/test/s3/sse/github_7562_copy_test.go @@ -0,0 +1,505 @@ +package sse_test + +import ( + "bytes" + "context" + "fmt" + "io" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestGitHub7562CopyFromEncryptedToTempToEncrypted reproduces the exact scenario from +// GitHub issue #7562: copying from an encrypted bucket to a temp bucket, then to another +// encrypted bucket fails with InternalError. +// +// Reproduction steps: +// 1. Create source bucket with SSE-S3 encryption enabled +// 2. Upload object (automatically encrypted) +// 3. Create temp bucket (no encryption) +// 4. Copy object from source to temp (decrypts) +// 5. Delete source bucket +// 6. Create destination bucket with SSE-S3 encryption +// 7. Copy object from temp to dest (should re-encrypt) - THIS FAILS +func TestGitHub7562CopyFromEncryptedToTempToEncrypted(t *testing.T) { + ctx := context.Background() + client, err := createS3Client(ctx, defaultConfig) + require.NoError(t, err, "Failed to create S3 client") + + // Create three buckets + srcBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-src-") + require.NoError(t, err, "Failed to create source bucket") + + tempBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-temp-") + require.NoError(t, err, "Failed to create temp bucket") + + destBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-dest-") + require.NoError(t, err, "Failed to create destination bucket") + + // Cleanup at the end + defer func() { + // Clean up in reverse order of creation + cleanupTestBucket(ctx, client, destBucket) + cleanupTestBucket(ctx, client, tempBucket) + // Note: srcBucket is deleted during the test + }() + + testData := []byte("Test data for GitHub issue #7562 - copy from encrypted to temp to encrypted bucket") + objectKey := "demo-file.txt" + + t.Logf("[1] Creating source bucket with SSE-S3 default encryption: %s", srcBucket) + + // Step 1: Enable SSE-S3 default encryption on source bucket + _, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{ + Bucket: aws.String(srcBucket), + ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{ + Rules: []types.ServerSideEncryptionRule{ + { + ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{ + SSEAlgorithm: types.ServerSideEncryptionAes256, + }, + }, + }, + }, + }) + require.NoError(t, err, "Failed to set source bucket default encryption") + + t.Log("[2] Uploading demo object to source bucket") + + // Step 2: Upload object to source bucket (will be automatically encrypted) + _, err = client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(srcBucket), + Key: aws.String(objectKey), + Body: bytes.NewReader(testData), + // No encryption header - bucket default applies + }) + require.NoError(t, err, "Failed to upload to source bucket") + + // Verify source object is encrypted + srcHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(srcBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to HEAD source object") + assert.Equal(t, types.ServerSideEncryptionAes256, srcHead.ServerSideEncryption, + "Source object should be SSE-S3 encrypted") + t.Logf("Source object encryption: %v", srcHead.ServerSideEncryption) + + t.Logf("[3] Creating temp bucket (no encryption): %s", tempBucket) + // Temp bucket already created without encryption + + t.Log("[4] Copying object from source to temp (should decrypt)") + + // Step 4: Copy to temp bucket (no encryption = decrypts) + _, err = client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(tempBucket), + Key: aws.String(objectKey), + CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)), + // No encryption header - data stored unencrypted + }) + require.NoError(t, err, "Failed to copy to temp bucket") + + // Verify temp object is NOT encrypted + tempHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(tempBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to HEAD temp object") + assert.Empty(t, tempHead.ServerSideEncryption, "Temp object should NOT be encrypted") + t.Logf("Temp object encryption: %v (should be empty)", tempHead.ServerSideEncryption) + + // Verify temp object content + tempGet, err := client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(tempBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to GET temp object") + tempData, err := io.ReadAll(tempGet.Body) + tempGet.Body.Close() + require.NoError(t, err, "Failed to read temp object") + assertDataEqual(t, testData, tempData, "Temp object data should match original") + + t.Log("[5] Deleting original source bucket") + + // Step 5: Delete source bucket + err = cleanupTestBucket(ctx, client, srcBucket) + require.NoError(t, err, "Failed to delete source bucket") + + t.Logf("[6] Creating destination bucket with SSE-S3 encryption: %s", destBucket) + + // Step 6: Enable SSE-S3 default encryption on destination bucket + _, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{ + Bucket: aws.String(destBucket), + ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{ + Rules: []types.ServerSideEncryptionRule{ + { + ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{ + SSEAlgorithm: types.ServerSideEncryptionAes256, + }, + }, + }, + }, + }) + require.NoError(t, err, "Failed to set destination bucket default encryption") + + t.Log("[7] Copying object from temp to dest (should re-encrypt) - THIS IS WHERE #7562 FAILS") + + // Step 7: Copy from temp to dest bucket (should re-encrypt with SSE-S3) + // THIS IS THE STEP THAT FAILS IN GITHUB ISSUE #7562 + _, err = client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(objectKey), + CopySource: aws.String(fmt.Sprintf("%s/%s", tempBucket, objectKey)), + // No encryption header - bucket default should apply + }) + require.NoError(t, err, "GitHub #7562: Failed to copy from temp to encrypted dest bucket") + + // Verify destination object is encrypted + destHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to HEAD destination object") + assert.Equal(t, types.ServerSideEncryptionAes256, destHead.ServerSideEncryption, + "Destination object should be SSE-S3 encrypted via bucket default") + t.Logf("Destination object encryption: %v", destHead.ServerSideEncryption) + + // Verify destination object content is correct + destGet, err := client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to GET destination object") + destData, err := io.ReadAll(destGet.Body) + destGet.Body.Close() + require.NoError(t, err, "Failed to read destination object") + assertDataEqual(t, testData, destData, "GitHub #7562: Destination object data mismatch after re-encryption") + + t.Log("[done] GitHub #7562 reproduction test completed successfully!") +} + +// TestGitHub7562SimpleScenario tests the simpler variant: just copy unencrypted to encrypted bucket +func TestGitHub7562SimpleScenario(t *testing.T) { + ctx := context.Background() + client, err := createS3Client(ctx, defaultConfig) + require.NoError(t, err, "Failed to create S3 client") + + // Create two buckets + srcBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-simple-src-") + require.NoError(t, err, "Failed to create source bucket") + defer cleanupTestBucket(ctx, client, srcBucket) + + destBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-simple-dest-") + require.NoError(t, err, "Failed to create destination bucket") + defer cleanupTestBucket(ctx, client, destBucket) + + testData := []byte("Simple test for unencrypted to encrypted copy") + objectKey := "test-object.txt" + + t.Logf("Source bucket (no encryption): %s", srcBucket) + t.Logf("Dest bucket (SSE-S3 default): %s", destBucket) + + // Upload to unencrypted source bucket + _, err = client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(srcBucket), + Key: aws.String(objectKey), + Body: bytes.NewReader(testData), + }) + require.NoError(t, err, "Failed to upload to source bucket") + + // Enable SSE-S3 on destination bucket + _, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{ + Bucket: aws.String(destBucket), + ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{ + Rules: []types.ServerSideEncryptionRule{ + { + ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{ + SSEAlgorithm: types.ServerSideEncryptionAes256, + }, + }, + }, + }, + }) + require.NoError(t, err, "Failed to set dest bucket encryption") + + // Copy to encrypted bucket (should use bucket default encryption) + _, err = client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(objectKey), + CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)), + }) + require.NoError(t, err, "Failed to copy to encrypted bucket") + + // Verify destination is encrypted + destHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to HEAD dest object") + assert.Equal(t, types.ServerSideEncryptionAes256, destHead.ServerSideEncryption, + "Object should be encrypted via bucket default") + + // Verify content + destGet, err := client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to GET dest object") + destData, err := io.ReadAll(destGet.Body) + destGet.Body.Close() + require.NoError(t, err, "Failed to read dest object") + assertDataEqual(t, testData, destData, "Data mismatch") +} + +// TestGitHub7562DebugMetadata helps debug what metadata is present on objects at each step +func TestGitHub7562DebugMetadata(t *testing.T) { + ctx := context.Background() + client, err := createS3Client(ctx, defaultConfig) + require.NoError(t, err, "Failed to create S3 client") + + // Create three buckets + srcBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-debug-src-") + require.NoError(t, err, "Failed to create source bucket") + + tempBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-debug-temp-") + require.NoError(t, err, "Failed to create temp bucket") + + destBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-debug-dest-") + require.NoError(t, err, "Failed to create destination bucket") + + defer func() { + cleanupTestBucket(ctx, client, destBucket) + cleanupTestBucket(ctx, client, tempBucket) + }() + + testData := []byte("Debug metadata test for GitHub #7562") + objectKey := "debug-file.txt" + + // Enable SSE-S3 on source + _, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{ + Bucket: aws.String(srcBucket), + ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{ + Rules: []types.ServerSideEncryptionRule{ + { + ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{ + SSEAlgorithm: types.ServerSideEncryptionAes256, + }, + }, + }, + }, + }) + require.NoError(t, err, "Failed to set source bucket encryption") + + // Upload + _, err = client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(srcBucket), + Key: aws.String(objectKey), + Body: bytes.NewReader(testData), + }) + require.NoError(t, err, "Failed to upload") + + // Log source object headers + srcHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(srcBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to HEAD source") + t.Logf("=== SOURCE OBJECT (encrypted) ===") + t.Logf("ServerSideEncryption: %v", srcHead.ServerSideEncryption) + t.Logf("Metadata: %v", srcHead.Metadata) + t.Logf("ContentLength: %d", aws.ToInt64(srcHead.ContentLength)) + + // Copy to temp + _, err = client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(tempBucket), + Key: aws.String(objectKey), + CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)), + }) + require.NoError(t, err, "Failed to copy to temp") + + // Log temp object headers + tempHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(tempBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to HEAD temp") + t.Logf("=== TEMP OBJECT (should be unencrypted) ===") + t.Logf("ServerSideEncryption: %v (should be empty)", tempHead.ServerSideEncryption) + t.Logf("Metadata: %v", tempHead.Metadata) + t.Logf("ContentLength: %d", aws.ToInt64(tempHead.ContentLength)) + + // Verify temp is NOT encrypted + if tempHead.ServerSideEncryption != "" { + t.Logf("WARNING: Temp object unexpectedly has encryption: %v", tempHead.ServerSideEncryption) + } + + // Delete source bucket + cleanupTestBucket(ctx, client, srcBucket) + + // Enable SSE-S3 on dest + _, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{ + Bucket: aws.String(destBucket), + ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{ + Rules: []types.ServerSideEncryptionRule{ + { + ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{ + SSEAlgorithm: types.ServerSideEncryptionAes256, + }, + }, + }, + }, + }) + require.NoError(t, err, "Failed to set dest bucket encryption") + + // Copy to dest - THIS IS WHERE #7562 FAILS + t.Log("=== COPYING TO ENCRYPTED DEST ===") + _, err = client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(objectKey), + CopySource: aws.String(fmt.Sprintf("%s/%s", tempBucket, objectKey)), + }) + if err != nil { + t.Logf("!!! COPY FAILED (GitHub #7562): %v", err) + t.FailNow() + } + + // Log dest object headers + destHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to HEAD dest") + t.Logf("=== DEST OBJECT (should be encrypted) ===") + t.Logf("ServerSideEncryption: %v", destHead.ServerSideEncryption) + t.Logf("Metadata: %v", destHead.Metadata) + t.Logf("ContentLength: %d", aws.ToInt64(destHead.ContentLength)) + + // Verify dest IS encrypted + assert.Equal(t, types.ServerSideEncryptionAes256, destHead.ServerSideEncryption, + "Dest object should be encrypted") + + // Verify content is readable + destGet, err := client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to GET dest") + destData, err := io.ReadAll(destGet.Body) + destGet.Body.Close() + require.NoError(t, err, "Failed to read dest") + assertDataEqual(t, testData, destData, "Data mismatch") + + t.Log("=== DEBUG TEST PASSED ===") +} + +// TestGitHub7562LargeFile tests the issue with larger files that might trigger multipart handling +func TestGitHub7562LargeFile(t *testing.T) { + ctx := context.Background() + client, err := createS3Client(ctx, defaultConfig) + require.NoError(t, err, "Failed to create S3 client") + + srcBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-large-src-") + require.NoError(t, err, "Failed to create source bucket") + + tempBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-large-temp-") + require.NoError(t, err, "Failed to create temp bucket") + + destBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-large-dest-") + require.NoError(t, err, "Failed to create destination bucket") + + defer func() { + cleanupTestBucket(ctx, client, destBucket) + cleanupTestBucket(ctx, client, tempBucket) + }() + + // Use larger file to potentially trigger different code paths + testData := generateTestData(5 * 1024 * 1024) // 5MB + objectKey := "large-file.bin" + + t.Logf("Testing with %d byte file", len(testData)) + + // Enable SSE-S3 on source + _, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{ + Bucket: aws.String(srcBucket), + ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{ + Rules: []types.ServerSideEncryptionRule{ + { + ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{ + SSEAlgorithm: types.ServerSideEncryptionAes256, + }, + }, + }, + }, + }) + require.NoError(t, err, "Failed to set source bucket encryption") + + // Upload + _, err = client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(srcBucket), + Key: aws.String(objectKey), + Body: bytes.NewReader(testData), + }) + require.NoError(t, err, "Failed to upload") + + // Copy to temp (decrypt) + _, err = client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(tempBucket), + Key: aws.String(objectKey), + CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)), + }) + require.NoError(t, err, "Failed to copy to temp") + + // Delete source + cleanupTestBucket(ctx, client, srcBucket) + + // Enable SSE-S3 on dest + _, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{ + Bucket: aws.String(destBucket), + ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{ + Rules: []types.ServerSideEncryptionRule{ + { + ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{ + SSEAlgorithm: types.ServerSideEncryptionAes256, + }, + }, + }, + }, + }) + require.NoError(t, err, "Failed to set dest bucket encryption") + + // Copy to dest (re-encrypt) - GitHub #7562 + _, err = client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(objectKey), + CopySource: aws.String(fmt.Sprintf("%s/%s", tempBucket, objectKey)), + }) + require.NoError(t, err, "GitHub #7562: Large file copy to encrypted bucket failed") + + // Verify + destHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to HEAD dest") + assert.Equal(t, types.ServerSideEncryptionAes256, destHead.ServerSideEncryption) + assert.Equal(t, int64(len(testData)), aws.ToInt64(destHead.ContentLength)) + + // Verify content + destGet, err := client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Failed to GET dest") + destData, err := io.ReadAll(destGet.Body) + destGet.Body.Close() + require.NoError(t, err, "Failed to read dest") + assertDataEqual(t, testData, destData, "Large file data mismatch") + + t.Log("Large file test passed!") +} + diff --git a/test/tus/tus_integration_test.go b/test/tus/tus_integration_test.go index ff8734bda..a03c21dab 100644 --- a/test/tus/tus_integration_test.go +++ b/test/tus/tus_integration_test.go @@ -173,7 +173,7 @@ func startTestCluster(t *testing.T, ctx context.Context) (*TestCluster, error) { // Wait a bit more for the cluster to fully stabilize // Volumes are created lazily, and we need to ensure the master topology is ready - time.Sleep(2 * time.Second) + time.Sleep(5 * time.Second) return cluster, nil } diff --git a/weed/s3api/s3api_object_handlers_copy.go b/weed/s3api/s3api_object_handlers_copy.go index 0c465d3db..4e465919c 100644 --- a/weed/s3api/s3api_object_handlers_copy.go +++ b/weed/s3api/s3api_object_handlers_copy.go @@ -167,6 +167,14 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request } // Copy extended attributes from source, filtering out conflicting encryption metadata + // Pre-compute encryption state once for efficiency + srcHasSSEC := IsSSECEncrypted(entry.Extended) + srcHasSSEKMS := IsSSEKMSEncrypted(entry.Extended) + srcHasSSES3 := IsSSES3EncryptedInternal(entry.Extended) + dstWantsSSEC := IsSSECRequest(r) + dstWantsSSEKMS := IsSSEKMSRequest(r) + dstWantsSSES3 := IsSSES3RequestInternal(r) + for k, v := range entry.Extended { // Skip encryption-specific headers that might conflict with destination encryption type skipHeader := false @@ -177,17 +185,9 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request skipHeader = true } - // If we're doing cross-encryption, skip conflicting headers - if !skipHeader && len(entry.GetChunks()) > 0 { - // Detect source and destination encryption types - srcHasSSEC := IsSSECEncrypted(entry.Extended) - srcHasSSEKMS := IsSSEKMSEncrypted(entry.Extended) - srcHasSSES3 := IsSSES3EncryptedInternal(entry.Extended) - dstWantsSSEC := IsSSECRequest(r) - dstWantsSSEKMS := IsSSEKMSRequest(r) - dstWantsSSES3 := IsSSES3RequestInternal(r) - - // Use helper function to determine if header should be skipped + // Filter conflicting headers for cross-encryption or encrypted→unencrypted copies + // This applies to both inline files (no chunks) and chunked files - fixes GitHub #7562 + if !skipHeader { skipHeader = shouldSkipEncryptionHeader(k, srcHasSSEC, srcHasSSEKMS, srcHasSSES3, dstWantsSSEC, dstWantsSSEKMS, dstWantsSSES3) @@ -210,10 +210,31 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request dstEntry.Extended[k] = v } - // For zero-size files or files without chunks, use the original approach + // For zero-size files or files without chunks, handle inline content + // This includes encrypted inline files that need decryption/re-encryption if entry.Attributes.FileSize == 0 || len(entry.GetChunks()) == 0 { - // Just copy the entry structure without chunks for zero-size files dstEntry.Chunks = nil + + // Handle inline encrypted content - fixes GitHub #7562 + if len(entry.Content) > 0 { + inlineContent, inlineMetadata, inlineErr := s3a.processInlineContentForCopy( + entry, r, dstBucket, dstObject, + srcHasSSEC, srcHasSSEKMS, srcHasSSES3, + dstWantsSSEC, dstWantsSSEKMS, dstWantsSSES3) + if inlineErr != nil { + glog.Errorf("CopyObjectHandler inline content error: %v", inlineErr) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + dstEntry.Content = inlineContent + + // Apply inline destination metadata + if inlineMetadata != nil { + for k, v := range inlineMetadata { + dstEntry.Extended[k] = v + } + } + } } else { // Use unified copy strategy approach dstChunks, dstMetadata, copyErr := s3a.executeUnifiedCopyStrategy(entry, r, dstBucket, srcObject, dstObject) @@ -2405,3 +2426,233 @@ func shouldSkipEncryptionHeader(headerKey string, // Default: don't skip the header return false } + +// processInlineContentForCopy handles encryption/decryption for inline content during copy +// This fixes GitHub #7562 where small files stored inline weren't properly decrypted/re-encrypted +func (s3a *S3ApiServer) processInlineContentForCopy( + entry *filer_pb.Entry, r *http.Request, dstBucket, dstObject string, + srcSSEC, srcSSEKMS, srcSSES3 bool, + dstSSEC, dstSSEKMS, dstSSES3 bool) ([]byte, map[string][]byte, error) { + + content := entry.Content + var dstMetadata map[string][]byte + + // Check if source is encrypted and needs decryption + srcEncrypted := srcSSEC || srcSSEKMS || srcSSES3 + + // Check if destination needs encryption (explicit request or bucket default) + dstNeedsEncryption := dstSSEC || dstSSEKMS || dstSSES3 + if !dstNeedsEncryption { + // Check bucket default encryption + bucketMetadata, err := s3a.getBucketMetadata(dstBucket) + if err == nil && bucketMetadata != nil && bucketMetadata.Encryption != nil { + switch bucketMetadata.Encryption.SseAlgorithm { + case "aws:kms": + dstSSEKMS = true + dstNeedsEncryption = true + case "AES256": + dstSSES3 = true + dstNeedsEncryption = true + } + } + } + + // Decrypt source content if encrypted + if srcEncrypted { + decryptedContent, decErr := s3a.decryptInlineContent(entry, srcSSEC, srcSSEKMS, srcSSES3, r) + if decErr != nil { + return nil, nil, fmt.Errorf("failed to decrypt inline content: %w", decErr) + } + content = decryptedContent + glog.V(3).Infof("Decrypted inline content: %d bytes", len(content)) + } + + // Re-encrypt if destination needs encryption + if dstNeedsEncryption { + encryptedContent, encMetadata, encErr := s3a.encryptInlineContent(content, dstBucket, dstObject, dstSSEC, dstSSEKMS, dstSSES3, r) + if encErr != nil { + return nil, nil, fmt.Errorf("failed to encrypt inline content: %w", encErr) + } + content = encryptedContent + dstMetadata = encMetadata + glog.V(3).Infof("Encrypted inline content: %d bytes", len(content)) + } + + return content, dstMetadata, nil +} + +// decryptInlineContent decrypts inline content from an encrypted source +func (s3a *S3ApiServer) decryptInlineContent(entry *filer_pb.Entry, srcSSEC, srcSSEKMS, srcSSES3 bool, r *http.Request) ([]byte, error) { + content := entry.Content + + if srcSSES3 { + // Get SSE-S3 key from metadata + keyData, exists := entry.Extended[s3_constants.SeaweedFSSSES3Key] + if !exists { + return nil, fmt.Errorf("SSE-S3 key not found in metadata") + } + + keyManager := GetSSES3KeyManager() + sseKey, err := DeserializeSSES3Metadata(keyData, keyManager) + if err != nil { + return nil, fmt.Errorf("failed to deserialize SSE-S3 key: %w", err) + } + + // Get IV + iv := sseKey.IV + if len(iv) == 0 { + return nil, fmt.Errorf("SSE-S3 IV not found") + } + + // Decrypt content + decryptedReader, err := CreateSSES3DecryptedReader(bytes.NewReader(content), sseKey, iv) + if err != nil { + return nil, fmt.Errorf("failed to create SSE-S3 decrypted reader: %w", err) + } + return io.ReadAll(decryptedReader) + + } else if srcSSEKMS { + // Get SSE-KMS key from metadata + keyData, exists := entry.Extended[s3_constants.SeaweedFSSSEKMSKey] + if !exists { + return nil, fmt.Errorf("SSE-KMS key not found in metadata") + } + + sseKey, err := DeserializeSSEKMSMetadata(keyData) + if err != nil { + return nil, fmt.Errorf("failed to deserialize SSE-KMS key: %w", err) + } + + // Decrypt content + decryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(content), sseKey) + if err != nil { + return nil, fmt.Errorf("failed to create SSE-KMS decrypted reader: %w", err) + } + return io.ReadAll(decryptedReader) + + } else if srcSSEC { + // Get SSE-C key from request headers + sourceKey, err := ParseSSECCopySourceHeaders(r) + if err != nil { + return nil, fmt.Errorf("failed to parse SSE-C copy source headers: %w", err) + } + + // Get IV from metadata + iv, err := GetSSECIVFromMetadata(entry.Extended) + if err != nil { + return nil, fmt.Errorf("failed to get SSE-C IV: %w", err) + } + + // Decrypt content + decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(content), sourceKey, iv) + if err != nil { + return nil, fmt.Errorf("failed to create SSE-C decrypted reader: %w", err) + } + return io.ReadAll(decryptedReader) + } + + // Source not encrypted, return as-is + return content, nil +} + +// encryptInlineContent encrypts inline content for the destination +func (s3a *S3ApiServer) encryptInlineContent(content []byte, dstBucket, dstObject string, + dstSSEC, dstSSEKMS, dstSSES3 bool, r *http.Request) ([]byte, map[string][]byte, error) { + + dstMetadata := make(map[string][]byte) + + if dstSSES3 { + // Generate SSE-S3 key + keyManager := GetSSES3KeyManager() + key, err := keyManager.GetOrCreateKey("") + if err != nil { + return nil, nil, fmt.Errorf("failed to generate SSE-S3 key: %w", err) + } + + // Encrypt content + encryptedReader, iv, err := CreateSSES3EncryptedReader(bytes.NewReader(content), key) + if err != nil { + return nil, nil, fmt.Errorf("failed to create SSE-S3 encrypted reader: %w", err) + } + + encryptedContent, err := io.ReadAll(encryptedReader) + if err != nil { + return nil, nil, fmt.Errorf("failed to read encrypted content: %w", err) + } + + // Store IV on key and serialize metadata + key.IV = iv + keyData, err := SerializeSSES3Metadata(key) + if err != nil { + return nil, nil, fmt.Errorf("failed to serialize SSE-S3 metadata: %w", err) + } + + dstMetadata[s3_constants.SeaweedFSSSES3Key] = keyData + dstMetadata[s3_constants.AmzServerSideEncryption] = []byte("AES256") + + return encryptedContent, dstMetadata, nil + + } else if dstSSEKMS { + // Parse SSE-KMS headers + keyID, encryptionContext, bucketKeyEnabled, err := ParseSSEKMSCopyHeaders(r) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse SSE-KMS headers: %w", err) + } + + // Build encryption context if needed + if encryptionContext == nil { + encryptionContext = BuildEncryptionContext(dstBucket, dstObject, bucketKeyEnabled) + } + + // Encrypt content + encryptedReader, sseKey, err := CreateSSEKMSEncryptedReaderWithBucketKey( + bytes.NewReader(content), keyID, encryptionContext, bucketKeyEnabled) + if err != nil { + return nil, nil, fmt.Errorf("failed to create SSE-KMS encrypted reader: %w", err) + } + + encryptedContent, err := io.ReadAll(encryptedReader) + if err != nil { + return nil, nil, fmt.Errorf("failed to read encrypted content: %w", err) + } + + // Serialize metadata + keyData, err := SerializeSSEKMSMetadata(sseKey) + if err != nil { + return nil, nil, fmt.Errorf("failed to serialize SSE-KMS metadata: %w", err) + } + + dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = keyData + dstMetadata[s3_constants.AmzServerSideEncryption] = []byte("aws:kms") + + return encryptedContent, dstMetadata, nil + + } else if dstSSEC { + // Parse SSE-C headers + destKey, err := ParseSSECHeaders(r) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse SSE-C headers: %w", err) + } + + // Encrypt content + encryptedReader, iv, err := CreateSSECEncryptedReader(bytes.NewReader(content), destKey) + if err != nil { + return nil, nil, fmt.Errorf("failed to create SSE-C encrypted reader: %w", err) + } + + encryptedContent, err := io.ReadAll(encryptedReader) + if err != nil { + return nil, nil, fmt.Errorf("failed to read encrypted content: %w", err) + } + + // Store IV in metadata + StoreSSECIVInMetadata(dstMetadata, iv) + dstMetadata[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] = []byte("AES256") + dstMetadata[s3_constants.AmzServerSideEncryptionCustomerKeyMD5] = []byte(destKey.KeyMD5) + + return encryptedContent, dstMetadata, nil + } + + // No encryption needed + return content, nil, nil +}