From bb444911ae9bb17cccdd97ab5b1ce6e5b8d1feec Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 30 Oct 2025 11:06:57 -0700 Subject: [PATCH 1/3] Add nginx reverse proxy documentation for S3 API Fixes #7407 Add comprehensive documentation and example configuration for using nginx as a reverse proxy with SeaweedFS S3 API while maintaining AWS Signature V4 authentication compatibility. Changes: - Add docker/nginx/README.md with detailed setup guide - Add docker/nginx/s3-example.conf with working configuration - Update docker/nginx/proxy.conf with important S3 notes The documentation covers: - Critical requirements for AWS Signature V4 authentication - Common mistakes and why they break S3 authentication - Complete working nginx configurations - Debugging tips and troubleshooting - Performance tuning recommendations --- docker/nginx/README.md | 353 +++++++++++++++++++++++++++++++++++ docker/nginx/proxy.conf | 10 +- docker/nginx/s3-example.conf | 83 ++++++++ 3 files changed, 445 insertions(+), 1 deletion(-) create mode 100644 docker/nginx/README.md create mode 100644 docker/nginx/s3-example.conf diff --git a/docker/nginx/README.md b/docker/nginx/README.md new file mode 100644 index 000000000..46f471487 --- /dev/null +++ b/docker/nginx/README.md @@ -0,0 +1,353 @@ +# Nginx Reverse Proxy Configuration for SeaweedFS S3 API + +This guide explains how to properly configure nginx as a reverse proxy for SeaweedFS S3 API while maintaining AWS Signature V4 authentication compatibility. + +## The Challenge + +AWS Signature V4 authentication calculates a cryptographic signature based on the exact request including headers, URI, and body. When using nginx as a reverse proxy, any modification to the signed request components will cause signature verification to fail. + +## Critical Requirements + +1. **Preserve the Authorization header**: Must pass through untouched +2. **Preserve all X-Amz-* headers**: These are part of the signature calculation +3. **Preserve the Host header**: Use `$http_host` instead of `$host` to maintain the original port +4. **Do not modify the request URI**: Avoid path rewriting +5. **Disable buffering for chunked uploads**: Required for streaming uploads +6. **Preserve the request body**: Must not be modified + +## Recommended Nginx Configuration + +### Basic Configuration for S3 API + +```nginx +upstream seaweedfs_s3 { + server s3:8333; + keepalive 32; +} + +server { + listen 443 ssl http2; + server_name your-domain.com; + + # SSL Configuration + ssl_certificate /etc/nginx/certs/server.crt; + ssl_certificate_key /etc/nginx/certs/server.key; + + # Logging + access_log /var/log/nginx/s3-access.log; + error_log /var/log/nginx/s3-error.log; + + # Client upload limits + client_max_body_size 0; # No limit for S3 uploads + client_body_timeout 300s; + + # Disable buffering for AWS chunked uploads + proxy_buffering off; + proxy_request_buffering off; + + # HTTP version and connection settings + proxy_http_version 1.1; + proxy_set_header Connection ""; + + # Timeouts + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + + location / { + proxy_pass http://seaweedfs_s3; + + # CRITICAL: Preserve original Host header including port + # Use $http_host instead of $host to preserve the port + proxy_set_header Host $http_host; + + # CRITICAL: Pass all headers through unchanged + # AWS Signature V4 includes these in signature calculation + proxy_pass_request_headers on; + + # Optional: Forward client IP information + # (These are NOT part of AWS signature) + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # CRITICAL: Do not modify request body + proxy_pass_request_body on; + + # Ignore invalid headers (S3 may send non-standard headers) + ignore_invalid_headers off; + } + + # Health check endpoint + location /health { + return 200 "OK\n"; + add_header Content-Type text/plain; + } +} +``` + +### Configuration with mTLS Client Certificate Authentication + +If you need to add mTLS authentication **in addition to** S3 authentication: + +```nginx +upstream seaweedfs_s3 { + server s3:8333; + keepalive 32; +} + +server { + listen 443 ssl http2; + server_name your-domain.com; + + # SSL Configuration + ssl_certificate /etc/nginx/certs/server.crt; + ssl_certificate_key /etc/nginx/certs/server.key; + + # mTLS Configuration + ssl_client_certificate /etc/nginx/certs/ca.crt; + ssl_verify_client optional; # or 'on' to require client certificates + ssl_verify_depth 2; + + # Logging with mTLS info + log_format mtls '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + 'verify=$ssl_client_verify dn="$ssl_client_s_dn"'; + + access_log /var/log/nginx/s3-mtls-access.log mtls; + error_log /var/log/nginx/s3-error.log; + + # Client upload limits + client_max_body_size 0; + client_body_timeout 300s; + + # Disable buffering for AWS chunked uploads + proxy_buffering off; + proxy_request_buffering off; + + # HTTP version and connection settings + proxy_http_version 1.1; + proxy_set_header Connection ""; + + # Timeouts + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + + location / { + proxy_pass http://seaweedfs_s3; + + # CRITICAL: Preserve original Host header + proxy_set_header Host $http_host; + + # CRITICAL: Pass all headers through unchanged + proxy_pass_request_headers on; + + # Forward client IP information + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Optional: Forward mTLS certificate info to backend + proxy_set_header X-SSL-Client-Verify $ssl_client_verify; + proxy_set_header X-SSL-Client-DN $ssl_client_s_dn; + + # CRITICAL: Do not modify request body + proxy_pass_request_body on; + + # Ignore invalid headers + ignore_invalid_headers off; + } +} +``` + +## Common Mistakes to Avoid + +### ❌ DO NOT explicitly set AWS headers + +```nginx +# WRONG - Do not do this! +proxy_set_header Authorization $http_authorization; +proxy_set_header X-Amz-Date $http_x_amz_date; +proxy_set_header X-Amz-Content-Sha256 $http_x_amz_content_sha256; +``` + +**Why?** When you explicitly set headers, nginx may normalize or modify them. Use `proxy_pass_request_headers on` instead to pass ALL headers through unchanged. + +### ❌ DO NOT use `proxy_set_header Host $host` + +```nginx +# WRONG - Loses port information +proxy_set_header Host $host; +``` + +**Why?** `$host` contains only the hostname without the port. AWS Signature V4 includes the Host header with port in signature calculation. Use `$http_host` instead. + +### ❌ DO NOT enable request buffering for large uploads + +```nginx +# WRONG for S3 uploads +proxy_request_buffering on; +``` + +**Why?** This breaks AWS chunked transfer encoding used for large file uploads. Keep it off for S3 API endpoints. + +### ❌ DO NOT rewrite paths + +```nginx +# WRONG - Path rewriting breaks signature +location /s3/ { + rewrite ^/s3/(.*) /$1 break; + proxy_pass http://seaweedfs_s3; +} +``` + +**Why?** The URI is part of the AWS Signature V4 calculation. Any path modification will cause signature verification to fail. + +## Testing Your Configuration + +### Test with AWS CLI + +```bash +# Configure AWS CLI +aws configure set aws_access_key_id your_access_key +aws configure set aws_secret_access_key your_secret_key +aws configure set region us-east-1 + +# Test through nginx proxy +aws s3 ls s3://your-bucket/ --endpoint-url https://your-nginx-domain +``` + +### Test with boto3 (Python) + +```python +import boto3 + +s3_client = boto3.client( + 's3', + endpoint_url='https://your-nginx-domain', + aws_access_key_id='your_access_key', + aws_secret_access_key='your_secret_key', + region_name='us-east-1' +) + +# List buckets +response = s3_client.list_buckets() +print(response) + +# Upload a file +s3_client.upload_file('local_file.txt', 'bucket-name', 'remote_file.txt') +``` + +## Debugging + +### Enable detailed logging + +Add to your nginx configuration: + +```nginx +error_log /var/log/nginx/error_debug.log debug; + +log_format detailed '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_authorization" "$http_x_amz_date" ' + 'upstream="$upstream_addr" ' + 'upstream_status=$upstream_status'; + +access_log /var/log/nginx/detailed.log detailed; +``` + +### Check SeaweedFS logs + +Look for authentication errors in SeaweedFS S3 logs: +- `could not find accessKey` - Access key not configured in SeaweedFS +- `signature mismatch` - Request was modified by proxy +- `InvalidAccessKeyId` - Access key doesn't exist +- `SignatureDoesNotMatch` - Signature calculation failed + +### Common Issues + +1. **"Could not find accessKey" error** + - Verify the access key exists in SeaweedFS: `weed shell` → `s3.configure.list` + - Check that Authorization header is being forwarded + - Verify SeaweedFS can read its configuration + +2. **"SignatureDoesNotMatch" error** + - Check if nginx is modifying headers or URI + - Verify `proxy_set_header Host $http_host` is used + - Disable any header manipulation + - Check for path rewriting rules + +3. **"RequestTimeTooSkewed" error** + - Ensure server clocks are synchronized (use NTP) + - Check timezone settings on both nginx and SeaweedFS servers + +## Performance Tuning + +For production deployments with high throughput: + +```nginx +upstream seaweedfs_s3 { + server s3:8333; + keepalive 64; # Increase keepalive connections + keepalive_timeout 90s; + keepalive_requests 10000; +} + +server { + # ... other config ... + + # Increase worker connections if needed + # (in nginx.conf, not server block) + # worker_connections 10000; + + # Enable caching for GET requests (optional) + proxy_cache_path /var/cache/nginx/s3 levels=1:2 keys_zone=s3_cache:10m + max_size=10g inactive=60m use_temp_path=off; + + location / { + # Only cache GET requests + proxy_cache s3_cache; + proxy_cache_methods GET HEAD; + proxy_cache_valid 200 60m; + proxy_cache_key "$scheme$request_method$host$request_uri"; + + # Don't cache authenticated requests by default + proxy_cache_bypass $http_authorization; + proxy_no_cache $http_authorization; + + # ... rest of proxy config ... + } +} +``` + +## Docker Compose Example + +```yaml +version: '3' + +services: + seaweedfs-s3: + image: chrislusf/seaweedfs:latest + ports: + - "8333:8333" + command: 's3 -ip=0.0.0.0 -port=8333' + volumes: + - ./s3_config:/etc/seaweedfs + + nginx: + image: nginx:latest + ports: + - "443:443" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + - ./certs:/etc/nginx/certs:ro + depends_on: + - seaweedfs-s3 +``` + +## Additional Resources + +- [AWS Signature Version 4 Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) +- [SeaweedFS S3 Configuration](https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API) +- [Nginx Proxy Module Documentation](https://nginx.org/en/docs/http/ngx_http_proxy_module.html) diff --git a/docker/nginx/proxy.conf b/docker/nginx/proxy.conf index 59ff30ce2..11042804a 100644 --- a/docker/nginx/proxy.conf +++ b/docker/nginx/proxy.conf @@ -1,5 +1,13 @@ # HTTP 1.1 support proxy_http_version 1.1; + +# IMPORTANT: For S3 API with AWS Signature V4 authentication, you MUST: +# 1. Use $http_host (not $host) to preserve port numbers +# 2. Keep proxy_pass_request_headers on (default) to pass all headers unchanged +# 3. Set proxy_request_buffering off for chunked uploads +# 4. NOT explicitly set Authorization or X-Amz-* headers +# See docker/nginx/README.md for detailed S3 reverse proxy configuration + #proxy_buffering off; proxy_set_header Host $http_host; proxy_set_header Upgrade $http_upgrade; @@ -20,7 +28,7 @@ proxy_buffers 64 1m; # buffers used for reading a response from the proxied ser proxy_buffer_size 8k; # maximum size of the data that nginx can receive from the server at a time is set proxy_busy_buffers_size 2m; -proxy_request_buffering on; # PUT buffering +proxy_request_buffering on; # PUT buffering (set to 'off' for S3 chunked uploads) client_body_buffer_size 64m; # buffer size for reading client request body client_max_body_size 64m; diff --git a/docker/nginx/s3-example.conf b/docker/nginx/s3-example.conf new file mode 100644 index 000000000..fd7830973 --- /dev/null +++ b/docker/nginx/s3-example.conf @@ -0,0 +1,83 @@ +# Example nginx configuration for SeaweedFS S3 API reverse proxy +# See README.md in this directory for detailed explanation + +upstream seaweedfs_s3 { + # Point to your SeaweedFS S3 service + server s3:8333; + # For local development: server 127.0.0.1:8333; + + # Keep connections alive for better performance + keepalive 32; +} + +server { + listen 443 ssl http2; + server_name _; # Replace with your domain + + # SSL Configuration + ssl_certificate /etc/nginx/certs/server.crt; + ssl_certificate_key /etc/nginx/certs/server.key; + + # Optional: Client certificate authentication (mTLS) + # ssl_client_certificate /etc/nginx/certs/ca.crt; + # ssl_verify_client optional; + # ssl_verify_depth 2; + + # Logging + access_log /var/log/nginx/s3-access.log; + error_log /var/log/nginx/s3-error.log; + + # Client upload limits + client_max_body_size 0; # No limit for S3 uploads + client_body_timeout 300s; + + # CRITICAL: Disable buffering for AWS chunked uploads + proxy_buffering off; + proxy_request_buffering off; + + # HTTP version and connection settings + proxy_http_version 1.1; + proxy_set_header Connection ""; + + # Timeouts + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + + location / { + proxy_pass http://seaweedfs_s3; + + # CRITICAL: Preserve original Host header including port + # Use $http_host instead of $host to preserve the port + proxy_set_header Host $http_host; + + # CRITICAL: Pass all headers through unchanged + # AWS Signature V4 includes these in signature calculation + proxy_pass_request_headers on; + + # Optional: Forward client IP information + # (These are NOT part of AWS signature) + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # CRITICAL: Do not modify request body + proxy_pass_request_body on; + + # Ignore invalid headers (S3 may send non-standard headers) + ignore_invalid_headers off; + } + + # Health check endpoint + location /health { + return 200 "OK\n"; + add_header Content-Type text/plain; + } +} + +# Optional: HTTP to HTTPS redirect +server { + listen 80; + server_name _; + return 301 https://$host$request_uri; +} From 836ab41d23345d92f7a705d0e1ed1ac0f4ad3bda Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 30 Oct 2025 20:27:44 -0700 Subject: [PATCH 2/3] Fix IPv6 host header formatting to match AWS SDK behavior Follow-up to PR #7403 When a default port (80 for HTTP, 443 for HTTPS) is stripped from an IPv6 address, the square brackets should also be removed to match AWS SDK behavior for S3 signature calculation. Reference: https://github.com/aws/aws-sdk-go-v2/blob/main/aws/signer/internal/v4/host.go The AWS SDK's stripPort function explicitly removes brackets when returning an IPv6 address without a port. Changes: - Update extractHostHeader to strip brackets from IPv6 addresses when no port or default port is used - Update test expectations to match AWS SDK behavior - Add detailed comments explaining the AWS SDK compatibility requirement This ensures S3 signature validation works correctly with IPv6 addresses behind reverse proxies, matching AWS S3 canonical request format. Fixes the issue raised in PR #7403 comment: https://github.com/seaweedfs/seaweedfs/pull/7403#issuecomment-3471105438 --- weed/s3api/auth_signature_v4.go | 11 ++++++++--- weed/s3api/auth_signature_v4_test.go | 12 ++++++------ weed/s3api/auto_signature_v4_test.go | 12 ++++++------ 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 4d273de6a..b77540255 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -648,9 +648,14 @@ func extractHostHeader(r *http.Request) string { return net.JoinHostPort(host, port) } - // No port or default port, just ensure host is correctly formatted (IPv6 brackets). - if strings.Contains(host, ":") && !strings.HasPrefix(host, "[") { - return "[" + host + "]" + // No port or default port was stripped. According to AWS SDK behavior (aws-sdk-go-v2), + // when a default port is removed from an IPv6 address, the brackets should also be removed. + // This matches AWS S3 signature calculation requirements. + // Reference: https://github.com/aws/aws-sdk-go-v2/blob/main/aws/signer/internal/v4/host.go + // The stripPort function returns IPv6 without brackets when port is stripped. + if strings.Contains(host, ":") { + // This is an IPv6 address. Strip brackets to match AWS SDK behavior. + return strings.Trim(host, "[]") } return host } diff --git a/weed/s3api/auth_signature_v4_test.go b/weed/s3api/auth_signature_v4_test.go index 6850e9d2b..9ec4f232e 100644 --- a/weed/s3api/auth_signature_v4_test.go +++ b/weed/s3api/auth_signature_v4_test.go @@ -192,20 +192,20 @@ func TestExtractHostHeader(t *testing.T) { expected: "[::1]:8080", }, { - name: "IPv6 address without brackets and standard port, should return bracketed IPv6", + name: "IPv6 address without brackets and standard port, should strip brackets per AWS SDK", hostHeader: "backend:8333", forwardedHost: "::1", forwardedPort: "80", forwardedProto: "http", - expected: "[::1]", + expected: "::1", }, { - name: "IPv6 address without brackets and standard HTTPS port, should return bracketed IPv6", + name: "IPv6 address without brackets and standard HTTPS port, should strip brackets per AWS SDK", hostHeader: "backend:8333", forwardedHost: "2001:db8::1", forwardedPort: "443", forwardedProto: "https", - expected: "[2001:db8::1]", + expected: "2001:db8::1", }, { name: "IPv6 address with brackets but no port, should add port", @@ -216,12 +216,12 @@ func TestExtractHostHeader(t *testing.T) { expected: "[2001:db8::1]:8080", }, { - name: "IPv6 full address with brackets and default port (should strip port)", + name: "IPv6 full address with brackets and default port (should strip port and brackets)", hostHeader: "backend:8333", forwardedHost: "[2001:db8:85a3::8a2e:370:7334]:443", forwardedPort: "443", forwardedProto: "https", - expected: "[2001:db8:85a3::8a2e:370:7334]", + expected: "2001:db8:85a3::8a2e:370:7334", }, { name: "IPv4-mapped IPv6 address without brackets, should add brackets with port", diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index 6039081a1..47c55e077 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -451,25 +451,25 @@ func TestSignatureV4WithoutProxy(t *testing.T) { name: "IPv6 HTTP with standard port", host: "[::1]:80", proto: "http", - expectedHost: "[::1]", + expectedHost: "::1", }, { name: "IPv6 HTTPS with standard port", host: "[::1]:443", proto: "https", - expectedHost: "[::1]", + expectedHost: "::1", }, { name: "IPv6 HTTP without port", host: "::1", proto: "http", - expectedHost: "[::1]", + expectedHost: "::1", }, { name: "IPv6 HTTPS without port", host: "::1", proto: "https", - expectedHost: "[::1]", + expectedHost: "::1", }, } @@ -608,7 +608,7 @@ func TestSignatureV4WithForwardedPort(t *testing.T) { forwardedHost: "[::1]:443", forwardedPort: "443", forwardedProto: "https", - expectedHost: "[::1]", + expectedHost: "::1", }, { name: "IPv6 X-Forwarded-Host with standard http port already included (Traefik/HAProxy style)", @@ -616,7 +616,7 @@ func TestSignatureV4WithForwardedPort(t *testing.T) { forwardedHost: "[::1]:80", forwardedPort: "80", forwardedProto: "http", - expectedHost: "[::1]", + expectedHost: "::1", }, { name: "IPv6 with port in brackets", From 1f1957020682b0b02e66891ea110ea2f2a93c4ea Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 30 Oct 2025 20:31:32 -0700 Subject: [PATCH 3/3] Update docker/nginx/README.md Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- docker/nginx/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/README.md b/docker/nginx/README.md index 46f471487..0d0dd3d45 100644 --- a/docker/nginx/README.md +++ b/docker/nginx/README.md @@ -80,8 +80,8 @@ server { # Health check endpoint location /health { - return 200 "OK\n"; add_header Content-Type text/plain; + return 200 "OK\n"; } } ```