From bb444911ae9bb17cccdd97ab5b1ce6e5b8d1feec Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Thu, 30 Oct 2025 11:06:57 -0700
Subject: [PATCH] Add nginx reverse proxy documentation for S3 API

Fixes #7407

Add comprehensive documentation and example configuration for using
nginx as a reverse proxy with SeaweedFS S3 API while maintaining AWS
Signature V4 authentication compatibility.

Changes:
- Add docker/nginx/README.md with detailed setup guide
- Add docker/nginx/s3-example.conf with working configuration
- Update docker/nginx/proxy.conf with important S3 notes

The documentation covers:
- Critical requirements for AWS Signature V4 authentication
- Common mistakes and why they break S3 authentication
- Complete working nginx configurations
- Debugging tips and troubleshooting
- Performance tuning recommendations
---
 docker/nginx/README.md       | 353 +++++++++++++++++++++++++++++++++++
 docker/nginx/proxy.conf      |  10 +-
 docker/nginx/s3-example.conf |  83 ++++++++
 3 files changed, 445 insertions(+), 1 deletion(-)
 create mode 100644 docker/nginx/README.md
 create mode 100644 docker/nginx/s3-example.conf

diff --git a/docker/nginx/README.md b/docker/nginx/README.md
new file mode 100644
index 000000000..46f471487
--- /dev/null
+++ b/docker/nginx/README.md
@@ -0,0 +1,353 @@
+# Nginx Reverse Proxy Configuration for SeaweedFS S3 API
+
+This guide explains how to properly configure nginx as a reverse proxy for SeaweedFS S3 API while maintaining AWS Signature V4 authentication compatibility.
+
+## The Challenge
+
+AWS Signature V4 authentication calculates a cryptographic signature based on the exact request including headers, URI, and body. When using nginx as a reverse proxy, any modification to the signed request components will cause signature verification to fail.
+
+## Critical Requirements
+
+1. **Preserve the Authorization header**: Must pass through untouched
+2. **Preserve all X-Amz-* headers**: These are part of the signature calculation
+3. **Preserve the Host header**: Use `$http_host` instead of `$host` to maintain the original port
+4. **Do not modify the request URI**: Avoid path rewriting
+5. **Disable buffering for chunked uploads**: Required for streaming uploads
+6. **Preserve the request body**: Must not be modified
+
+## Recommended Nginx Configuration
+
+### Basic Configuration for S3 API
+
+```nginx
+upstream seaweedfs_s3 {
+    server s3:8333;
+    keepalive 32;
+}
+
+server {
+    listen 443 ssl http2;
+    server_name your-domain.com;
+
+    # SSL Configuration
+    ssl_certificate     /etc/nginx/certs/server.crt;
+    ssl_certificate_key /etc/nginx/certs/server.key;
+
+    # Logging
+    access_log /var/log/nginx/s3-access.log;
+    error_log /var/log/nginx/s3-error.log;
+
+    # Client upload limits
+    client_max_body_size 0;  # No limit for S3 uploads
+    client_body_timeout 300s;
+    
+    # Disable buffering for AWS chunked uploads
+    proxy_buffering off;
+    proxy_request_buffering off;
+
+    # HTTP version and connection settings
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+    
+    # Timeouts
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300s;
+    proxy_read_timeout 300s;
+
+    location / {
+        proxy_pass http://seaweedfs_s3;
+
+        # CRITICAL: Preserve original Host header including port
+        # Use $http_host instead of $host to preserve the port
+        proxy_set_header Host $http_host;
+
+        # CRITICAL: Pass all headers through unchanged
+        # AWS Signature V4 includes these in signature calculation
+        proxy_pass_request_headers on;
+
+        # Optional: Forward client IP information
+        # (These are NOT part of AWS signature)
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        # CRITICAL: Do not modify request body
+        proxy_pass_request_body on;
+
+        # Ignore invalid headers (S3 may send non-standard headers)
+        ignore_invalid_headers off;
+    }
+
+    # Health check endpoint
+    location /health {
+        return 200 "OK\n";
+        add_header Content-Type text/plain;
+    }
+}
+```
+
+### Configuration with mTLS Client Certificate Authentication
+
+If you need to add mTLS authentication **in addition to** S3 authentication:
+
+```nginx
+upstream seaweedfs_s3 {
+    server s3:8333;
+    keepalive 32;
+}
+
+server {
+    listen 443 ssl http2;
+    server_name your-domain.com;
+
+    # SSL Configuration
+    ssl_certificate     /etc/nginx/certs/server.crt;
+    ssl_certificate_key /etc/nginx/certs/server.key;
+    
+    # mTLS Configuration
+    ssl_client_certificate /etc/nginx/certs/ca.crt;
+    ssl_verify_client optional;  # or 'on' to require client certificates
+    ssl_verify_depth 2;
+
+    # Logging with mTLS info
+    log_format mtls '$remote_addr - $remote_user [$time_local] '
+                    '"$request" $status $body_bytes_sent '
+                    'verify=$ssl_client_verify dn="$ssl_client_s_dn"';
+    
+    access_log /var/log/nginx/s3-mtls-access.log mtls;
+    error_log /var/log/nginx/s3-error.log;
+
+    # Client upload limits
+    client_max_body_size 0;
+    client_body_timeout 300s;
+    
+    # Disable buffering for AWS chunked uploads
+    proxy_buffering off;
+    proxy_request_buffering off;
+
+    # HTTP version and connection settings
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+    
+    # Timeouts
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300s;
+    proxy_read_timeout 300s;
+
+    location / {
+        proxy_pass http://seaweedfs_s3;
+
+        # CRITICAL: Preserve original Host header
+        proxy_set_header Host $http_host;
+
+        # CRITICAL: Pass all headers through unchanged
+        proxy_pass_request_headers on;
+
+        # Forward client IP information
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        
+        # Optional: Forward mTLS certificate info to backend
+        proxy_set_header X-SSL-Client-Verify $ssl_client_verify;
+        proxy_set_header X-SSL-Client-DN $ssl_client_s_dn;
+
+        # CRITICAL: Do not modify request body
+        proxy_pass_request_body on;
+
+        # Ignore invalid headers
+        ignore_invalid_headers off;
+    }
+}
+```
+
+## Common Mistakes to Avoid
+
+### ❌ DO NOT explicitly set AWS headers
+
+```nginx
+# WRONG - Do not do this!
+proxy_set_header Authorization $http_authorization;
+proxy_set_header X-Amz-Date $http_x_amz_date;
+proxy_set_header X-Amz-Content-Sha256 $http_x_amz_content_sha256;
+```
+
+**Why?** When you explicitly set headers, nginx may normalize or modify them. Use `proxy_pass_request_headers on` instead to pass ALL headers through unchanged.
+
+### ❌ DO NOT use `proxy_set_header Host $host`
+
+```nginx
+# WRONG - Loses port information
+proxy_set_header Host $host;
+```
+
+**Why?** `$host` contains only the hostname without the port. AWS Signature V4 includes the Host header with port in signature calculation. Use `$http_host` instead.
+
+### ❌ DO NOT enable request buffering for large uploads
+
+```nginx
+# WRONG for S3 uploads
+proxy_request_buffering on;
+```
+
+**Why?** This breaks AWS chunked transfer encoding used for large file uploads. Keep it off for S3 API endpoints.
+
+### ❌ DO NOT rewrite paths
+
+```nginx
+# WRONG - Path rewriting breaks signature
+location /s3/ {
+    rewrite ^/s3/(.*) /$1 break;
+    proxy_pass http://seaweedfs_s3;
+}
+```
+
+**Why?** The URI is part of the AWS Signature V4 calculation. Any path modification will cause signature verification to fail.
+
+## Testing Your Configuration
+
+### Test with AWS CLI
+
+```bash
+# Configure AWS CLI
+aws configure set aws_access_key_id your_access_key
+aws configure set aws_secret_access_key your_secret_key
+aws configure set region us-east-1
+
+# Test through nginx proxy
+aws s3 ls s3://your-bucket/ --endpoint-url https://your-nginx-domain
+```
+
+### Test with boto3 (Python)
+
+```python
+import boto3
+
+s3_client = boto3.client(
+    's3',
+    endpoint_url='https://your-nginx-domain',
+    aws_access_key_id='your_access_key',
+    aws_secret_access_key='your_secret_key',
+    region_name='us-east-1'
+)
+
+# List buckets
+response = s3_client.list_buckets()
+print(response)
+
+# Upload a file
+s3_client.upload_file('local_file.txt', 'bucket-name', 'remote_file.txt')
+```
+
+## Debugging
+
+### Enable detailed logging
+
+Add to your nginx configuration:
+
+```nginx
+error_log /var/log/nginx/error_debug.log debug;
+
+log_format detailed '$remote_addr - $remote_user [$time_local] '
+                    '"$request" $status $body_bytes_sent '
+                    '"$http_authorization" "$http_x_amz_date" '
+                    'upstream="$upstream_addr" '
+                    'upstream_status=$upstream_status';
+
+access_log /var/log/nginx/detailed.log detailed;
+```
+
+### Check SeaweedFS logs
+
+Look for authentication errors in SeaweedFS S3 logs:
+- `could not find accessKey` - Access key not configured in SeaweedFS
+- `signature mismatch` - Request was modified by proxy
+- `InvalidAccessKeyId` - Access key doesn't exist
+- `SignatureDoesNotMatch` - Signature calculation failed
+
+### Common Issues
+
+1. **"Could not find accessKey" error**
+   - Verify the access key exists in SeaweedFS: `weed shell` → `s3.configure.list`
+   - Check that Authorization header is being forwarded
+   - Verify SeaweedFS can read its configuration
+
+2. **"SignatureDoesNotMatch" error**
+   - Check if nginx is modifying headers or URI
+   - Verify `proxy_set_header Host $http_host` is used
+   - Disable any header manipulation
+   - Check for path rewriting rules
+
+3. **"RequestTimeTooSkewed" error**
+   - Ensure server clocks are synchronized (use NTP)
+   - Check timezone settings on both nginx and SeaweedFS servers
+
+## Performance Tuning
+
+For production deployments with high throughput:
+
+```nginx
+upstream seaweedfs_s3 {
+    server s3:8333;
+    keepalive 64;  # Increase keepalive connections
+    keepalive_timeout 90s;
+    keepalive_requests 10000;
+}
+
+server {
+    # ... other config ...
+    
+    # Increase worker connections if needed
+    # (in nginx.conf, not server block)
+    # worker_connections 10000;
+    
+    # Enable caching for GET requests (optional)
+    proxy_cache_path /var/cache/nginx/s3 levels=1:2 keys_zone=s3_cache:10m 
+                     max_size=10g inactive=60m use_temp_path=off;
+    
+    location / {
+        # Only cache GET requests
+        proxy_cache s3_cache;
+        proxy_cache_methods GET HEAD;
+        proxy_cache_valid 200 60m;
+        proxy_cache_key "$scheme$request_method$host$request_uri";
+        
+        # Don't cache authenticated requests by default
+        proxy_cache_bypass $http_authorization;
+        proxy_no_cache $http_authorization;
+        
+        # ... rest of proxy config ...
+    }
+}
+```
+
+## Docker Compose Example
+
+```yaml
+version: '3'
+
+services:
+  seaweedfs-s3:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "8333:8333"
+    command: 's3 -ip=0.0.0.0 -port=8333'
+    volumes:
+      - ./s3_config:/etc/seaweedfs
+
+  nginx:
+    image: nginx:latest
+    ports:
+      - "443:443"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./certs:/etc/nginx/certs:ro
+    depends_on:
+      - seaweedfs-s3
+```
+
+## Additional Resources
+
+- [AWS Signature Version 4 Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html)
+- [SeaweedFS S3 Configuration](https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API)
+- [Nginx Proxy Module Documentation](https://nginx.org/en/docs/http/ngx_http_proxy_module.html)
diff --git a/docker/nginx/proxy.conf b/docker/nginx/proxy.conf
index 59ff30ce2..11042804a 100644
--- a/docker/nginx/proxy.conf
+++ b/docker/nginx/proxy.conf
@@ -1,5 +1,13 @@
 # HTTP 1.1 support
 proxy_http_version 1.1;
+
+# IMPORTANT: For S3 API with AWS Signature V4 authentication, you MUST:
+# 1. Use $http_host (not $host) to preserve port numbers
+# 2. Keep proxy_pass_request_headers on (default) to pass all headers unchanged
+# 3. Set proxy_request_buffering off for chunked uploads
+# 4. NOT explicitly set Authorization or X-Amz-* headers
+# See docker/nginx/README.md for detailed S3 reverse proxy configuration
+
 #proxy_buffering off;
 proxy_set_header Host $http_host;
 proxy_set_header Upgrade $http_upgrade;
@@ -20,7 +28,7 @@ proxy_buffers 64 1m;  # buffers used for reading a response from the proxied ser
 proxy_buffer_size 8k; # maximum size of the data that nginx can receive from the server at a time is set
 proxy_busy_buffers_size 2m;
 
-proxy_request_buffering on;  # PUT buffering
+proxy_request_buffering on;  # PUT buffering (set to 'off' for S3 chunked uploads)
 client_body_buffer_size 64m; # buffer size for reading client request body
 client_max_body_size    64m;
 
diff --git a/docker/nginx/s3-example.conf b/docker/nginx/s3-example.conf
new file mode 100644
index 000000000..fd7830973
--- /dev/null
+++ b/docker/nginx/s3-example.conf
@@ -0,0 +1,83 @@
+# Example nginx configuration for SeaweedFS S3 API reverse proxy
+# See README.md in this directory for detailed explanation
+
+upstream seaweedfs_s3 {
+    # Point to your SeaweedFS S3 service
+    server s3:8333;
+    # For local development: server 127.0.0.1:8333;
+    
+    # Keep connections alive for better performance
+    keepalive 32;
+}
+
+server {
+    listen 443 ssl http2;
+    server_name _;  # Replace with your domain
+
+    # SSL Configuration
+    ssl_certificate     /etc/nginx/certs/server.crt;
+    ssl_certificate_key /etc/nginx/certs/server.key;
+    
+    # Optional: Client certificate authentication (mTLS)
+    # ssl_client_certificate /etc/nginx/certs/ca.crt;
+    # ssl_verify_client optional;
+    # ssl_verify_depth 2;
+
+    # Logging
+    access_log /var/log/nginx/s3-access.log;
+    error_log /var/log/nginx/s3-error.log;
+
+    # Client upload limits
+    client_max_body_size 0;  # No limit for S3 uploads
+    client_body_timeout 300s;
+    
+    # CRITICAL: Disable buffering for AWS chunked uploads
+    proxy_buffering off;
+    proxy_request_buffering off;
+
+    # HTTP version and connection settings
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+    
+    # Timeouts
+    proxy_connect_timeout 300s;
+    proxy_send_timeout 300s;
+    proxy_read_timeout 300s;
+
+    location / {
+        proxy_pass http://seaweedfs_s3;
+
+        # CRITICAL: Preserve original Host header including port
+        # Use $http_host instead of $host to preserve the port
+        proxy_set_header Host $http_host;
+
+        # CRITICAL: Pass all headers through unchanged
+        # AWS Signature V4 includes these in signature calculation
+        proxy_pass_request_headers on;
+
+        # Optional: Forward client IP information
+        # (These are NOT part of AWS signature)
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        # CRITICAL: Do not modify request body
+        proxy_pass_request_body on;
+
+        # Ignore invalid headers (S3 may send non-standard headers)
+        ignore_invalid_headers off;
+    }
+
+    # Health check endpoint
+    location /health {
+        return 200 "OK\n";
+        add_header Content-Type text/plain;
+    }
+}
+
+# Optional: HTTP to HTTPS redirect
+server {
+    listen 80;
+    server_name _;
+    return 301 https://$host$request_uri;
+}