From cd2fac45514875d6f066d650d634fbc13ccedd40 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 26 Nov 2025 13:03:09 -0800 Subject: [PATCH] S3: pass HTTP 429 from volume servers to S3 clients (#7556) With the recent changes (commit c1b8d4bf0) that made S3 directly access volume servers instead of proxying through filer, we need to properly handle HTTP 429 (Too Many Requests) errors from volume servers. This change ensures that when volume servers rate limit requests with HTTP 429, the S3 API properly translates this to an S3-compatible error response (ErrRequestBytesExceed with HTTP 503) instead of returning a generic InternalError. Changes: - Add ErrTooManyRequests sentinel error in weed/util/http - Detect HTTP 429 in ReadUrlAsStream and wrap with ErrTooManyRequests - Check for ErrTooManyRequests in GetObjectHandler and map to S3 error - Return ErrRequestBytesExceed (HTTP 503) for rate limiting scenarios This addresses the same issue as PR #7482 but for the new direct volume server access path instead of the filer proxy path. Fixes: Rate limiting errors from volume servers being masked as 500 --- weed/s3api/s3api_object_handlers.go | 8 +++++++- weed/util/http/http_global_client_util.go | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index b1446c3e7..c403698e5 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -25,6 +25,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/util/mem" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" "github.com/seaweedfs/seaweedfs/weed/glog" ) @@ -740,7 +741,12 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) return } // Response not yet written - safe to write S3 error response - s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + // Check if error is due to volume server rate limiting (HTTP 429) + if errors.Is(err, util_http.ErrTooManyRequests) { + s3err.WriteErrorResponse(w, r, s3err.ErrRequestBytesExceed) + } else { + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + } return } } diff --git a/weed/util/http/http_global_client_util.go b/weed/util/http/http_global_client_util.go index 38f129365..3a969fdc8 100644 --- a/weed/util/http/http_global_client_util.go +++ b/weed/util/http/http_global_client_util.go @@ -24,6 +24,7 @@ import ( ) var ErrNotFound = fmt.Errorf("not found") +var ErrTooManyRequests = fmt.Errorf("too many requests") var ( jwtSigningReadKey security.SigningKey @@ -332,6 +333,9 @@ func ReadUrlAsStream(ctx context.Context, fileUrl, jwt string, cipherKey []byte, if r.StatusCode == http.StatusNotFound { return true, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrNotFound) } + if r.StatusCode == http.StatusTooManyRequests { + return false, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrTooManyRequests) + } retryable = r.StatusCode >= 499 return retryable, fmt.Errorf("%s: %s", fileUrl, r.Status) }