From d4df6ca481c04015182658bf54a394e063af15f3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 25 Nov 2025 20:39:31 -0800 Subject: [PATCH] Remove filer address from upload paths - pass path directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eliminate unnecessary filer address from upload URLs by passing file paths directly instead of full URLs that get immediately parsed. Changes: - Rename genPartUploadUrl() → genPartUploadPath() (returns path only) - Rename toFilerUrl() → toFilerPath() (returns path only) - Update putToFiler() to accept filePath instead of uploadUrl - Remove URL parsing code (no longer needed) - Remove net/url import (no longer used) - Keep old function names as deprecated wrappers for compatibility Benefits: - Cleaner code - no fake URL construction/parsing - No dependency on filer address for internal operations - More accurate naming (these are paths, not URLs) - Eliminates confusion about HA concerns This completely removes the filer address from upload operations - it was never actually used for routing, only parsed for the path. --- weed/s3api/s3api_object_handlers.go | 11 +++++++---- weed/s3api/s3api_object_handlers_multipart.go | 14 +++++++++----- weed/s3api/s3api_object_handlers_put.go | 19 +++---------------- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index d65c1b69a..f7447251f 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -404,11 +404,14 @@ func newListEntry(entry *filer_pb.Entry, key string, dir string, name string, bu return listEntry } -func (s3a *S3ApiServer) toFilerUrl(bucket, object string) string { +func (s3a *S3ApiServer) toFilerPath(bucket, object string) string { object = urlPathEscape(removeDuplicateSlashes(object)) - destUrl := fmt.Sprintf("http://%s%s/%s%s", - s3a.getFilerAddress().ToHttpAddress(), s3a.option.BucketsPath, bucket, object) - return destUrl + return fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object) +} + +// Deprecated: Use toFilerPath instead - no need for full URL +func (s3a *S3ApiServer) toFilerUrl(bucket, object string) string { + return s3a.toFilerPath(bucket, object) } // hasConditionalHeaders checks if the request has any conditional headers diff --git a/weed/s3api/s3api_object_handlers_multipart.go b/weed/s3api/s3api_object_handlers_multipart.go index c17ab3da6..ab69107e1 100644 --- a/weed/s3api/s3api_object_handlers_multipart.go +++ b/weed/s3api/s3api_object_handlers_multipart.go @@ -437,12 +437,16 @@ func (s3a *S3ApiServer) genUploadsFolder(bucket string) string { return fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, s3_constants.MultipartUploadsFolder) } +func (s3a *S3ApiServer) genPartUploadPath(bucket, uploadID string, partID int) string { + // Returns just the file path - no filer address needed + // Upload traffic goes directly to volume servers, not through filer + return fmt.Sprintf("%s/%s/%04d_%s.part", + s3a.genUploadsFolder(bucket), uploadID, partID, uuid.NewString()) +} + +// Deprecated: Use genPartUploadPath instead - no need for full URL func (s3a *S3ApiServer) genPartUploadUrl(bucket, uploadID string, partID int) string { - // Note: This URL is used internally to generate the file path for putToFiler - // The filer address is parsed out - actual upload goes directly to volume servers - // The filer address here is not actually used for upload traffic - return fmt.Sprintf("http://%s%s/%s/%04d_%s.part", - s3a.getFilerAddress().ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID, uuid.NewString()) + return s3a.genPartUploadPath(bucket, uploadID, partID) } // Generate uploadID hash string from object diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index 540a1e512..fce329b03 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "net/http" - "net/url" "path/filepath" "strconv" "strings" @@ -248,9 +247,10 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) writeSuccessResponseEmpty(w, r) } -func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader, bucket string, partNumber int) (etag string, code s3err.ErrorCode, sseMetadata SSEResponseMetadata) { +func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader io.Reader, bucket string, partNumber int) (etag string, code s3err.ErrorCode, sseMetadata SSEResponseMetadata) { // NEW OPTIMIZATION: Write directly to volume servers, bypassing filer proxy // This eliminates the filer proxy overhead for PUT operations + // Note: filePath is now passed directly instead of URL (no parsing needed) // For SSE, encrypt with offset=0 for all parts // Each part is encrypted independently, then decrypted using metadata during GET @@ -311,20 +311,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader glog.V(4).Infof("putToFiler: explicit encryption already applied, skipping bucket default encryption") } - // Parse the upload URL to extract the file path - // uploadUrl format: http://filer:8888/path/to/bucket/object (or https://, IPv6, etc.) - // Use proper URL parsing instead of string manipulation for robustness - parsedUrl, parseErr := url.Parse(uploadUrl) - if parseErr != nil { - glog.Errorf("putToFiler: failed to parse uploadUrl %q: %v", uploadUrl, parseErr) - return "", s3err.ErrInternalError, SSEResponseMetadata{} - } - - // Use parsedUrl.Path directly - it's already decoded by url.Parse() - // Per Go documentation: "Path is stored in decoded form: /%47%6f%2f becomes /Go/" - // Calling PathUnescape again would double-decode and fail on keys like "b%ar" - filePath := parsedUrl.Path - + // filePath is already provided directly - no URL parsing needed // Step 1 & 2: Use auto-chunking to handle large files without OOM // This splits large uploads into 8MB chunks, preventing memory issues on both S3 API and volume servers const chunkSize = 8 * 1024 * 1024 // 8MB chunks (S3 standard)