Browse Source

Remove filer address from upload paths - pass path directly

Eliminate unnecessary filer address from upload URLs by passing file
paths directly instead of full URLs that get immediately parsed.

Changes:
- Rename genPartUploadUrl() → genPartUploadPath() (returns path only)
- Rename toFilerUrl() → toFilerPath() (returns path only)
- Update putToFiler() to accept filePath instead of uploadUrl
- Remove URL parsing code (no longer needed)
- Remove net/url import (no longer used)
- Keep old function names as deprecated wrappers for compatibility

Benefits:
- Cleaner code - no fake URL construction/parsing
- No dependency on filer address for internal operations
- More accurate naming (these are paths, not URLs)
- Eliminates confusion about HA concerns

This completely removes the filer address from upload operations - it was
never actually used for routing, only parsed for the path.
pull/7550/head
Chris Lu 4 days ago
parent
commit
d4df6ca481
  1. 11
      weed/s3api/s3api_object_handlers.go
  2. 14
      weed/s3api/s3api_object_handlers_multipart.go
  3. 19
      weed/s3api/s3api_object_handlers_put.go

11
weed/s3api/s3api_object_handlers.go

@ -404,11 +404,14 @@ func newListEntry(entry *filer_pb.Entry, key string, dir string, name string, bu
return listEntry return listEntry
} }
func (s3a *S3ApiServer) toFilerUrl(bucket, object string) string {
func (s3a *S3ApiServer) toFilerPath(bucket, object string) string {
object = urlPathEscape(removeDuplicateSlashes(object)) object = urlPathEscape(removeDuplicateSlashes(object))
destUrl := fmt.Sprintf("http://%s%s/%s%s",
s3a.getFilerAddress().ToHttpAddress(), s3a.option.BucketsPath, bucket, object)
return destUrl
return fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)
}
// Deprecated: Use toFilerPath instead - no need for full URL
func (s3a *S3ApiServer) toFilerUrl(bucket, object string) string {
return s3a.toFilerPath(bucket, object)
} }
// hasConditionalHeaders checks if the request has any conditional headers // hasConditionalHeaders checks if the request has any conditional headers

14
weed/s3api/s3api_object_handlers_multipart.go

@ -437,12 +437,16 @@ func (s3a *S3ApiServer) genUploadsFolder(bucket string) string {
return fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, s3_constants.MultipartUploadsFolder) return fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, s3_constants.MultipartUploadsFolder)
} }
func (s3a *S3ApiServer) genPartUploadPath(bucket, uploadID string, partID int) string {
// Returns just the file path - no filer address needed
// Upload traffic goes directly to volume servers, not through filer
return fmt.Sprintf("%s/%s/%04d_%s.part",
s3a.genUploadsFolder(bucket), uploadID, partID, uuid.NewString())
}
// Deprecated: Use genPartUploadPath instead - no need for full URL
func (s3a *S3ApiServer) genPartUploadUrl(bucket, uploadID string, partID int) string { func (s3a *S3ApiServer) genPartUploadUrl(bucket, uploadID string, partID int) string {
// Note: This URL is used internally to generate the file path for putToFiler
// The filer address is parsed out - actual upload goes directly to volume servers
// The filer address here is not actually used for upload traffic
return fmt.Sprintf("http://%s%s/%s/%04d_%s.part",
s3a.getFilerAddress().ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID, uuid.NewString())
return s3a.genPartUploadPath(bucket, uploadID, partID)
} }
// Generate uploadID hash string from object // Generate uploadID hash string from object

19
weed/s3api/s3api_object_handlers_put.go

@ -8,7 +8,6 @@ import (
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"net/url"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
@ -248,9 +247,10 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
writeSuccessResponseEmpty(w, r) writeSuccessResponseEmpty(w, r)
} }
func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader, bucket string, partNumber int) (etag string, code s3err.ErrorCode, sseMetadata SSEResponseMetadata) {
func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader io.Reader, bucket string, partNumber int) (etag string, code s3err.ErrorCode, sseMetadata SSEResponseMetadata) {
// NEW OPTIMIZATION: Write directly to volume servers, bypassing filer proxy // NEW OPTIMIZATION: Write directly to volume servers, bypassing filer proxy
// This eliminates the filer proxy overhead for PUT operations // This eliminates the filer proxy overhead for PUT operations
// Note: filePath is now passed directly instead of URL (no parsing needed)
// For SSE, encrypt with offset=0 for all parts // For SSE, encrypt with offset=0 for all parts
// Each part is encrypted independently, then decrypted using metadata during GET // Each part is encrypted independently, then decrypted using metadata during GET
@ -311,20 +311,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader
glog.V(4).Infof("putToFiler: explicit encryption already applied, skipping bucket default encryption") glog.V(4).Infof("putToFiler: explicit encryption already applied, skipping bucket default encryption")
} }
// Parse the upload URL to extract the file path
// uploadUrl format: http://filer:8888/path/to/bucket/object (or https://, IPv6, etc.)
// Use proper URL parsing instead of string manipulation for robustness
parsedUrl, parseErr := url.Parse(uploadUrl)
if parseErr != nil {
glog.Errorf("putToFiler: failed to parse uploadUrl %q: %v", uploadUrl, parseErr)
return "", s3err.ErrInternalError, SSEResponseMetadata{}
}
// Use parsedUrl.Path directly - it's already decoded by url.Parse()
// Per Go documentation: "Path is stored in decoded form: /%47%6f%2f becomes /Go/"
// Calling PathUnescape again would double-decode and fail on keys like "b%ar"
filePath := parsedUrl.Path
// filePath is already provided directly - no URL parsing needed
// Step 1 & 2: Use auto-chunking to handle large files without OOM // Step 1 & 2: Use auto-chunking to handle large files without OOM
// This splits large uploads into 8MB chunks, preventing memory issues on both S3 API and volume servers // This splits large uploads into 8MB chunks, preventing memory issues on both S3 API and volume servers
const chunkSize = 8 * 1024 * 1024 // 8MB chunks (S3 standard) const chunkSize = 8 * 1024 * 1024 // 8MB chunks (S3 standard)

Loading…
Cancel
Save