From fa8698f6c349dc426fe6b832274d617b9500256f Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Sun, 23 Apr 2023 17:04:37 +0800 Subject: [PATCH] perf(fix): fix bugs Signed-off-by: changlin.shi --- weed/s3api/bucket_metadata.go | 2 +- weed/s3api/s3api_acp.go | 13 ++++--------- weed/s3api/s3api_bucket_handlers.go | 3 ++- weed/s3api/s3api_object_copy_handlers.go | 8 ++++++-- weed/s3api/s3api_server.go | 2 ++ .../server/filer_server_handlers_write_autochunk.go | 2 +- weed/util/http_util.go | 1 + 7 files changed, 17 insertions(+), 14 deletions(-) diff --git a/weed/s3api/bucket_metadata.go b/weed/s3api/bucket_metadata.go index f4dd4cfe6..d3e47810d 100644 --- a/weed/s3api/bucket_metadata.go +++ b/weed/s3api/bucket_metadata.go @@ -190,7 +190,7 @@ func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*Bucket if err != nil { if err == filer_pb.ErrNotFound { // The bucket doesn't actually exist and should no longer loaded from the filer - glog.Warning("bucket not found in filer: ", bucketName) + glog.V(4).Info("bucket not found in filer: ", bucketName) r.notFound[bucketName] = struct{}{} return nil, s3err.ErrNoSuchBucket } diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index 528bf7e1d..bed718055 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -34,7 +34,7 @@ func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s return s3err.ErrAccessDenied } -//Check access for PutBucketAclHandler +// Check access for PutBucketAclHandler func (s3a *S3ApiServer) checkAccessForPutBucketAcl(requestAccountId, bucket string) (*BucketMetaData, s3err.ErrorCode) { bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) if errCode != s3err.ErrNone { @@ -104,7 +104,7 @@ func (s3a *S3ApiServer) checkAccessForReadBucket(r *http.Request, bucket, aclAct return nil, s3err.ErrAccessDenied } -//Check ObjectAcl-Read related access +// Check ObjectAcl-Read related access // includes: // - GetObjectAclHandler func (s3a *S3ApiServer) checkAccessForReadObjectAcl(r *http.Request, bucket, object string) (acp *s3.AccessControlPolicy, errCode s3err.ErrorCode) { @@ -370,14 +370,9 @@ func (s3a *S3ApiServer) checkAccessForWriteObject(r *http.Request, bucket, objec } objectOwnerId := s3acl.GetAcpOwner(entry.Extended, *bucketMetadata.Owner.ID) - //object owner is immutable - if requestOwnerId != "" && objectOwnerId != requestOwnerId { - return s3err.ErrAccessDenied - } - //Only the owner of the bucket and the owner of the object can overwrite the object - if s3acl.ValidateAccount(requestOwnerId, objectOwnerId, *bucketMetadata.Owner.ID) { - glog.V(3).Infof("checkAccessForWriteObject denied! request account id: %s, expect account id: %s", requestAccountId, *bucketMetadata.Owner.ID) + //object owner is immutable + if !s3acl.ValidateAccount(requestOwnerId, objectOwnerId, *bucketMetadata.Owner.ID) { return s3err.ErrAccessDenied } diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index ad9f6abac..7f07c0f9d 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -6,9 +6,9 @@ import ( "errors" "fmt" "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket" "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket" "github.com/seaweedfs/seaweedfs/weed/util" "math" "net/http" @@ -146,6 +146,7 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) entry.Extended = make(map[string][]byte) } entry.Extended[s3_constants.AmzIdentityId] = []byte(identityId) + s3a.bucketRegistry.LoadBucketMetadata(entry) } if objectOwnership != "" { if entry.Extended == nil { diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go index 83120ef07..9bdc1cce3 100644 --- a/weed/s3api/s3api_object_copy_handlers.go +++ b/weed/s3api/s3api_object_copy_handlers.go @@ -98,8 +98,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request return } glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl) - destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject) - etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body, destination) + etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body, "") if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) @@ -213,6 +212,11 @@ func processMetadata(reqHeader, existing http.Header, replaceMeta, replaceTaggin } } + // content-encoding + if contentEncoding, ok := existing["Content-Encoding"]; ok { + reqHeader["Content-Encoding"] = contentEncoding + } + if !replaceMeta { for header, _ := range reqHeader { if strings.HasPrefix(header, s3_constants.AmzUserMetaPrefix) { diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index ac181eac2..30e2e6bb6 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -68,6 +68,7 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer s3ApiServer.client = &http.Client{Transport: &http.Transport{ MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024, + DisableCompression: true, }} } else { s3ApiServer.client = &http.Client{ @@ -75,6 +76,7 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { return net.Dial("unix", option.LocalFilerSocket) }, + DisableCompression: true, }, } } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 5400194ee..33f585e61 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -237,7 +237,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa for k, v := range r.Header { if len(v) > 0 && len(v[0]) > 0 { - if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" { + if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" || k == "Content-Encoding" || (k == "Content-Type" && v[0] != "application/octet-stream") { entry.Extended[k] = []byte(v[0]) } if k == "Response-Content-Disposition" { diff --git a/weed/util/http_util.go b/weed/util/http_util.go index 384da6df8..4cd9fd83e 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -23,6 +23,7 @@ func init() { Transport = &http.Transport{ MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024, + DisableCompression: true, } client = &http.Client{ Transport: Transport,