From b917be79559ec171f351ac842f9d4c20bb335e41 Mon Sep 17 00:00:00 2001 From: "ruitao.liu" Date: Thu, 29 Oct 2020 14:57:19 +0800 Subject: [PATCH 1/2] S3 bucket list, response with uploaded storageclass. --- weed/s3api/s3api_objects_list_handlers.go | 11 +- weed/server/filer_server_handlers_read.go | 4 +- .../filer_server_handlers_write_autochunk.go | 15 +- weed/util/http_header.go | 141 ++++++++++++++++++ 4 files changed, 157 insertions(+), 14 deletions(-) create mode 100644 weed/util/http_header.go diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 23406d6df..d291b588c 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -4,7 +4,6 @@ import ( "context" "encoding/xml" "fmt" - "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "io" "net/http" "net/url" @@ -13,6 +12,10 @@ import ( "strings" "time" + "github.com/chrislusf/seaweedfs/weed/util" + + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) @@ -137,6 +140,10 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m }) } } else { + storageClass := "STANDARD" + if v, ok := entry.Extended[util.AmzStorageClass]; ok { + storageClass = string(v) + } contents = append(contents, ListEntry{ Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), @@ -146,7 +153,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m ID: fmt.Sprintf("%x", entry.Attributes.Uid), DisplayName: entry.Attributes.UserName, }, - StorageClass: "STANDARD", + StorageClass: StorageClass(storageClass), }) } }) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 16edb6167..42d540df8 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -97,12 +97,12 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, if r.Method == "GET" { tagCount := 0 for k, _ := range entry.Extended { - if strings.HasPrefix(k, "X-Amz-Tagging-") { + if strings.HasPrefix(k, util.AmzObjectTagging+"-") { tagCount++ } } if tagCount > 0 { - w.Header().Set("x-amz-tag-count", strconv.Itoa(tagCount)) + w.Header().Set(util.AmzTagCount, strconv.Itoa(tagCount)) } } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index d996c6208..5396cc5ca 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -312,31 +312,26 @@ func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http } func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) { - var ( - storageClass = "X-Amz-Storage-Class" - objectTagging = "X-Amz-Tagging" - userMetaPrefix = "X-Amz-Meta-" - ) if entry.Extended == nil { entry.Extended = make(map[string][]byte) } - if sc := r.Header.Get(storageClass); sc != "" { - entry.Extended[storageClass] = []byte(sc) + if sc := r.Header.Get(util.AmzStorageClass); sc != "" { + entry.Extended[util.AmzStorageClass] = []byte(sc) } - if tags := r.Header.Get(objectTagging); tags != "" { + if tags := r.Header.Get(util.AmzObjectTagging); tags != "" { for _, v := range strings.Split(tags, "&") { tag := strings.Split(v, "=") if len(tag) == 2 { - entry.Extended[objectTagging+"-"+tag[0]] = []byte(tag[1]) + entry.Extended[util.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1]) } } } for header, values := range r.Header { - if strings.HasPrefix(header, userMetaPrefix) { + if strings.HasPrefix(header, util.AmzUserMetaPrefix) { for _, value := range values { entry.Extended[header] = []byte(value) } diff --git a/weed/util/http_header.go b/weed/util/http_header.go new file mode 100644 index 000000000..f7633da83 --- /dev/null +++ b/weed/util/http_header.go @@ -0,0 +1,141 @@ +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +// Standard S3 HTTP response constants +const ( + LastModified = "Last-Modified" + Date = "Date" + ETag = "ETag" + ContentType = "Content-Type" + ContentMD5 = "Content-Md5" + ContentEncoding = "Content-Encoding" + Expires = "Expires" + ContentLength = "Content-Length" + ContentLanguage = "Content-Language" + ContentRange = "Content-Range" + Connection = "Connection" + AcceptRanges = "Accept-Ranges" + AmzBucketRegion = "X-Amz-Bucket-Region" + ServerInfo = "Server" + RetryAfter = "Retry-After" + Location = "Location" + CacheControl = "Cache-Control" + ContentDisposition = "Content-Disposition" + Authorization = "Authorization" + Action = "Action" + Range = "Range" +) + +// Non standard S3 HTTP response constants +const ( + XCache = "X-Cache" + XCacheLookup = "X-Cache-Lookup" +) + +// Standard S3 HTTP request constants +const ( + IfModifiedSince = "If-Modified-Since" + IfUnmodifiedSince = "If-Unmodified-Since" + IfMatch = "If-Match" + IfNoneMatch = "If-None-Match" + + // S3 storage class + AmzStorageClass = "x-amz-storage-class" + + // S3 user-defined metadata + AmzUserMetaPrefix = "X-Amz-Meta-" + + // S3 object version ID + AmzVersionID = "x-amz-version-id" + AmzDeleteMarker = "x-amz-delete-marker" + + // S3 object tagging + AmzObjectTagging = "X-Amz-Tagging" + AmzTagCount = "x-amz-tagging-count" + AmzTagDirective = "X-Amz-Tagging-Directive" + + // S3 extensions + AmzCopySourceIfModifiedSince = "x-amz-copy-source-if-modified-since" + AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since" + + AmzCopySourceIfNoneMatch = "x-amz-copy-source-if-none-match" + AmzCopySourceIfMatch = "x-amz-copy-source-if-match" + + AmzCopySource = "X-Amz-Copy-Source" + AmzCopySourceVersionID = "X-Amz-Copy-Source-Version-Id" + AmzCopySourceRange = "X-Amz-Copy-Source-Range" + AmzMetadataDirective = "X-Amz-Metadata-Directive" + AmzObjectLockMode = "X-Amz-Object-Lock-Mode" + AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date" + AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold" + AmzObjectLockBypassGovernance = "X-Amz-Bypass-Governance-Retention" + AmzBucketReplicationStatus = "X-Amz-Replication-Status" + // Multipart parts count + AmzMpPartsCount = "x-amz-mp-parts-count" + + // Object date/time of expiration + AmzExpiration = "x-amz-expiration" + + // Dummy putBucketACL + AmzACL = "x-amz-acl" + + // Signature V4 related contants. + AmzContentSha256 = "X-Amz-Content-Sha256" + AmzDate = "X-Amz-Date" + AmzAlgorithm = "X-Amz-Algorithm" + AmzExpires = "X-Amz-Expires" + AmzSignedHeaders = "X-Amz-SignedHeaders" + AmzSignature = "X-Amz-Signature" + AmzCredential = "X-Amz-Credential" + AmzSecurityToken = "X-Amz-Security-Token" + AmzDecodedContentLength = "X-Amz-Decoded-Content-Length" + + AmzMetaUnencryptedContentLength = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length" + AmzMetaUnencryptedContentMD5 = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5" + + // Signature v2 related constants + AmzSignatureV2 = "Signature" + AmzAccessKeyID = "AWSAccessKeyId" + + // Response request id. + AmzRequestID = "x-amz-request-id" + + // Deployment id. + MinioDeploymentID = "x-minio-deployment-id" + + // Server-Status + MinIOServerStatus = "x-minio-server-status" + + // Delete special flag to force delete a bucket + MinIOForceDelete = "x-minio-force-delete" + + // Header indicates if the mtime should be preserved by client + MinIOSourceMTime = "x-minio-source-mtime" + + // Header indicates if the etag should be preserved by client + MinIOSourceETag = "x-minio-source-etag" +) + +// Common http query params S3 API +const ( + VersionID = "versionId" + + PartNumber = "partNumber" + + UploadID = "uploadId" +) From 22a9ea05129d8c4776187d667f9e31b03ddbb2e0 Mon Sep 17 00:00:00 2001 From: "ruitao.liu" Date: Thu, 29 Oct 2020 16:05:40 +0800 Subject: [PATCH 2/2] adjust s3 header file. --- weed/s3api/filer_util_tags.go | 3 +- weed/s3api/http/header.go | 30 ++++ weed/s3api/s3api_objects_list_handlers.go | 8 +- weed/server/filer_server_handlers_read.go | 5 +- .../filer_server_handlers_write_autochunk.go | 11 +- weed/util/http_header.go | 141 ------------------ 6 files changed, 44 insertions(+), 154 deletions(-) create mode 100644 weed/s3api/http/header.go delete mode 100644 weed/util/http_header.go diff --git a/weed/s3api/filer_util_tags.go b/weed/s3api/filer_util_tags.go index 3d4da7825..75d3b37d0 100644 --- a/weed/s3api/filer_util_tags.go +++ b/weed/s3api/filer_util_tags.go @@ -4,10 +4,11 @@ import ( "strings" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" ) const ( - S3TAG_PREFIX = "s3-" + S3TAG_PREFIX = xhttp.AmzObjectTagging + "-" ) func (s3a *S3ApiServer) getTags(parentDirectoryPath string, entryName string) (tags map[string]string, err error) { diff --git a/weed/s3api/http/header.go b/weed/s3api/http/header.go new file mode 100644 index 000000000..2802b560f --- /dev/null +++ b/weed/s3api/http/header.go @@ -0,0 +1,30 @@ +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package http + +// Standard S3 HTTP request constants +const ( + // S3 storage class + AmzStorageClass = "x-amz-storage-class" + + // S3 user-defined metadata + AmzUserMetaPrefix = "X-Amz-Meta-" + + // S3 object tagging + AmzObjectTagging = "X-Amz-Tagging" + AmzTagCount = "x-amz-tagging-count" +) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index d291b588c..5d63f1039 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -12,12 +12,10 @@ import ( "strings" "time" - "github.com/chrislusf/seaweedfs/weed/util" - - "github.com/chrislusf/seaweedfs/weed/s3api/s3err" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" ) type ListBucketResultV2 struct { @@ -141,7 +139,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m } } else { storageClass := "STANDARD" - if v, ok := entry.Extended[util.AmzStorageClass]; ok { + if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok { storageClass = string(v) } contents = append(contents, ListEntry{ diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 42d540df8..7b08e1686 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -15,6 +15,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/images" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -97,12 +98,12 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, if r.Method == "GET" { tagCount := 0 for k, _ := range entry.Extended { - if strings.HasPrefix(k, util.AmzObjectTagging+"-") { + if strings.HasPrefix(k, xhttp.AmzObjectTagging+"-") { tagCount++ } } if tagCount > 0 { - w.Header().Set(util.AmzTagCount, strconv.Itoa(tagCount)) + w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount)) } } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 5396cc5ca..d308dafa2 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -18,6 +18,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" @@ -317,21 +318,21 @@ func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) { entry.Extended = make(map[string][]byte) } - if sc := r.Header.Get(util.AmzStorageClass); sc != "" { - entry.Extended[util.AmzStorageClass] = []byte(sc) + if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" { + entry.Extended[xhttp.AmzStorageClass] = []byte(sc) } - if tags := r.Header.Get(util.AmzObjectTagging); tags != "" { + if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" { for _, v := range strings.Split(tags, "&") { tag := strings.Split(v, "=") if len(tag) == 2 { - entry.Extended[util.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1]) + entry.Extended[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1]) } } } for header, values := range r.Header { - if strings.HasPrefix(header, util.AmzUserMetaPrefix) { + if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) { for _, value := range values { entry.Extended[header] = []byte(value) } diff --git a/weed/util/http_header.go b/weed/util/http_header.go deleted file mode 100644 index f7633da83..000000000 --- a/weed/util/http_header.go +++ /dev/null @@ -1,141 +0,0 @@ -/* - * MinIO Cloud Storage, (C) 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package util - -// Standard S3 HTTP response constants -const ( - LastModified = "Last-Modified" - Date = "Date" - ETag = "ETag" - ContentType = "Content-Type" - ContentMD5 = "Content-Md5" - ContentEncoding = "Content-Encoding" - Expires = "Expires" - ContentLength = "Content-Length" - ContentLanguage = "Content-Language" - ContentRange = "Content-Range" - Connection = "Connection" - AcceptRanges = "Accept-Ranges" - AmzBucketRegion = "X-Amz-Bucket-Region" - ServerInfo = "Server" - RetryAfter = "Retry-After" - Location = "Location" - CacheControl = "Cache-Control" - ContentDisposition = "Content-Disposition" - Authorization = "Authorization" - Action = "Action" - Range = "Range" -) - -// Non standard S3 HTTP response constants -const ( - XCache = "X-Cache" - XCacheLookup = "X-Cache-Lookup" -) - -// Standard S3 HTTP request constants -const ( - IfModifiedSince = "If-Modified-Since" - IfUnmodifiedSince = "If-Unmodified-Since" - IfMatch = "If-Match" - IfNoneMatch = "If-None-Match" - - // S3 storage class - AmzStorageClass = "x-amz-storage-class" - - // S3 user-defined metadata - AmzUserMetaPrefix = "X-Amz-Meta-" - - // S3 object version ID - AmzVersionID = "x-amz-version-id" - AmzDeleteMarker = "x-amz-delete-marker" - - // S3 object tagging - AmzObjectTagging = "X-Amz-Tagging" - AmzTagCount = "x-amz-tagging-count" - AmzTagDirective = "X-Amz-Tagging-Directive" - - // S3 extensions - AmzCopySourceIfModifiedSince = "x-amz-copy-source-if-modified-since" - AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since" - - AmzCopySourceIfNoneMatch = "x-amz-copy-source-if-none-match" - AmzCopySourceIfMatch = "x-amz-copy-source-if-match" - - AmzCopySource = "X-Amz-Copy-Source" - AmzCopySourceVersionID = "X-Amz-Copy-Source-Version-Id" - AmzCopySourceRange = "X-Amz-Copy-Source-Range" - AmzMetadataDirective = "X-Amz-Metadata-Directive" - AmzObjectLockMode = "X-Amz-Object-Lock-Mode" - AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date" - AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold" - AmzObjectLockBypassGovernance = "X-Amz-Bypass-Governance-Retention" - AmzBucketReplicationStatus = "X-Amz-Replication-Status" - // Multipart parts count - AmzMpPartsCount = "x-amz-mp-parts-count" - - // Object date/time of expiration - AmzExpiration = "x-amz-expiration" - - // Dummy putBucketACL - AmzACL = "x-amz-acl" - - // Signature V4 related contants. - AmzContentSha256 = "X-Amz-Content-Sha256" - AmzDate = "X-Amz-Date" - AmzAlgorithm = "X-Amz-Algorithm" - AmzExpires = "X-Amz-Expires" - AmzSignedHeaders = "X-Amz-SignedHeaders" - AmzSignature = "X-Amz-Signature" - AmzCredential = "X-Amz-Credential" - AmzSecurityToken = "X-Amz-Security-Token" - AmzDecodedContentLength = "X-Amz-Decoded-Content-Length" - - AmzMetaUnencryptedContentLength = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length" - AmzMetaUnencryptedContentMD5 = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5" - - // Signature v2 related constants - AmzSignatureV2 = "Signature" - AmzAccessKeyID = "AWSAccessKeyId" - - // Response request id. - AmzRequestID = "x-amz-request-id" - - // Deployment id. - MinioDeploymentID = "x-minio-deployment-id" - - // Server-Status - MinIOServerStatus = "x-minio-server-status" - - // Delete special flag to force delete a bucket - MinIOForceDelete = "x-minio-force-delete" - - // Header indicates if the mtime should be preserved by client - MinIOSourceMTime = "x-minio-source-mtime" - - // Header indicates if the etag should be preserved by client - MinIOSourceETag = "x-minio-source-etag" -) - -// Common http query params S3 API -const ( - VersionID = "versionId" - - PartNumber = "partNumber" - - UploadID = "uploadId" -)