Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

add namespace for s3

pull/865/head
Chris Lu 6 years ago
parent
70ac2f6ea8
commit
cb07d15254
6 changed files with 121 additions and 54 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 13
      weed/s3api/filer_multipart.go
  2. 26
      weed/s3api/filer_multipart_test.go
  3. 1
      weed/s3api/s3api_bucket_handlers_test.go
  4. 46
      weed/s3api/s3api_objects_list_handlers.go
  5. 38
      weed/s3api/s3api_objects_list_handlers_test.go
  6. 51
      weed/s3api/s3api_xsd_generated.go

13
weed/s3api/filer_multipart.go
View File

@ -1,6 +1,7 @@
package s3api
import (
"encoding/xml"
"fmt"
"path/filepath"
"strconv"
@ -16,6 +17,7 @@ import (
)
type InitiateMultipartUploadResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"`
s3.CreateMultipartUploadOutput
}
@ -34,7 +36,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
}
output = &InitiateMultipartUploadResult{
s3.CreateMultipartUploadOutput{
CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
Bucket: input.Bucket,
Key: input.Key,
UploadId: aws.String(uploadIdString),
@ -45,6 +47,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
}
type CompleteMultipartUploadResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult"`
s3.CompleteMultipartUploadOutput
}
@ -95,7 +98,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
}
output = &CompleteMultipartUploadResult{
s3.CompleteMultipartUploadOutput{
CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
Bucket: input.Bucket,
ETag: aws.String("\"" + filer2.ETag(finalParts) + "\""),
Key: input.Key,
@ -128,13 +131,14 @@ func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput
}
type ListMultipartUploadsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult"`
s3.ListMultipartUploadsOutput
}
func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code ErrorCode) {
output = &ListMultipartUploadsResult{
s3.ListMultipartUploadsOutput{
ListMultipartUploadsOutput: s3.ListMultipartUploadsOutput{
Bucket: input.Bucket,
Delimiter: input.Delimiter,
EncodingType: input.EncodingType,
@ -164,12 +168,13 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput
}
type ListPartsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"`
s3.ListPartsOutput
}
func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListPartsResult, code ErrorCode) {
output = &ListPartsResult{
s3.ListPartsOutput{
ListPartsOutput: s3.ListPartsOutput{
Bucket: input.Bucket,
Key: input.Key,
UploadId: input.UploadId,

26
weed/s3api/filer_multipart_test.go
View File

@ -0,0 +1,26 @@
package s3api
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"testing"
)
func TestInitiateMultipartUploadResult(t *testing.T) {
expected := `<?xml version="1.0" encoding="UTF-8"?>
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Bucket>example-bucket</Bucket><Key>example-object</Key><UploadId>VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA</UploadId></InitiateMultipartUploadResult>`
response := &InitiateMultipartUploadResult{
CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
Bucket: aws.String("example-bucket"),
Key: aws.String("example-object"),
UploadId: aws.String("VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"),
},
}
encoded := string(encodeResponse(response))
if encoded != expected {
t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
}
}

1
weed/s3api/s3api_bucket_handlers_test.go
View File

@ -33,7 +33,6 @@ func TestListBucketsHandler(t *testing.T) {
}
encoded := string(encodeResponse(response))
println(encoded)
if encoded != expected {
t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
}

46
weed/s3api/s3api_objects_list_handlers.go
View File

@ -9,8 +9,6 @@ import (
"strconv"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@ -85,7 +83,7 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
writeSuccessResponseXML(w, encodeResponse(response))
}
func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response *s3.ListObjectsOutput, err error) {
func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response ListBucketResult, err error) {
// convert full path prefix into directory name and prefix for entry name
dir, prefix := filepath.Split(originalPrefix)
@ -106,8 +104,8 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
return fmt.Errorf("list buckets: %v", err)
}
var contents []*s3.Object
var commonPrefixes []*s3.CommonPrefix
var contents []ListEntry
var commonPrefixes []PrefixEntry
var counter int
var lastEntryName string
var isTruncated bool
@ -119,32 +117,32 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
}
lastEntryName = entry.Name
if entry.IsDirectory {
commonPrefixes = append(commonPrefixes, &s3.CommonPrefix{
Prefix: aws.String(fmt.Sprintf("%s%s/", dir, entry.Name)),
commonPrefixes = append(commonPrefixes, PrefixEntry{
Prefix: fmt.Sprintf("%s%s/", dir, entry.Name),
})
} else {
contents = append(contents, &s3.Object{
Key: aws.String(fmt.Sprintf("%s%s", dir, entry.Name)),
LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0)),
ETag: aws.String("\"" + filer2.ETag(entry.Chunks) + "\""),
Size: aws.Int64(int64(filer2.TotalSize(entry.Chunks))),
Owner: &s3.Owner{
ID: aws.String("bcaf161ca5fb16fd081034f"),
DisplayName: aws.String("webfile"),
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s%s", dir, entry.Name),
LastModified: time.Unix(entry.Attributes.Mtime, 0),
ETag: "\"" + filer2.ETag(entry.Chunks) + "\"",
Size: int64(filer2.TotalSize(entry.Chunks)),
Owner: CanonicalUser{
ID: "bcaf161ca5fb16fd081034f",
DisplayName: "webfile",
},
StorageClass: aws.String("STANDARD"),
StorageClass: "STANDARD",
})
}
}
response = &s3.ListObjectsOutput{
Name: aws.String(bucket),
Prefix: aws.String(originalPrefix),
Marker: aws.String(marker),
NextMarker: aws.String(lastEntryName),
MaxKeys: aws.Int64(int64(maxKeys)),
Delimiter: aws.String("/"),
IsTruncated: aws.Bool(isTruncated),
response = ListBucketResult{
Name: bucket,
Prefix: originalPrefix,
Marker: marker,
NextMarker: lastEntryName,
MaxKeys: maxKeys,
Delimiter: "/",
IsTruncated: isTruncated,
Contents: contents,
CommonPrefixes: commonPrefixes,
}

38
weed/s3api/s3api_objects_list_handlers_test.go
View File

@ -0,0 +1,38 @@
package s3api
import (
"testing"
"time"
)
func TestListObjectsHandler(t *testing.T) {
// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
expected := `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>test_container</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>1.zip</Key><ETag>&#34;4397da7a7649e8085de9916c240e8166&#34;</ETag><Size>1234567</Size><Owner><ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID></Owner><StorageClass>STANDARD</StorageClass><LastModified>2011-04-09T12:34:49</LastModified></Contents></ListBucketResult>`
response := ListBucketResult{
Name: "test_container",
Prefix: "",
Marker: "",
NextMarker: "",
MaxKeys: 1000,
IsTruncated: false,
Contents: []ListEntry{{
Key: "1.zip",
LastModified: time.Date(2011, 4, 9, 12, 34, 49, 0, time.UTC),
ETag: "\"4397da7a7649e8085de9916c240e8166\"",
Size: 1234567,
Owner: CanonicalUser{
ID: "65a011niqo39cdf8ec533ec3d1ccaafsa932",
},
StorageClass: "STANDARD",
}},
}
encoded := string(encodeResponse(response))
if encoded != expected {
t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
}
}

51
weed/s3api/s3api_xsd_generated.go
View File

@ -25,8 +25,8 @@ type BucketLoggingStatus struct {
}
type CanonicalUser struct {
ID string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ID"`
DisplayName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DisplayName,omitempty"`
ID string `xml:"ID"`
DisplayName string `xml:"DisplayName,omitempty"`
}
type CopyObject struct {
@ -506,15 +506,15 @@ func (t *ListAllMyBuckets) UnmarshalXML(d *xml.Decoder, start xml.StartElement)
}
type ListAllMyBucketsEntry struct {
Name string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"`
CreationDate time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
Name string `xml:"Name"`
CreationDate time.Time `xml:"CreationDate"`
}
func (t *ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type T ListAllMyBucketsEntry
var layout struct {
*T
CreationDate *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
CreationDate *xsdDateTime `xml:"CreationDate"`
}
layout.T = (*T)(t)
layout.CreationDate = (*xsdDateTime)(&layout.T.CreationDate)
@ -524,7 +524,7 @@ func (t *ListAllMyBucketsEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElem
type T ListAllMyBucketsEntry
var overlay struct {
*T
CreationDate *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
CreationDate *xsdDateTime `xml:"CreationDate"`
}
overlay.T = (*T)(t)
overlay.CreationDate = (*xsdDateTime)(&overlay.T.CreationDate)
@ -532,7 +532,7 @@ func (t *ListAllMyBucketsEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElem
}
type ListAllMyBucketsList struct {
Bucket []ListAllMyBucketsEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket,omitempty"`
Bucket []ListAllMyBucketsEntry `xml:"Bucket,omitempty"`
}
type ListAllMyBucketsResponse struct {
@ -577,32 +577,33 @@ type ListBucketResponse struct {
}
type ListBucketResult struct {
Metadata []MetadataEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Metadata,omitempty"`
Name string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"`
Prefix string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Prefix"`
Marker string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Marker"`
NextMarker string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NextMarker,omitempty"`
MaxKeys int `xml:"http://s3.amazonaws.com/doc/2006-03-01/ MaxKeys"`
Delimiter string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Delimiter,omitempty"`
IsTruncated bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IsTruncated"`
Contents []ListEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Contents,omitempty"`
CommonPrefixes []PrefixEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CommonPrefixes,omitempty"`
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
Metadata []MetadataEntry `xml:"Metadata,omitempty"`
Name string `xml:"Name"`
Prefix string `xml:"Prefix"`
Marker string `xml:"Marker"`
NextMarker string `xml:"NextMarker,omitempty"`
MaxKeys int `xml:"MaxKeys"`
Delimiter string `xml:"Delimiter,omitempty"`
IsTruncated bool `xml:"IsTruncated"`
Contents []ListEntry `xml:"Contents,omitempty"`
CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
}
type ListEntry struct {
Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"`
LastModified time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
ETag string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ETag"`
Size int64 `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Size"`
Owner CanonicalUser `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Owner,omitempty"`
StorageClass StorageClass `xml:"http://s3.amazonaws.com/doc/2006-03-01/ StorageClass"`
Key string `xml:"Key"`
LastModified time.Time `xml:"LastModified"`
ETag string `xml:"ETag"`
Size int64 `xml:"Size"`
Owner CanonicalUser `xml:"Owner,omitempty"`
StorageClass StorageClass `xml:"StorageClass"`
}
func (t *ListEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type T ListEntry
var layout struct {
*T
LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
LastModified *xsdDateTime `xml:"LastModified"`
}
layout.T = (*T)(t)
layout.LastModified = (*xsdDateTime)(&layout.T.LastModified)
@ -612,7 +613,7 @@ func (t *ListEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
type T ListEntry
var overlay struct {
*T
LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
LastModified *xsdDateTime `xml:"LastModified"`
}
overlay.T = (*T)(t)
overlay.LastModified = (*xsdDateTime)(&overlay.T.LastModified)

Write Preview
Loading…
Cancel
Save