add namespace for s3

weed/s3api/filer_multipart.go

@@ -1,6 +1,7 @@
 package s3api
 
 import (
+	"encoding/xml"
 	"fmt"
 	"path/filepath"
 	"strconv"
@@ -16,6 +17,7 @@ import (
 )
 
 type InitiateMultipartUploadResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"`
 	s3.CreateMultipartUploadOutput
 }
 
@@ -34,7 +36,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
 	}
 
 	output = &InitiateMultipartUploadResult{
-		s3.CreateMultipartUploadOutput{
+		CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
 			Bucket:   input.Bucket,
 			Key:      input.Key,
 			UploadId: aws.String(uploadIdString),
@@ -45,6 +47,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
 }
 
 type CompleteMultipartUploadResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult"`
 	s3.CompleteMultipartUploadOutput
 }
 
@@ -95,7 +98,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
 	}
 
 	output = &CompleteMultipartUploadResult{
-		s3.CompleteMultipartUploadOutput{
+		CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
 			Bucket: input.Bucket,
 			ETag:   aws.String("\"" + filer2.ETag(finalParts) + "\""),
 			Key:    input.Key,
@@ -128,13 +131,14 @@ func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput
 }
 
 type ListMultipartUploadsResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult"`
 	s3.ListMultipartUploadsOutput
 }
 
 func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code ErrorCode) {
 
 	output = &ListMultipartUploadsResult{
-		s3.ListMultipartUploadsOutput{
+		ListMultipartUploadsOutput: s3.ListMultipartUploadsOutput{
 			Bucket:       input.Bucket,
 			Delimiter:    input.Delimiter,
 			EncodingType: input.EncodingType,
@@ -164,12 +168,13 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput
 }
 
 type ListPartsResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"`
 	s3.ListPartsOutput
 }
 
 func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListPartsResult, code ErrorCode) {
 	output = &ListPartsResult{
-		s3.ListPartsOutput{
+		ListPartsOutput: s3.ListPartsOutput{
 			Bucket:           input.Bucket,
 			Key:              input.Key,
 			UploadId:         input.UploadId,

weed/s3api/filer_multipart_test.go

@@ -0,0 +1,26 @@
+package s3api
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"testing"
+)
+
+func TestInitiateMultipartUploadResult(t *testing.T) {
+
+	expected := `<?xml version="1.0" encoding="UTF-8"?>
+<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Bucket>example-bucket</Bucket><Key>example-object</Key><UploadId>VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA</UploadId></InitiateMultipartUploadResult>`
+	response := &InitiateMultipartUploadResult{
+		CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
+			Bucket:   aws.String("example-bucket"),
+			Key:      aws.String("example-object"),
+			UploadId: aws.String("VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"),
+		},
+	}
+
+	encoded := string(encodeResponse(response))
+	if encoded != expected {
+		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
+	}
+
+}

weed/s3api/s3api_bucket_handlers_test.go

@@ -33,7 +33,6 @@ func TestListBucketsHandler(t *testing.T) {
 	}
 
 	encoded := string(encodeResponse(response))
-	println(encoded)
 	if encoded != expected {
 		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
 	}

weed/s3api/s3api_objects_list_handlers.go

@@ -9,8 +9,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@@ -85,7 +83,7 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 	writeSuccessResponseXML(w, encodeResponse(response))
 }
 
-func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response *s3.ListObjectsOutput, err error) {
+func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response ListBucketResult, err error) {
 
 	// convert full path prefix into directory name and prefix for entry name
 	dir, prefix := filepath.Split(originalPrefix)
@@ -106,8 +104,8 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
 			return fmt.Errorf("list buckets: %v", err)
 		}
 
-		var contents []*s3.Object
+		var contents []ListEntry
-		var commonPrefixes []*s3.CommonPrefix
+		var commonPrefixes []PrefixEntry
 		var counter int
 		var lastEntryName string
 		var isTruncated bool
@@ -119,32 +117,32 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
 			}
 			lastEntryName = entry.Name
 			if entry.IsDirectory {
-				commonPrefixes = append(commonPrefixes, &s3.CommonPrefix{
+				commonPrefixes = append(commonPrefixes, PrefixEntry{
-					Prefix: aws.String(fmt.Sprintf("%s%s/", dir, entry.Name)),
+					Prefix: fmt.Sprintf("%s%s/", dir, entry.Name),
 				})
 			} else {
-				contents = append(contents, &s3.Object{
+				contents = append(contents, ListEntry{
-					Key:          aws.String(fmt.Sprintf("%s%s", dir, entry.Name)),
+					Key:          fmt.Sprintf("%s%s", dir, entry.Name),
-					LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0)),
+					LastModified: time.Unix(entry.Attributes.Mtime, 0),
-					ETag:         aws.String("\"" + filer2.ETag(entry.Chunks) + "\""),
+					ETag:         "\"" + filer2.ETag(entry.Chunks) + "\"",
-					Size:         aws.Int64(int64(filer2.TotalSize(entry.Chunks))),
+					Size:         int64(filer2.TotalSize(entry.Chunks)),
-					Owner: &s3.Owner{
+					Owner: CanonicalUser{
-						ID:          aws.String("bcaf161ca5fb16fd081034f"),
+						ID:          "bcaf161ca5fb16fd081034f",
-						DisplayName: aws.String("webfile"),
+						DisplayName: "webfile",
 					},
-					StorageClass: aws.String("STANDARD"),
+					StorageClass: "STANDARD",
 				})
 			}
 		}
 
-		response = &s3.ListObjectsOutput{
+		response = ListBucketResult{
-			Name:           aws.String(bucket),
+			Name:           bucket,
-			Prefix:         aws.String(originalPrefix),
+			Prefix:         originalPrefix,
-			Marker:         aws.String(marker),
+			Marker:         marker,
-			NextMarker:     aws.String(lastEntryName),
+			NextMarker:     lastEntryName,
-			MaxKeys:        aws.Int64(int64(maxKeys)),
+			MaxKeys:        maxKeys,
-			Delimiter:      aws.String("/"),
+			Delimiter:      "/",
-			IsTruncated:    aws.Bool(isTruncated),
+			IsTruncated:    isTruncated,
 			Contents:       contents,
 			CommonPrefixes: commonPrefixes,
 		}

weed/s3api/s3api_objects_list_handlers_test.go

@@ -0,0 +1,38 @@
+package s3api
+
+import (
+	"testing"
+	"time"
+)
+
+func TestListObjectsHandler(t *testing.T) {
+
+	// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
+
+	expected := `<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>test_container</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>1.zip</Key><ETag>&#34;4397da7a7649e8085de9916c240e8166&#34;</ETag><Size>1234567</Size><Owner><ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID></Owner><StorageClass>STANDARD</StorageClass><LastModified>2011-04-09T12:34:49</LastModified></Contents></ListBucketResult>`
+
+	response := ListBucketResult{
+		Name:        "test_container",
+		Prefix:      "",
+		Marker:      "",
+		NextMarker:  "",
+		MaxKeys:     1000,
+		IsTruncated: false,
+		Contents: []ListEntry{{
+			Key:          "1.zip",
+			LastModified: time.Date(2011, 4, 9, 12, 34, 49, 0, time.UTC),
+			ETag:         "\"4397da7a7649e8085de9916c240e8166\"",
+			Size:         1234567,
+			Owner: CanonicalUser{
+				ID: "65a011niqo39cdf8ec533ec3d1ccaafsa932",
+			},
+			StorageClass: "STANDARD",
+		}},
+	}
+
+	encoded := string(encodeResponse(response))
+	if encoded != expected {
+		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
+	}
+}

weed/s3api/s3api_xsd_generated.go

@@ -25,8 +25,8 @@ type BucketLoggingStatus struct {
 }
 
 type CanonicalUser struct {
-	ID          string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ID"`
+	ID          string `xml:"ID"`
-	DisplayName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DisplayName,omitempty"`
+	DisplayName string `xml:"DisplayName,omitempty"`
 }
 
 type CopyObject struct {
@@ -506,15 +506,15 @@ func (t *ListAllMyBuckets) UnmarshalXML(d *xml.Decoder, start xml.StartElement)
 }
 
 type ListAllMyBucketsEntry struct {
-	Name         string    `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"`
+	Name         string    `xml:"Name"`
-	CreationDate time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
+	CreationDate time.Time `xml:"CreationDate"`
 }
 
 func (t *ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type T ListAllMyBucketsEntry
 	var layout struct {
 		*T
-		CreationDate *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
+		CreationDate *xsdDateTime `xml:"CreationDate"`
 	}
 	layout.T = (*T)(t)
 	layout.CreationDate = (*xsdDateTime)(&layout.T.CreationDate)
@@ -524,7 +524,7 @@ func (t *ListAllMyBucketsEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElem
 	type T ListAllMyBucketsEntry
 	var overlay struct {
 		*T
-		CreationDate *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
+		CreationDate *xsdDateTime `xml:"CreationDate"`
 	}
 	overlay.T = (*T)(t)
 	overlay.CreationDate = (*xsdDateTime)(&overlay.T.CreationDate)
@@ -532,7 +532,7 @@ func (t *ListAllMyBucketsEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElem
 }
 
 type ListAllMyBucketsList struct {
-	Bucket []ListAllMyBucketsEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket,omitempty"`
+	Bucket []ListAllMyBucketsEntry `xml:"Bucket,omitempty"`
 }
 
 type ListAllMyBucketsResponse struct {
@@ -577,32 +577,33 @@ type ListBucketResponse struct {
 }
 
 type ListBucketResult struct {
-	Metadata       []MetadataEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Metadata,omitempty"`
+	XMLName        xml.Name        `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
-	Name           string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"`
+	Metadata       []MetadataEntry `xml:"Metadata,omitempty"`
-	Prefix         string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Prefix"`
+	Name           string          `xml:"Name"`
-	Marker         string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Marker"`
+	Prefix         string          `xml:"Prefix"`
-	NextMarker     string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NextMarker,omitempty"`
+	Marker         string          `xml:"Marker"`
-	MaxKeys        int             `xml:"http://s3.amazonaws.com/doc/2006-03-01/ MaxKeys"`
+	NextMarker     string          `xml:"NextMarker,omitempty"`
-	Delimiter      string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Delimiter,omitempty"`
+	MaxKeys        int             `xml:"MaxKeys"`
-	IsTruncated    bool            `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IsTruncated"`
+	Delimiter      string          `xml:"Delimiter,omitempty"`
-	Contents       []ListEntry     `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Contents,omitempty"`
+	IsTruncated    bool            `xml:"IsTruncated"`
-	CommonPrefixes []PrefixEntry   `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CommonPrefixes,omitempty"`
+	Contents       []ListEntry     `xml:"Contents,omitempty"`
+	CommonPrefixes []PrefixEntry   `xml:"CommonPrefixes,omitempty"`
 }
 
 type ListEntry struct {
-	Key          string        `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"`
+	Key          string        `xml:"Key"`
-	LastModified time.Time     `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
+	LastModified time.Time     `xml:"LastModified"`
-	ETag         string        `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ETag"`
+	ETag         string        `xml:"ETag"`
-	Size         int64         `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Size"`
+	Size         int64         `xml:"Size"`
-	Owner        CanonicalUser `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Owner,omitempty"`
+	Owner        CanonicalUser `xml:"Owner,omitempty"`
-	StorageClass StorageClass  `xml:"http://s3.amazonaws.com/doc/2006-03-01/ StorageClass"`
+	StorageClass StorageClass  `xml:"StorageClass"`
 }
 
 func (t *ListEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type T ListEntry
 	var layout struct {
 		*T
-		LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
+		LastModified *xsdDateTime `xml:"LastModified"`
 	}
 	layout.T = (*T)(t)
 	layout.LastModified = (*xsdDateTime)(&layout.T.LastModified)
@@ -612,7 +613,7 @@ func (t *ListEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	type T ListEntry
 	var overlay struct {
 		*T
-		LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
+		LastModified *xsdDateTime `xml:"LastModified"`
 	}
 	overlay.T = (*T)(t)
 	overlay.LastModified = (*xsdDateTime)(&overlay.T.LastModified)