Merge pull request #3047 from shichanglin5/master

fix the problem of metadata and tagging loss when files are copied

weed/s3api/http/header.go

@@ -28,11 +28,14 @@ const (
 	AmzStorageClass = "x-amz-storage-class"
 
 	// S3 user-defined metadata
-	AmzUserMetaPrefix = "X-Amz-Meta-"
+	AmzUserMetaPrefix    = "X-Amz-Meta-"
+	AmzUserMetaDirective = "X-Amz-Metadata-Directive"
 
 	// S3 object tagging
-	AmzObjectTagging = "X-Amz-Tagging"
-	AmzTagCount      = "x-amz-tagging-count"
+	AmzObjectTagging          = "X-Amz-Tagging"
+	AmzObjectTaggingPrefix    = "X-Amz-Tagging-"
+	AmzObjectTaggingDirective = "X-Amz-Tagging-Directive"
+	AmzTagCount               = "x-amz-tagging-count"
 )
 
 // Non-Standard S3 HTTP request constants

weed/s3api/s3api_object_copy_handlers.go

@@ -3,9 +3,10 @@ package s3api
 import (
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	headers "github.com/chrislusf/seaweedfs/weed/s3api/http"
 	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
-	weed_server "github.com/chrislusf/seaweedfs/weed/server"
+	"modernc.org/strutil"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -15,6 +16,11 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
+const (
+	DirectiveCopy    = "COPY"
+	DirectiveReplace = "REPLACE"
+)
+
 func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
 
 	dstBucket, dstObject := xhttp.GetBucketAndObject(r)
@@ -30,7 +36,9 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 
 	glog.V(3).Infof("CopyObjectHandler %s %s => %s %s", srcBucket, srcObject, dstBucket, dstObject)
 
-	if (srcBucket == dstBucket && srcObject == dstObject || cpSrcPath == "") && isReplace(r) {
+	replaceMeta, replaceTagging := replaceDirective(r.Header)
+
+	if (srcBucket == dstBucket && srcObject == dstObject || cpSrcPath == "") && (replaceMeta || replaceTagging) {
 		fullPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
 		dir, name := fullPath.DirAndName()
 		entry, err := s3a.getEntry(dir, name)
@@ -38,7 +46,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 			return
 		}
-		entry.Extended = weed_server.SaveAmzMetaData(r, entry.Extended, isReplace(r))
+		entry.Extended = processMetadataBytes(r.Header, entry.Extended, replaceMeta, replaceTagging)
 		err = s3a.touch(dir, name, entry)
 		if err != nil {
 			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
@@ -80,6 +88,11 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 	}
 	defer util.CloseResponse(resp)
 
+	tagErr := processMetadata(r.Header, resp.Header, replaceMeta, replaceTagging, s3a.getTags, dir, name)
+	if tagErr != nil {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
+		return
+	}
 	glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl)
 	etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body)
 
@@ -182,6 +195,107 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 
 }
 
-func isReplace(r *http.Request) bool {
-	return r.Header.Get("X-Amz-Metadata-Directive") == "REPLACE"
+func replaceDirective(reqHeader http.Header) (replaceMeta, replaceTagging bool) {
+	return reqHeader.Get(headers.AmzUserMetaDirective) == DirectiveReplace, reqHeader.Get(headers.AmzObjectTaggingDirective) == DirectiveReplace
+}
+
+func processMetadata(reqHeader, existing http.Header, replaceMeta, replaceTagging bool, getTags func(parentDirectoryPath string, entryName string) (tags map[string]string, err error), dir, name string) (err error) {
+	if sc := reqHeader.Get(xhttp.AmzStorageClass); len(sc) == 0 {
+		if sc := existing[xhttp.AmzStorageClass]; len(sc) > 0 {
+			reqHeader[xhttp.AmzStorageClass] = sc
+		}
+	}
+
+	if !replaceMeta {
+		for header, _ := range reqHeader {
+			if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) {
+				delete(reqHeader, header)
+			}
+		}
+		for k, v := range existing {
+			if strings.HasPrefix(k, xhttp.AmzUserMetaPrefix) {
+				reqHeader[k] = v
+			}
+		}
+	}
+
+	if !replaceTagging {
+		for header, _ := range reqHeader {
+			if strings.HasPrefix(header, xhttp.AmzObjectTagging) {
+				delete(reqHeader, header)
+			}
+		}
+
+		found := false
+		for k, _ := range existing {
+			if strings.HasPrefix(k, xhttp.AmzObjectTaggingPrefix) {
+				found = true
+				break
+			}
+		}
+
+		if found {
+			tags, err := getTags(dir, name)
+			if err != nil {
+				return err
+			}
+
+			var tagArr []string
+			for k, v := range tags {
+				tagArr = append(tagArr, fmt.Sprintf("%s=%s", k, v))
+			}
+			tagStr := strutil.JoinFields(tagArr, "&")
+			reqHeader.Set(xhttp.AmzObjectTagging, tagStr)
+		}
+	}
+	return
+}
+
+func processMetadataBytes(reqHeader http.Header, existing map[string][]byte, replaceMeta, replaceTagging bool) (metadata map[string][]byte) {
+	metadata = make(map[string][]byte)
+
+	if sc := existing[xhttp.AmzStorageClass]; len(sc) > 0 {
+		metadata[xhttp.AmzStorageClass] = sc
+	}
+	if sc := reqHeader.Get(xhttp.AmzStorageClass); len(sc) > 0 {
+		metadata[xhttp.AmzStorageClass] = []byte(sc)
+	}
+
+	if replaceMeta {
+		for header, values := range reqHeader {
+			if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) {
+				for _, value := range values {
+					metadata[header] = []byte(value)
+				}
+			}
+		}
+	} else {
+		for k, v := range existing {
+			if strings.HasPrefix(k, xhttp.AmzUserMetaPrefix) {
+				metadata[k] = v
+			}
+		}
+	}
+
+	if replaceTagging {
+		if tags := reqHeader.Get(xhttp.AmzObjectTagging); tags != "" {
+			for _, v := range strings.Split(tags, "&") {
+				tag := strings.Split(v, "=")
+				if len(tag) == 2 {
+					metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
+				} else if len(tag) == 1 {
+					metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = nil
+				}
+			}
+		}
+	} else {
+		for k, v := range existing {
+			if strings.HasPrefix(k, xhttp.AmzObjectTagging) {
+				metadata[k] = v
+			}
+		}
+		delete(metadata, xhttp.AmzTagCount)
+	}
+
+	return
 }

weed/s3api/s3api_object_copy_handlers_test.go

@@ -0,0 +1,426 @@
+package s3api
+
+import (
+	"fmt"
+	headers "github.com/chrislusf/seaweedfs/weed/s3api/http"
+	"net/http"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+)
+
+type H map[string]string
+
+func (h H) String() string {
+	pairs := make([]string, 0, len(h))
+	for k, v := range h {
+		pairs = append(pairs, fmt.Sprintf("%s : %s", k, v))
+	}
+	sort.Strings(pairs)
+	join := strings.Join(pairs, "\n")
+	return "\n" + join + "\n"
+}
+
+var processMetadataTestCases = []struct {
+	caseId   int
+	request  H
+	existing H
+	getTags  H
+	want     H
+}{
+	{
+		201,
+		H{
+			"User-Agent":         "firefox",
+			"X-Amz-Meta-My-Meta": "request",
+			"X-Amz-Tagging":      "A=B&a=b&type=request",
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-Type": "existing",
+		},
+		H{
+			"A":    "B",
+			"a":    "b",
+			"type": "existing",
+		},
+		H{
+			"User-Agent":         "firefox",
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging":      "A=B&a=b&type=existing",
+		},
+	},
+	{
+		202,
+		H{
+			"User-Agent":                 "firefox",
+			"X-Amz-Meta-My-Meta":         "request",
+			"X-Amz-Tagging":              "A=B&a=b&type=request",
+			headers.AmzUserMetaDirective: DirectiveReplace,
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-Type": "existing",
+		},
+		H{
+			"A":    "B",
+			"a":    "b",
+			"type": "existing",
+		},
+		H{
+			"User-Agent":                 "firefox",
+			"X-Amz-Meta-My-Meta":         "request",
+			"X-Amz-Tagging":              "A=B&a=b&type=existing",
+			headers.AmzUserMetaDirective: DirectiveReplace,
+		},
+	},
+
+	{
+		203,
+		H{
+			"User-Agent":                      "firefox",
+			"X-Amz-Meta-My-Meta":              "request",
+			"X-Amz-Tagging":                   "A=B&a=b&type=request",
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-Type": "existing",
+		},
+		H{
+			"A":    "B",
+			"a":    "b",
+			"type": "existing",
+		},
+		H{
+			"User-Agent":                      "firefox",
+			"X-Amz-Meta-My-Meta":              "existing",
+			"X-Amz-Tagging":                   "A=B&a=b&type=request",
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+	},
+
+	{
+		204,
+		H{
+			"User-Agent":                      "firefox",
+			"X-Amz-Meta-My-Meta":              "request",
+			"X-Amz-Tagging":                   "A=B&a=b&type=request",
+			headers.AmzUserMetaDirective:      DirectiveReplace,
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-Type": "existing",
+		},
+		H{
+			"A":    "B",
+			"a":    "b",
+			"type": "existing",
+		},
+		H{
+			"User-Agent":                      "firefox",
+			"X-Amz-Meta-My-Meta":              "request",
+			"X-Amz-Tagging":                   "A=B&a=b&type=request",
+			headers.AmzUserMetaDirective:      DirectiveReplace,
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+	},
+
+	{
+		205,
+		H{
+			"User-Agent":                      "firefox",
+			"X-Amz-Meta-My-Meta":              "request",
+			"X-Amz-Tagging":                   "A=B&a=b&type=request",
+			headers.AmzUserMetaDirective:      DirectiveReplace,
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+		H{},
+		H{},
+		H{
+			"User-Agent":                      "firefox",
+			"X-Amz-Meta-My-Meta":              "request",
+			"X-Amz-Tagging":                   "A=B&a=b&type=request",
+			headers.AmzUserMetaDirective:      DirectiveReplace,
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+	},
+
+	{
+		206,
+		H{
+			"User-Agent":                      "firefox",
+			headers.AmzUserMetaDirective:      DirectiveReplace,
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-Type": "existing",
+		},
+		H{
+			"A":    "B",
+			"a":    "b",
+			"type": "existing",
+		},
+		H{
+			"User-Agent":                      "firefox",
+			headers.AmzUserMetaDirective:      DirectiveReplace,
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+	},
+
+	{
+		207,
+		H{
+			"User-Agent":                      "firefox",
+			"X-Amz-Meta-My-Meta":              "request",
+			headers.AmzUserMetaDirective:      DirectiveReplace,
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-Type": "existing",
+		},
+		H{
+			"A":    "B",
+			"a":    "b",
+			"type": "existing",
+		},
+		H{
+			"User-Agent":                      "firefox",
+			"X-Amz-Meta-My-Meta":              "request",
+			headers.AmzUserMetaDirective:      DirectiveReplace,
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+	},
+}
+var processMetadataBytesTestCases = []struct {
+	caseId   int
+	request  H
+	existing H
+	want     H
+}{
+	{
+		101,
+		H{
+			"User-Agent":         "firefox",
+			"X-Amz-Meta-My-Meta": "request",
+			"X-Amz-Tagging":      "A=B&a=b&type=request",
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-type": "existing",
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-type": "existing",
+		},
+	},
+
+	{
+		102,
+		H{
+			"User-Agent":                 "firefox",
+			"X-Amz-Meta-My-Meta":         "request",
+			"X-Amz-Tagging":              "A=B&a=b&type=request",
+			headers.AmzUserMetaDirective: DirectiveReplace,
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-type": "existing",
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "request",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-type": "existing",
+		},
+	},
+
+	{
+		103,
+		H{
+			"User-Agent":                      "firefox",
+			"X-Amz-Meta-My-Meta":              "request",
+			"X-Amz-Tagging":                   "A=B&a=b&type=request",
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-type": "existing",
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-type": "request",
+		},
+	},
+
+	{
+		104,
+		H{
+			"User-Agent":                      "firefox",
+			"X-Amz-Meta-My-Meta":              "request",
+			"X-Amz-Tagging":                   "A=B&a=b&type=request",
+			headers.AmzUserMetaDirective:      DirectiveReplace,
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-type": "existing",
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "request",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-type": "request",
+		},
+	},
+
+	{
+		105,
+		H{
+			"User-Agent":                      "firefox",
+			headers.AmzUserMetaDirective:      DirectiveReplace,
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+		H{
+			"X-Amz-Meta-My-Meta": "existing",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-type": "existing",
+		},
+		H{},
+	},
+
+	{
+		107,
+		H{
+			"User-Agent":                      "firefox",
+			"X-Amz-Meta-My-Meta":              "request",
+			"X-Amz-Tagging":                   "A=B&a=b&type=request",
+			headers.AmzUserMetaDirective:      DirectiveReplace,
+			headers.AmzObjectTaggingDirective: DirectiveReplace,
+		},
+		H{},
+		H{
+			"X-Amz-Meta-My-Meta": "request",
+			"X-Amz-Tagging-A":    "B",
+			"X-Amz-Tagging-a":    "b",
+			"X-Amz-Tagging-type": "request",
+		},
+	},
+}
+
+func TestProcessMetadata(t *testing.T) {
+	for _, tc := range processMetadataTestCases {
+		reqHeader := transferHToHeader(tc.request)
+		existing := transferHToHeader(tc.existing)
+		replaceMeta, replaceTagging := replaceDirective(reqHeader)
+
+		err := processMetadata(reqHeader, existing, replaceMeta, replaceTagging, func(_ string, _ string) (tags map[string]string, err error) {
+			return tc.getTags, nil
+		}, "", "")
+		if err != nil {
+			t.Error(err)
+		}
+
+		result := transferHeaderToH(reqHeader)
+		fmtTagging(result, tc.want)
+
+		if !reflect.DeepEqual(result, tc.want) {
+			t.Error(fmt.Errorf("\n### CaseID: %d ###"+
+				"\nRequest:%v"+
+				"\nExisting:%v"+
+				"\nGetTags:%v"+
+				"\nWant:%v"+
+				"\nActual:%v",
+				tc.caseId, tc.request, tc.existing, tc.getTags, tc.want, result))
+		}
+	}
+}
+
+func TestProcessMetadataBytes(t *testing.T) {
+	for _, tc := range processMetadataBytesTestCases {
+		reqHeader := transferHToHeader(tc.request)
+		existing := transferHToBytesArr(tc.existing)
+		replaceMeta, replaceTagging := replaceDirective(reqHeader)
+		extends := processMetadataBytes(reqHeader, existing, replaceMeta, replaceTagging)
+
+		result := transferBytesArrToH(extends)
+		fmtTagging(result, tc.want)
+
+		if !reflect.DeepEqual(result, tc.want) {
+			t.Error(fmt.Errorf("\n### CaseID: %d ###"+
+				"\nRequest:%v"+
+				"\nExisting:%v"+
+				"\nWant:%v"+
+				"\nActual:%v",
+				tc.caseId, tc.request, tc.existing, tc.want, result))
+		}
+	}
+}
+
+func fmtTagging(maps ...map[string]string) {
+	for _, m := range maps {
+		if tagging := m[headers.AmzObjectTagging]; len(tagging) > 0 {
+			split := strings.Split(tagging, "&")
+			sort.Strings(split)
+			m[headers.AmzObjectTagging] = strings.Join(split, "&")
+		}
+	}
+}
+
+func transferHToHeader(data map[string]string) http.Header {
+	header := http.Header{}
+	for k, v := range data {
+		header.Add(k, v)
+	}
+	return header
+}
+
+func transferHToBytesArr(data map[string]string) map[string][]byte {
+	m := make(map[string][]byte, len(data))
+	for k, v := range data {
+		m[k] = []byte(v)
+	}
+	return m
+}
+
+func transferBytesArrToH(data map[string][]byte) H {
+	m := make(map[string]string, len(data))
+	for k, v := range data {
+		m[k] = string(v)
+	}
+	return m
+}
+
+func transferHeaderToH(data map[string][]string) H {
+	m := make(map[string]string, len(data))
+	for k, v := range data {
+		m[k] = v[len(v)-1]
+	}
+	return m
+}