From a4ceb051a7aa195ba9e16995602a615b23f4b746 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 8 Nov 2018 21:41:02 -0800 Subject: [PATCH] use MD5 for ETag to be consistent with Amazon S3 --- .../main/java/com/seaweedfs/s3/PutObject.java | 28 ++++++++++++++++++- weed/s3api/s3api_object_handlers.go | 10 +++++-- weed/server/volume_server_handlers_read.go | 6 +++- weed/storage/crc.go | 13 ++++++++- 4 files changed, 51 insertions(+), 6 deletions(-) diff --git a/other/java/s3copy/copier/src/main/java/com/seaweedfs/s3/PutObject.java b/other/java/s3copy/copier/src/main/java/com/seaweedfs/s3/PutObject.java index 021d01e34..9af184d2b 100644 --- a/other/java/s3copy/copier/src/main/java/com/seaweedfs/s3/PutObject.java +++ b/other/java/s3copy/copier/src/main/java/com/seaweedfs/s3/PutObject.java @@ -12,13 +12,21 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.util.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import java.io.File; +import java.io.IOException; /** * Hello world! */ public class PutObject { + + private static Log log = LogFactory.getLog(PutObject.class); + public static void main(String[] args) { AWSCredentials credentials = new BasicAWSCredentials("ANY-ACCESSKEYID", "ANY-SECRETACCESSKEY"); @@ -39,10 +47,11 @@ public class PutObject { String fileObjKeyName = "fileObject2"; String fileName = args[0]; + String stringContent = "Uploaded String Object v3"; try { // Upload a text string as a new object. - s3Client.putObject(bucketName, stringObjKeyName, "Uploaded String Object v3"); + s3Client.putObject(bucketName, stringObjKeyName, stringContent); // Upload a file as a new object with ContentType and title specified. PutObjectRequest request = new PutObjectRequest(bucketName, fileObjKeyName, new File(fileName)); @@ -51,6 +60,23 @@ public class PutObject { metadata.addUserMetadata("x-amz-meta-title", "someTitle"); request.setMetadata(metadata); s3Client.putObject(request); + + S3Object written = s3Client.getObject(bucketName, stringObjKeyName); + + + try { + String expected = IOUtils.toString(written.getObjectContent()); + + if (!stringContent.equals(expected)){ + System.out.println("Failed to put and get back the content!"); + } + + } catch (IOException e) { + throw new SdkClientException("Error streaming content from S3 during download"); + } finally { + IOUtils.closeQuietly(written, log); + } + } catch (AmazonServiceException e) { // The call was transmitted successfully, but Amazon S3 couldn't process // it, so it returned an error response. diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 93816c7ee..d9cb0c072 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -63,6 +63,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) vars := mux.Vars(r) bucket := vars["bucket"] + object := getObject(vars) if strings.HasSuffix(r.URL.Path, "/") { writeErrorResponse(w, ErrNotImplemented, r.URL) @@ -70,7 +71,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) } destUrl := fmt.Sprintf("http://%s%s/%s%s", - s3a.option.Filer, s3a.option.BucketsPath, bucket, r.RequestURI) + s3a.option.Filer, s3a.option.BucketsPath, bucket, object) s3a.proxyToFiler(w, r, destUrl, passThroghResponse) @@ -80,9 +81,10 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request vars := mux.Vars(r) bucket := vars["bucket"] + object := getObject(vars) destUrl := fmt.Sprintf("http://%s%s/%s%s", - s3a.option.Filer, s3a.option.BucketsPath, bucket, r.RequestURI) + s3a.option.Filer, s3a.option.BucketsPath, bucket, object) s3a.proxyToFiler(w, r, destUrl, passThroghResponse) @@ -92,9 +94,10 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque vars := mux.Vars(r) bucket := vars["bucket"] + object := getObject(vars) destUrl := fmt.Sprintf("http://%s%s/%s%s", - s3a.option.Filer, s3a.option.BucketsPath, bucket, r.RequestURI) + s3a.option.Filer, s3a.option.BucketsPath, bucket, object) s3a.proxyToFiler(w, r, destUrl, func(proxyResonse *http.Response, w http.ResponseWriter) { for k, v := range proxyResonse.Header { @@ -125,6 +128,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des proxyReq.Header.Set("Host", s3a.option.Filer) proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr) + proxyReq.Header.Set("Etag-MD5", "True") for header, values := range r.Header { for _, value := range values { diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index 4da13883e..aa6dff087 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -92,7 +92,11 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNotModified) return } - setEtag(w, n.Etag()) + if r.Header.Get("ETag-MD5") == "True" { + setEtag(w, n.MD5()) + }else{ + setEtag(w, n.Etag()) + } if n.HasPairs() { pairMap := make(map[string]string) diff --git a/weed/storage/crc.go b/weed/storage/crc.go index e31e0f815..df44d1679 100644 --- a/weed/storage/crc.go +++ b/weed/storage/crc.go @@ -2,8 +2,9 @@ package storage import ( "fmt" - "github.com/klauspost/crc32" + "crypto/md5" + "github.com/klauspost/crc32" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -28,3 +29,13 @@ func (n *Needle) Etag() string { util.Uint32toBytes(bits, uint32(n.Checksum)) return fmt.Sprintf("%x", bits) } + +func (n *Needle) MD5() string { + + hash := md5.New() + + hash.Write(n.Data) + + return fmt.Sprintf("%x", hash.Sum(nil)) + +}