From be946c9e54db3297a68715d1789598d004b71e60 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 22 Dec 2018 13:11:07 -0800
Subject: [PATCH] filer store original data size instead of data size after
 gzip

---
 weed/server/common.go                       |  4 ++--
 weed/server/volume_server_handlers_write.go |  4 ++--
 weed/storage/needle.go                      | 16 ++++++++--------
 weed/storage/needle_parse_multipart.go      |  4 +++-
 4 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/weed/server/common.go b/weed/server/common.go
index 032b975ef..d88abfdc8 100644
--- a/weed/server/common.go
+++ b/weed/server/common.go
@@ -91,7 +91,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st
 	}
 
 	debug("parsing upload file...")
-	fname, data, mimeType, pairMap, isGzipped, lastModified, _, _, pe := storage.ParseUpload(r)
+	fname, data, mimeType, pairMap, isGzipped, originalDataSize, lastModified, _, _, pe := storage.ParseUpload(r)
 	if pe != nil {
 		writeJsonError(w, r, http.StatusBadRequest, pe)
 		return
@@ -134,7 +134,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st
 	m["fileName"] = fname
 	m["fid"] = assignResult.Fid
 	m["fileUrl"] = assignResult.PublicUrl + "/" + assignResult.Fid
-	m["size"] = uploadResult.Size
+	m["size"] = originalDataSize
 	m["eTag"] = uploadResult.ETag
 	writeJsonQuiet(w, r, http.StatusCreated, m)
 	return
diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go
index f1a7dd930..1fc5b1319 100644
--- a/weed/server/volume_server_handlers_write.go
+++ b/weed/server/volume_server_handlers_write.go
@@ -27,7 +27,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 		writeJsonError(w, r, http.StatusBadRequest, ve)
 		return
 	}
-	needle, ne := storage.NewNeedle(r, vs.FixJpgOrientation)
+	needle, originalSize, ne := storage.CreateNeedleFromRequest(r, vs.FixJpgOrientation)
 	if ne != nil {
 		writeJsonError(w, r, http.StatusBadRequest, ne)
 		return
@@ -44,7 +44,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 	if needle.HasName() {
 		ret.Name = string(needle.Name)
 	}
-	ret.Size = needle.DataSize
+	ret.Size = uint32(originalSize)
 	setEtag(w, needle.Etag())
 	writeJsonQuiet(w, r, httpStatus, ret)
 }
diff --git a/weed/storage/needle.go b/weed/storage/needle.go
index 4d61c36d0..9d47cc5f9 100644
--- a/weed/storage/needle.go
+++ b/weed/storage/needle.go
@@ -29,12 +29,12 @@ type Needle struct {
 
 	DataSize     uint32 `comment:"Data size"` //version2
 	Data         []byte `comment:"The actual file data"`
-	Flags        byte   `comment:"boolean flags"` //version2
-	NameSize     uint8  //version2
+	Flags        byte   `comment:"boolean flags"`          //version2
+	NameSize     uint8                                     //version2
 	Name         []byte `comment:"maximum 256 characters"` //version2
-	MimeSize     uint8  //version2
+	MimeSize     uint8                                     //version2
 	Mime         []byte `comment:"maximum 256 characters"` //version2
-	PairsSize    uint16 //version2
+	PairsSize    uint16                                    //version2
 	Pairs        []byte `comment:"additional name value pairs, json format, maximum 64kB"`
 	LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
 	Ttl          *TTL
@@ -50,7 +50,7 @@ func (n *Needle) String() (str string) {
 }
 
 func ParseUpload(r *http.Request) (
-	fileName string, data []byte, mimeType string, pairMap map[string]string, isGzipped bool,
+	fileName string, data []byte, mimeType string, pairMap map[string]string, isGzipped bool, originalDataSize int,
 	modifiedTime uint64, ttl *TTL, isChunkedFile bool, e error) {
 	pairMap = make(map[string]string)
 	for k, v := range r.Header {
@@ -60,7 +60,7 @@ func ParseUpload(r *http.Request) (
 	}
 
 	if r.Method == "POST" {
-		fileName, data, mimeType, isGzipped, isChunkedFile, e = parseMultipart(r)
+		fileName, data, mimeType, isGzipped, originalDataSize, isChunkedFile, e = parseMultipart(r)
 	} else {
 		isGzipped = false
 		mimeType = r.Header.Get("Content-Type")
@@ -76,11 +76,11 @@ func ParseUpload(r *http.Request) (
 
 	return
 }
-func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
+func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool) (n *Needle, originalSize int, e error) {
 	var pairMap map[string]string
 	fname, mimeType, isGzipped, isChunkedFile := "", "", false, false
 	n = new(Needle)
-	fname, n.Data, mimeType, pairMap, isGzipped, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r)
+	fname, n.Data, mimeType, pairMap, isGzipped, originalSize, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r)
 	if e != nil {
 		return
 	}
diff --git a/weed/storage/needle_parse_multipart.go b/weed/storage/needle_parse_multipart.go
index d42bc4629..8d5b96a27 100644
--- a/weed/storage/needle_parse_multipart.go
+++ b/weed/storage/needle_parse_multipart.go
@@ -12,7 +12,7 @@ import (
 )
 
 func parseMultipart(r *http.Request) (
-	fileName string, data []byte, mimeType string, isGzipped, isChunkedFile bool, e error) {
+	fileName string, data []byte, mimeType string, isGzipped bool, originalDataSize int, isChunkedFile bool, e error) {
 	form, fe := r.MultipartReader()
 	if fe != nil {
 		glog.V(0).Infoln("MultipartReader [ERROR]", fe)
@@ -64,6 +64,8 @@ func parseMultipart(r *http.Request) (
 		}
 	}
 
+	originalDataSize = len(data)
+
 	isChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
 
 	if !isChunkedFile {