support gzip file upload, fix problem during replication of gzipped data

go/operation/upload_content.go

@@ -21,12 +21,16 @@ type UploadResult struct {
 	Error string
 }
 
-func Upload(uploadUrl string, filename string, reader io.Reader) (*UploadResult, error) {
+func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool) (*UploadResult, error) {
 	body_buf := bytes.NewBufferString("")
 	body_writer := multipart.NewWriter(body_buf)
 	h := make(textproto.MIMEHeader)
 	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename))
 	h.Set("Content-Type", mime.TypeByExtension(strings.ToLower(filepath.Ext(filename))))
+	println("content is gzipped", isGzipped)
+	if isGzipped {
+		h.Set("Content-Encoding", "gzip")
+	}
 	file_writer, err := body_writer.CreatePart(h)
 	if err != nil {
 		log.Println("error creating form file", err)

go/replication/store_replicate.go

@@ -25,7 +25,7 @@ func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.Volum
 	if needToReplicate { //send to other replica locations
 		if r.FormValue("type") != "replicate" {
 			if !distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool {
-				_, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified,10), string(needle.Name), bytes.NewReader(needle.Data))
+				_, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified,10), string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped())
 				return err == nil
 			}) {
 				ret = 0

go/storage/needle.go

@@ -4,8 +4,8 @@ import (
 	"code.google.com/p/weed-fs/go/util"
 	"encoding/hex"
 	"errors"
-	"log"
 	"io/ioutil"
+	"log"
 	"mime"
 	"net/http"
 	"path"
@@ -73,7 +73,9 @@ func NewNeedle(r *http.Request) (n *Needle, e error) {
 		n.SetHasMime()
 		mtype = contentType
 	}
-	if IsGzippable(ext, mtype) {
+	if part.Header.Get("Content-Encoding") == "gzip" {
+		n.SetGzipped()
+	} else if IsGzippable(ext, mtype) {
 		if data, e = GzipData(data); e != nil {
 			return
 		}

go/weed/upload.go

@@ -78,7 +78,7 @@ func upload(filename string, server string, fid string) (int, error) {
 		debug("Failed to stat file:", filename)
 		return 0, fiErr
 	}
-	ret, e := operation.Upload("http://"+server+"/"+fid+"?ts="+strconv.Itoa(int(fi.ModTime().Unix())), path.Base(filename), fh)
+	ret, e := operation.Upload("http://"+server+"/"+fid+"?ts="+strconv.Itoa(int(fi.ModTime().Unix())), path.Base(filename), fh, false)
 	if e != nil {
 		return 0, e
 	}