diff --git a/weed/operation/compress.go b/weed/operation/compress.go index fedc877dd..a28fb33ec 100644 --- a/weed/operation/compress.go +++ b/weed/operation/compress.go @@ -16,45 +16,58 @@ import ( */ func IsGzippable(ext, mtype string, data []byte) bool { + shouldBeZipped, iAmSure := IsGzippableFileType(ext, mtype) + if iAmSure { + return shouldBeZipped + } + + isMostlyText := util.IsText(data) + + return isMostlyText +} + +/* +* Default more not to gzip since gzip can be done on client side. + */ +func IsGzippableFileType(ext, mtype string) (shouldBeZipped, iAmSure bool) { + // text if strings.HasPrefix(mtype, "text/") { - return true + return true, true } // images switch ext { case ".svg", ".bmp": - return true + return true, true } if strings.HasPrefix(mtype, "image/") { - return false + return false, true } // by file name extension switch ext { case ".zip", ".rar", ".gz", ".bz2", ".xz": - return false + return false, true case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json": - return true + return true, true case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp": - return true + return true, true case ".png", ".jpg", ".jpeg": - return false + return false, true } // by mime type if strings.HasPrefix(mtype, "application/") { if strings.HasSuffix(mtype, "xml") { - return true + return true, true } if strings.HasSuffix(mtype, "script") { - return true + return true, true } } - isMostlyText := util.IsText(data) - - return isMostlyText + return false, false } func GzipData(input []byte) ([]byte, error) { diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 2276c67b7..6bd299826 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -2,6 +2,7 @@ package operation import ( "bytes" + "compress/gzip" "encoding/json" "errors" "fmt" @@ -39,10 +40,24 @@ var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") // Upload sends a POST request to a volume server to upload the content func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { + contentIsGzipped := isGzipped + shouldGzipNow := false + if !isGzipped { + if shouldBeZipped, iAmSure := IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped { + shouldGzipNow = true + contentIsGzipped = true + } + } return upload_content(uploadUrl, func(w io.Writer) (err error) { - _, err = io.Copy(w, reader) + if shouldGzipNow { + gzWriter := gzip.NewWriter(w) + _, err = io.Copy(gzWriter, reader) + gzWriter.Close() + } else { + _, err = io.Copy(w, reader) + } return - }, filename, isGzipped, mtype, pairMap, jwt) + }, filename, contentIsGzipped, mtype, pairMap, jwt) } func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { body_buf := bytes.NewBufferString("")