From 2286eda575861cceb4464bf677e9885e069de385 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 26 Mar 2020 23:50:48 -0700 Subject: [PATCH] sampling whether the data can be gzipped --- weed/operation/upload_content.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 75775d7d0..b74663c36 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -78,9 +78,13 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i contentIsGzipped := isInputGzipped shouldGzipNow := false if !isInputGzipped { - if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); mtype == "" || iAmSure && shouldBeZipped { + if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped { shouldGzipNow = true contentIsGzipped = true + } else if len(data) > 128 { + var compressed []byte + compressed, err = util.GzipData(data[0:128]) + shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90% } } @@ -90,7 +94,12 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i // this could be double copying clearDataLen = len(data) if shouldGzipNow { - data, err = util.GzipData(data) + compressed, compressErr := util.GzipData(data) + // fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed)) + if compressErr == nil { + data = compressed + contentIsGzipped = true + } } else if isInputGzipped { // just to get the clear data length clearData, err := util.UnGzipData(data)