From 59c6422777d8a6703865c8654eb215140cebd29b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 20 Oct 2020 22:53:58 -0700 Subject: [PATCH] a better byte buffer pool --- go.mod | 1 + go.sum | 2 ++ weed/operation/upload_content.go | 8 ++++++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 2a65d47a1..a6af75ff4 100644 --- a/go.mod +++ b/go.mod @@ -70,6 +70,7 @@ require ( github.com/syndtr/goleveldb v1.0.0 github.com/tidwall/gjson v1.3.2 github.com/tidwall/match v1.0.1 + github.com/valyala/bytebufferpool v1.0.0 github.com/willf/bitset v1.1.10 // indirect github.com/willf/bloom v2.0.3+incompatible github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 // indirect diff --git a/go.sum b/go.sum index ce8ff4a8c..dcc4b772a 100644 --- a/go.sum +++ b/go.sum @@ -508,6 +508,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA= diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index e9002d09d..11a6df7c1 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -19,6 +19,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/valyala/bytebufferpool" ) type UploadResult struct { @@ -76,12 +77,14 @@ func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, is } func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) { - data, err = ioutil.ReadAll(reader) + buf := bytebufferpool.Get() + defer bytebufferpool.Put(buf) + _, err = buf.ReadFrom(reader) if err != nil { err = fmt.Errorf("read input: %v", err) return } - uploadResult, uploadErr := retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt) + uploadResult, uploadErr := retriedUploadData(uploadUrl, filename, cipher, buf.Bytes(), isInputCompressed, mtype, pairMap, jwt) return uploadResult, uploadErr, data } @@ -179,6 +182,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i } func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, originalDataSize int, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { + body_buf := bytes.NewBufferString("") body_writer := multipart.NewWriter(body_buf) h := make(textproto.MIMEHeader)