From 8d34eb005087f6cb305b8934d8614a88b62cff99 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 8 Oct 2020 23:31:26 -0700 Subject: [PATCH] mount:exponentially backoff if read error for about 10 minutes --- weed/filer/filechunk_manifest.go | 27 ++++++++++++++++----------- weed/filer/stream.go | 4 ++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/weed/filer/filechunk_manifest.go b/weed/filer/filechunk_manifest.go index 200fde438..0f84de4cd 100644 --- a/weed/filer/filechunk_manifest.go +++ b/weed/filer/filechunk_manifest.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "math" + "time" "github.com/golang/protobuf/proto" @@ -89,23 +90,27 @@ func fetchChunk(lookupFileIdFn LookupFileIdFunctionType, fileId string, cipherKe glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err) return nil, err } - return fetchChunkData(urlStrings, cipherKey, isGzipped, true, 0, 0) + return retriedFetchChunkData(urlStrings, cipherKey, isGzipped, true, 0, 0) } -func fetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) { +func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) { var err error var buffer bytes.Buffer - for _, urlString := range urlStrings { - err = util.ReadUrlAsStream(urlString, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) { - buffer.Write(data) - }) - if err != nil { - glog.V(0).Infof("read %s failed, err: %v", urlString, err) - buffer.Reset() - } else { - break + + for waitTime := time.Second; waitTime < 10*time.Minute; waitTime+=waitTime/2 { + for _, urlString := range urlStrings { + err = util.ReadUrlAsStream(urlString, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) { + buffer.Write(data) + }) + if err != nil { + glog.V(0).Infof("read %s failed, err: %v", urlString, err) + buffer.Reset() + } else { + break + } } + time.Sleep(waitTime) } return buffer.Bytes(), err diff --git a/weed/filer/stream.go b/weed/filer/stream.go index e1be18f69..f6e2a7643 100644 --- a/weed/filer/stream.go +++ b/weed/filer/stream.go @@ -33,7 +33,7 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f urlStrings := fileId2Url[chunkView.FileId] - data, err := fetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size)) + data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size)) if err == nil { return err } @@ -64,7 +64,7 @@ func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) return nil, err } - data, err := fetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size)) + data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size)) if err != nil { return nil, err }