Browse Source

randomly pick one replica to write

pull/5955/head
chrislu 4 months ago
parent
commit
bebc385607
  1. 30
      weed/operation/submit.go

30
weed/operation/submit.go

@ -4,6 +4,7 @@ import (
"context" "context"
"github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"io" "io"
"math/rand"
"mime" "mime"
"net/url" "net/url"
"os" "os"
@ -179,11 +180,8 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
id += "_" + strconv.FormatInt(i, 10) id += "_" + strconv.FormatInt(i, 10)
} }
} }
fileUrl := "http://" + ret.Url + "/" + id
if usePublicUrl {
fileUrl = "http://" + ret.PublicUrl + "/" + id
}
count, e := upload_one_chunk(
fileUrl := genFileUrl(ret, id, usePublicUrl)
count, e := uploadOneChunk(
baseName+"-"+strconv.FormatInt(i+1, 10), baseName+"-"+strconv.FormatInt(i+1, 10),
io.LimitReader(fi.Reader, chunkSize), io.LimitReader(fi.Reader, chunkSize),
masterFn, fileUrl, masterFn, fileUrl,
@ -202,7 +200,7 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
) )
retSize += count retSize += count
} }
err = upload_chunked_file_manifest(fileUrl, &cm, jwt)
err = uploadChunkedFileManifest(fileUrl, &cm, jwt)
if err != nil { if err != nil {
// delete all uploaded chunks // delete all uploaded chunks
cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption) cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
@ -232,7 +230,23 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
return return
} }
func upload_one_chunk(filename string, reader io.Reader, masterFn GetMasterFn,
func genFileUrl(ret *AssignResult, id string, usePublicUrl bool) string {
fileUrl := "http://" + ret.Url + "/" + id
if usePublicUrl {
fileUrl = "http://" + ret.PublicUrl + "/" + id
}
for _, replica := range ret.Replicas {
if rand.Intn(len(ret.Replicas)+1) == 0 {
fileUrl = "http://" + replica.Url + "/" + id
if usePublicUrl {
fileUrl = "http://" + replica.PublicUrl + "/" + id
}
}
}
return fileUrl
}
func uploadOneChunk(filename string, reader io.Reader, masterFn GetMasterFn,
fileUrl string, jwt security.EncodedJwt, fileUrl string, jwt security.EncodedJwt,
) (size uint32, e error) { ) (size uint32, e error) {
glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...") glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
@ -258,7 +272,7 @@ func upload_one_chunk(filename string, reader io.Reader, masterFn GetMasterFn,
return uploadResult.Size, nil return uploadResult.Size, nil
} }
func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt security.EncodedJwt) error {
func uploadChunkedFileManifest(fileUrl string, manifest *ChunkManifest, jwt security.EncodedJwt) error {
buf, e := manifest.Marshal() buf, e := manifest.Marshal()
if e != nil { if e != nil {
return e return e

Loading…
Cancel
Save