From 77c0de914a92e9588c452662ef5e8baa61578b1f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 20 Sep 2012 21:03:40 -0700 Subject: [PATCH] now any data node can accept storing files files are automatically copied to replicas --- weed-fs/src/cmd/weed/upload.go | 2 +- weed-fs/src/cmd/weed/volume.go | 22 ++++++++++++--------- weed-fs/src/pkg/operation/upload_content.go | 4 ++-- weed-fs/src/pkg/storage/needle.go | 7 ++++--- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/weed-fs/src/cmd/weed/upload.go b/weed-fs/src/cmd/weed/upload.go index c025c5029..01f3e1658 100644 --- a/weed-fs/src/cmd/weed/upload.go +++ b/weed-fs/src/cmd/weed/upload.go @@ -71,7 +71,7 @@ func upload(filename string, server string, fid string) (int) { } panic(err.Error()) } - ret, _ := operation.Upload(server, fid, filename, fh) + ret, _ := operation.Upload("http://"+server+"/"+fid, filename, fh) return ret.Size } diff --git a/weed-fs/src/cmd/weed/volume.go b/weed-fs/src/cmd/weed/volume.go index af4ec8d5e..4fcdf06dc 100644 --- a/weed-fs/src/cmd/weed/volume.go +++ b/weed-fs/src/cmd/weed/volume.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/json" "log" "math/rand" @@ -127,31 +128,34 @@ func GetHandler(w http.ResponseWriter, r *http.Request) { w.Write(n.Data) } func PostHandler(w http.ResponseWriter, r *http.Request) { + r.ParseForm() vid, _, _ := parseURLPath(r.URL.Path) volumeId, e := storage.NewVolumeId(vid) if e != nil { writeJson(w, r, e) } else { - needle, ne := storage.NewNeedle(r) + needle, filename, ne := storage.NewNeedle(r) if ne != nil { writeJson(w, r, ne) } else { ret := store.Write(volumeId, needle) - if ret > 0 { //send to other replica locations + if ret > 0 || !store.HasVolume(volumeId){ //send to other replica locations if r.FormValue("type") != "standard" { waitTime, err := strconv.Atoi(r.FormValue("wait")) lookupResult, lookupErr := operation.Lookup(*server, volumeId) if lookupErr == nil { sendFunc := func(background bool) { - postContentFunc := func(location operation.Location) bool{ - - return true + postContentFunc := func(location operation.Location) bool { + operation.Upload("http://"+location.PublicUrl+r.URL.Path+"?type=standard", filename, bytes.NewReader(needle.Data)) + return true } for _, location := range lookupResult.Locations { - if background { - go postContentFunc(location) - }else{ - postContentFunc(location) + if location.PublicUrl != *publicUrl { + if background { + go postContentFunc(location) + } else { + postContentFunc(location) + } } } } diff --git a/weed-fs/src/pkg/operation/upload_content.go b/weed-fs/src/pkg/operation/upload_content.go index 83e20bf3b..069a9ff4e 100644 --- a/weed-fs/src/pkg/operation/upload_content.go +++ b/weed-fs/src/pkg/operation/upload_content.go @@ -14,14 +14,14 @@ type UploadResult struct { Size int } -func Upload(server string, fid string, filename string, reader io.Reader) (*UploadResult, error) { +func Upload(uploadUrl string, filename string, reader io.Reader) (*UploadResult, error) { body_buf := bytes.NewBufferString("") body_writer := multipart.NewWriter(body_buf) file_writer, err := body_writer.CreateFormFile("file", filename) io.Copy(file_writer, reader) content_type := body_writer.FormDataContentType() body_writer.Close() - resp, err := http.Post("http://"+server+"/"+fid, content_type, body_buf) + resp, err := http.Post(uploadUrl, content_type, body_buf) if err != nil { return nil, err } diff --git a/weed-fs/src/pkg/storage/needle.go b/weed-fs/src/pkg/storage/needle.go index 8d3058a1a..d4db281dc 100644 --- a/weed-fs/src/pkg/storage/needle.go +++ b/weed-fs/src/pkg/storage/needle.go @@ -2,6 +2,7 @@ package storage import ( "encoding/hex" + "fmt" "io" "io/ioutil" "mime" @@ -21,17 +22,17 @@ type Needle struct { Padding []byte "Aligned to 8 bytes" } -func NewNeedle(r *http.Request) (n *Needle, e error) { +func NewNeedle(r *http.Request) (n *Needle, fname string, e error) { n = new(Needle) form, fe := r.MultipartReader() if fe != nil { - println("MultipartReader [ERROR]", fe) + fmt.Println("MultipartReader [ERROR]", fe) e = fe return } part, _ := form.NextPart() - fname := part.FileName() + fname = part.FileName() data, _ := ioutil.ReadAll(part) //log.Println("uploading file " + part.FileName()) dotIndex := strings.LastIndex(fname, ".")