diff --git a/.travis.yml b/.travis.yml index 79cec6d8a..1a2ae20ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: go go: - 1.3 - - release + - 1.4.1 - tip before_install: diff --git a/docs/usecases.rst b/docs/usecases.rst index 8a9a6d44a..ac0a2d0b3 100644 --- a/docs/usecases.rst +++ b/docs/usecases.rst @@ -52,4 +52,63 @@ However, if blocking servicing port is not feasible or trivial, a white list opt weed master -whiteList="::1,127.0.0.1" weed volume -whiteList="::1,127.0.0.1" - # "::1" is for IP v6 localhost. \ No newline at end of file + # "::1" is for IP v6 localhost. + + +Data Migration Example +############################# + +.. code-block:: bash + + weed master -mdir="/tmp/mdata" -defaultReplication="001" -ip="localhost" -port=9334 + weed volume -dir=/tmp/vol1/ -mserver="localhost:9334" -ip="localhost" -port=8081 + weed volume -dir=/tmp/vol2/ -mserver="localhost:9334" -ip="localhost" -port=8082 + weed volume -dir=/tmp/vol3/ -mserver="localhost:9334" -ip="localhost" -port=8083 + +.. code-block:: bash + + ls vol1 vol2 vol3 + vol1: + 1.dat 1.idx 2.dat 2.idx 3.dat 3.idx 5.dat 5.idx + vol2: + 2.dat 2.idx 3.dat 3.idx 4.dat 4.idx 6.dat 6.idx + vol3: + 1.dat 1.idx 4.dat 4.idx 5.dat 5.idx 6.dat 6.idx + +stop all of them + +move vol3/* to vol1 and vol2 + +it is ok to move x.dat and x.idx from one volumeserver to another volumeserver, +because they are exactly the same. +it can be checked by md5. + +.. code-block:: bash + + md5 vol1/1.dat vol2/1.dat + MD5 (vol1/1.dat) = c1a49a0ee550b44fef9f8ae9e55215c7 + MD5 (vol2/1.dat) = c1a49a0ee550b44fef9f8ae9e55215c7 + md5 vol1/1.idx vol2/1.idx + MD5 (vol1/1.idx) = b9edc95795dfb3b0f9063c9cc9ba8095 + MD5 (vol2/1.idx) = b9edc95795dfb3b0f9063c9cc9ba8095 + +.. code-block:: bash + + ls vol1 vol2 vol3 + vol1: + 1.dat 1.idx 2.dat 2.idx 3.dat 3.idx 4.dat 4.idx 5.dat 5.idx 6.dat 6.idx + vol2: + 1.dat 1.idx 2.dat 2.idx 3.dat 3.idx 4.dat 4.idx 5.dat 5.idx 6.dat 6.idx + vol3: + +start + +.. code-block:: bash + + weed master -mdir="/tmp/mdata" -defaultReplication="001" -ip="localhost" -port=9334 + weed volume -dir=/tmp/vol1/ -mserver="localhost:9334" -ip="localhost" -port=8081 + weed volume -dir=/tmp/vol2/ -mserver="localhost:9334" -ip="localhost" -port=8082 + +so we finished moving data of localhost:8083 to localhost:8081/localhost:8082 + + diff --git a/go/operation/delete_content.go b/go/operation/delete_content.go index 416a852b3..064cc8df9 100644 --- a/go/operation/delete_content.go +++ b/go/operation/delete_content.go @@ -66,11 +66,11 @@ func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) { continue } for _, location := range result.Locations { - if _, ok := server_to_fileIds[location.PublicUrl]; !ok { - server_to_fileIds[location.PublicUrl] = make([]string, 0) + if _, ok := server_to_fileIds[location.Url]; !ok { + server_to_fileIds[location.Url] = make([]string, 0) } - server_to_fileIds[location.PublicUrl] = append( - server_to_fileIds[location.PublicUrl], vid_to_fileIds[vid]...) + server_to_fileIds[location.Url] = append( + server_to_fileIds[location.Url], vid_to_fileIds[vid]...) } } diff --git a/go/operation/lookup.go b/go/operation/lookup.go index c05eb1d2b..8156c9779 100644 --- a/go/operation/lookup.go +++ b/go/operation/lookup.go @@ -72,7 +72,7 @@ func LookupFileId(server string, fileId string) (fullUrl string, err error) { if len(lookup.Locations) == 0 { return "", errors.New("File Not Found") } - return "http://" + lookup.Locations[rand.Intn(len(lookup.Locations))].PublicUrl + "/" + fileId, nil + return "http://" + lookup.Locations[rand.Intn(len(lookup.Locations))].Url + "/" + fileId, nil } // LookupVolumeIds find volume locations by cache and actual lookup diff --git a/go/operation/submit.go b/go/operation/submit.go index 62db46617..3ab6d78d9 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -137,7 +137,7 @@ func upload_one_chunk(filename string, reader io.Reader, master, replication str if err != nil { return "", 0, err } - fileUrl, fid := "http://"+ret.PublicUrl+"/"+ret.Fid, ret.Fid + fileUrl, fid := "http://"+ret.Url+"/"+ret.Fid, ret.Fid glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...") uploadResult, uploadError := Upload(fileUrl, filename, reader, false, "application/octet-stream") if uploadError != nil { diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index f4f0b1874..04ab4307d 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -203,7 +203,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { fileSize := int64(*b.fileSize + rand.Intn(64)) fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize} if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil { - fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection + fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection if _, err := fp.Upload(0, *b.server); err == nil { if rand.Intn(100) < *b.deletePercentage { s.total++ @@ -251,7 +251,7 @@ func readFiles(fileIdLineChan chan string, s *stat) { if _, now_ok := b.vid2server[vid]; !now_ok { if ret, err := operation.Lookup(*b.server, vid); err == nil { if len(ret.Locations) > 0 { - server = ret.Locations[0].PublicUrl + server = ret.Locations[0].Url b.vid2server[vid] = server } } diff --git a/go/weed/master.go b/go/weed/master.go index 13f6d7c43..af63d8c22 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -31,7 +31,6 @@ var ( mport = cmdMaster.Flag.Int("port", 9333, "http listen port") masterIp = cmdMaster.Flag.String("ip", "", "master listening ip address, default to listen on all network interfaces") masterBindIp = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") - mPublicIp = cmdMaster.Flag.String("publicIp", "", "peer accessible |") metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data") masterPeers = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list") volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.") @@ -76,19 +75,15 @@ func runMaster(cmd *Command, args []string) bool { go func() { time.Sleep(100 * time.Millisecond) - if *mPublicIp == "" { - if *masterIp == "" { - *mPublicIp = "localhost" - } else { - *mPublicIp = *masterIp - } + if *masterIp == "" { + *masterIp = "localhost" } - myPublicMasterAddress := *mPublicIp + ":" + strconv.Itoa(*mport) + myMasterAddress := *masterIp + ":" + strconv.Itoa(*mport) var peers []string if *masterPeers != "" { peers = strings.Split(*masterPeers, ",") } - raftServer := weed_server.NewRaftServer(r, peers, myPublicMasterAddress, *metaFolder, ms.Topo, *mpulse) + raftServer := weed_server.NewRaftServer(r, peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse) ms.SetRaftServer(raftServer) }() diff --git a/go/weed/server.go b/go/weed/server.go index 16b809c53..b779033cb 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -48,7 +48,7 @@ var cmdServer = &Command{ var ( serverIp = cmdServer.Flag.String("ip", "", "ip or server name") - serverPublicIp = cmdServer.Flag.String("publicIp", "", "ip or server name") + serverPublicUrl = cmdServer.Flag.String("publicUrl", "", "publicly accessible address") serverBindIp = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") serverMaxCpu = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") serverTimeout = cmdServer.Flag.Int("idleTimeout", 10, "connection idle seconds") @@ -99,19 +99,15 @@ func runServer(cmd *Command, args []string) bool { defer pprof.StopCPUProfile() } - if *serverPublicIp == "" { - if *serverIp == "" { - *serverPublicIp = "localhost" - } else { - *serverPublicIp = *serverIp - } + if *serverIp == "" { + *serverIp = "localhost" } if *filerOptions.redirectOnRead { *isStartingFiler = true } - *filerOptions.master = *serverPublicIp + ":" + strconv.Itoa(*masterPort) + *filerOptions.master = *serverIp + ":" + strconv.Itoa(*masterPort) if *filerOptions.defaultReplicaPlacement == "" { *filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement @@ -144,9 +140,11 @@ func runServer(cmd *Command, args []string) bool { if *masterMetaFolder == "" { *masterMetaFolder = folders[0] } - if *filerOptions.dir == "" { - *filerOptions.dir = *masterMetaFolder + "/filer" - os.MkdirAll(*filerOptions.dir, 0700) + if *isStartingFiler { + if *filerOptions.dir == "" { + *filerOptions.dir = *masterMetaFolder + "/filer" + os.MkdirAll(*filerOptions.dir, 0700) + } } if err := util.TestFolderWritable(*masterMetaFolder); err != nil { glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err) @@ -207,7 +205,7 @@ func runServer(cmd *Command, args []string) bool { go func() { raftWaitForMaster.Wait() time.Sleep(100 * time.Millisecond) - myAddress := *serverPublicIp + ":" + strconv.Itoa(*masterPort) + myAddress := *serverIp + ":" + strconv.Itoa(*masterPort) var peers []string if *serverPeers != "" { peers = strings.Split(*serverPeers, ",") @@ -227,7 +225,7 @@ func runServer(cmd *Command, args []string) bool { time.Sleep(100 * time.Millisecond) r := http.NewServeMux() volumeServer := weed_server.NewVolumeServer(r, r, - *serverIp, *volumePort, *volumeAdminPort, *serverPublicIp, + *serverIp, *volumePort, *volumeAdminPort, *serverPublicUrl, folders, maxCounts, *serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, serverWhiteList, *volumeFixJpgOrientation, diff --git a/go/weed/volume.go b/go/weed/volume.go index df1d603ac..df3974c8c 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -23,7 +23,7 @@ type VolumeServerOptions struct { folders []string folderMaxLimits []int ip *string - publicIp *string + publicUrl *string bindIp *string master *string pulseSeconds *int @@ -40,7 +40,7 @@ func init() { v.port = cmdVolume.Flag.Int("port", 8080, "http listen port") v.adminPort = cmdVolume.Flag.Int("port.admin", 0, "admin port to talk with master and other volume servers") v.ip = cmdVolume.Flag.String("ip", "", "ip or server name") - v.publicIp = cmdVolume.Flag.String("publicIp", "", "Publicly accessible ") + v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address") v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") v.master = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location") v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting") @@ -95,14 +95,8 @@ func runVolume(cmd *Command, args []string) bool { v.whiteList = strings.Split(*volumeWhiteListOption, ",") } - //derive default public ip address - if *v.publicIp == "" { - if *v.ip == "" { - *v.ip = "127.0.0.1" - *v.publicIp = "localhost" - } else { - *v.publicIp = *v.ip - } + if *v.ip == "" { + *v.ip = "127.0.0.1" } if *v.adminPort == 0 { @@ -117,7 +111,7 @@ func runVolume(cmd *Command, args []string) bool { } volumeServer := weed_server.NewVolumeServer(publicMux, adminMux, - *v.ip, *v.port, *v.adminPort, *v.publicIp, + *v.ip, *v.port, *v.adminPort, *v.publicUrl, v.folders, v.folderMaxLimits, *v.master, *v.pulseSeconds, *v.dataCenter, *v.rack, v.whiteList, diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index d259aa660..a2d93c246 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -96,7 +96,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st return } - url := "http://" + assignResult.PublicUrl + "/" + assignResult.Fid + url := "http://" + assignResult.Url + "/" + assignResult.Fid if lastModified != 0 { url = url + "?ts=" + strconv.FormatUint(lastModified, 10) } diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 0811b7973..ac894771a 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -80,7 +80,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, w.WriteHeader(http.StatusNotFound) return } - urlLocation := lookup.Locations[rand.Intn(len(lookup.Locations))].PublicUrl + urlLocation := lookup.Locations[rand.Intn(len(lookup.Locations))].Url urlString := "http://" + urlLocation + "/" + fileId if fs.redirectOnRead { http.Redirect(w, r, urlString, http.StatusFound) @@ -126,7 +126,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { return } - u, _ := url.Parse("http://" + assignResult.PublicUrl + "/" + assignResult.Fid) + u, _ := url.Parse("http://" + assignResult.Url + "/" + assignResult.Fid) glog.V(4).Infoln("post to", u) request := &http.Request{ Method: r.Method, diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 4d304efcb..33e45afd2 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -119,7 +119,7 @@ func (ms *MasterServer) redirectHandler(w http.ResponseWriter, r *http.Request) } machines := ms.Topo.Lookup("", volumeId) if machines != nil && len(machines) > 0 { - http.Redirect(w, r, "http://"+machines[0].PublicUrl+r.URL.Path, http.StatusMovedPermanently) + http.Redirect(w, r, "http://"+machines[0].Url()+r.URL.Path, http.StatusMovedPermanently) } else { writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("volume id %d not found.", volumeId)) } diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index 8846c4794..f3ad2974d 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -3,7 +3,6 @@ package weed_server import ( "math/rand" "net/http" - "strconv" "time" "github.com/chrislusf/weed-fs/go/glog" @@ -23,13 +22,12 @@ type VolumeServer struct { } func NewVolumeServer(publicMux, adminMux *http.ServeMux, ip string, - port, adminPort int, publicIp string, + port, adminPort int, publicUrl string, folders []string, maxCounts []int, masterNode string, pulseSeconds int, dataCenter string, rack string, whiteList []string, fixJpgOrientation bool) *VolumeServer { - publicUrl := publicIp + ":" + strconv.Itoa(port) vs := &VolumeServer{ masterNode: masterNode, pulseSeconds: pulseSeconds, diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index 6b47ee84d..c2c9e8523 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -61,7 +61,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) lookupResult, err := operation.Lookup(vs.masterNode, volumeId.String()) glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err) if err == nil && len(lookupResult.Locations) > 0 { - http.Redirect(w, r, "http://"+lookupResult.Locations[0].PublicUrl+r.URL.Path, http.StatusMovedPermanently) + http.Redirect(w, r, "http://"+lookupResult.Locations[0].Url+r.URL.Path, http.StatusMovedPermanently) } else { glog.V(2).Infoln("lookup error:", err, r.URL.Path) w.WriteHeader(http.StatusNotFound)