From e9a8999f63f79406249debb8023be014e8e12eda Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 21 Oct 2014 01:27:06 -0700 Subject: [PATCH 01/23] print error the correct way. --- go/storage/volume.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/go/storage/volume.go b/go/storage/volume.go index de79e9107..60ee1a0c4 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -2,9 +2,9 @@ package storage import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "errors" "fmt" + "github.com/chrislusf/weed-fs/go/glog" "io" "os" "path" @@ -72,7 +72,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error { if e != nil { if !os.IsPermission(e) { - return fmt.Errorf("cannot load Volume Data %s.dat: %s", fileName, e.Error()) + return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, e) } } @@ -92,12 +92,12 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error { if v.readOnly { glog.V(1).Infoln("open to read file", fileName+".idx") if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); e != nil { - return fmt.Errorf("cannot read Volume Index %s.idx: %s", fileName, e.Error()) + return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, e) } } else { glog.V(1).Infoln("open to write file", fileName+".idx") if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); e != nil { - return fmt.Errorf("cannot write Volume Index %s.idx: %s", fileName, e.Error()) + return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, e) } } glog.V(0).Infoln("loading file", fileName+".idx", "readonly", v.readOnly) @@ -115,7 +115,7 @@ func (v *Volume) Size() int64 { if e == nil { return stat.Size() } - glog.V(0).Infof("Failed to read file size %s %s", v.dataFile.Name(), e.Error()) + glog.V(0).Infof("Failed to read file size %s %v", v.dataFile.Name(), e) return -1 } func (v *Volume) Close() { @@ -170,6 +170,7 @@ func (v *Volume) write(n *Needle) (size uint32, err error) { } var offset int64 if offset, err = v.dataFile.Seek(0, 2); err != nil { + glog.V(0).Infof("faile to seek the end of file: %v", err) return } @@ -177,21 +178,21 @@ func (v *Volume) write(n *Needle) (size uint32, err error) { if offset%NeedlePaddingSize != 0 { offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize) if offset, err = v.dataFile.Seek(offset, 0); err != nil { - glog.V(4).Infof("failed to align in datafile %s: %s", v.dataFile.Name(), err.Error()) + glog.V(0).Infof("failed to align in datafile %s: %v", v.dataFile.Name(), err) return } } if size, err = n.Append(v.dataFile, v.Version()); err != nil { if e := v.dataFile.Truncate(offset); e != nil { - err = fmt.Errorf("%s\ncannot truncate %s: %s", err, v.dataFile.Name(), e.Error()) + err = fmt.Errorf("%s\ncannot truncate %s: %v", err, v.dataFile.Name(), e) } return } nv, ok := v.nm.Get(n.Id) if !ok || int64(nv.Offset)*NeedlePaddingSize < offset { if _, err = v.nm.Put(n.Id, uint32(offset/NeedlePaddingSize), n.Size); err != nil { - glog.V(4).Infof("failed to save in needle map %d: %s", n.Id, err.Error()) + glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err) } } if v.lastModifiedTime < n.LastModified { @@ -292,13 +293,13 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId, offset := int64(SuperBlockSize) n, rest, e := ReadNeedleHeader(v.dataFile, version, offset) if e != nil { - err = fmt.Errorf("cannot read needle header: %s", e) + err = fmt.Errorf("cannot read needle header: %v", e) return } for n != nil { if readNeedleBody { if err = n.ReadNeedleBody(v.dataFile, version, offset+int64(NeedleHeaderSize), rest); err != nil { - err = fmt.Errorf("cannot read needle body: %s", err) + err = fmt.Errorf("cannot read needle body: %v", err) return } } @@ -310,7 +311,7 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId, if err == io.EOF { return nil } - return fmt.Errorf("cannot read needle header: %s", err) + return fmt.Errorf("cannot read needle header: %v", err) } } @@ -360,7 +361,7 @@ func (v *Volume) ensureConvertIdxToCdb(fileName string) (cdbCanRead bool) { defer indexFile.Close() glog.V(0).Infof("converting %s.idx to %s.cdb", fileName, fileName) if e = ConvertIndexToCdb(fileName+".cdb", indexFile); e != nil { - glog.V(0).Infof("error converting %s.idx to %s.cdb: %s", fileName, fileName, e.Error()) + glog.V(0).Infof("error converting %s.idx to %s.cdb: %v", fileName, fileName, e) return false } return true From 670b240a266f23fe9d8e5120c61d32065a1519b3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 21 Oct 2014 01:27:40 -0700 Subject: [PATCH 02/23] Fix help text error. --- go/weed/compact.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/weed/compact.go b/go/weed/compact.go index a99e6c93e..71c4ea90f 100644 --- a/go/weed/compact.go +++ b/go/weed/compact.go @@ -12,7 +12,7 @@ func init() { var cmdCompact = &Command{ UsageLine: "compact -dir=/tmp -volumeId=234", - Short: "run weed tool compact on volume file if corrupted", + Short: "run weed tool compact on volume file", Long: `Force an compaction to remove deleted files from volume files. The compacted .dat file is stored as .cpd file. The compacted .idx file is stored as .cpx file. From b5aa2ef6050bc052e73536481893011f55005099 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 21 Oct 2014 01:28:17 -0700 Subject: [PATCH 03/23] Add master bind ip address option. --- go/weed/master.go | 3 ++- go/weed/volume.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go/weed/master.go b/go/weed/master.go index 6617c8ca6..f96ff4c08 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -29,6 +29,7 @@ var cmdMaster = &Command{ var ( mport = cmdMaster.Flag.Int("port", 9333, "http listen port") masterIp = cmdMaster.Flag.String("ip", "", "master listening ip address, default to listen on all network interfaces") + masterBindIp = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") mPublicIp = cmdMaster.Flag.String("publicIp", "", "peer accessible |") metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data") masterPeers = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list") @@ -61,7 +62,7 @@ func runMaster(cmd *Command, args []string) bool { *volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold, masterWhiteList, ) - listeningAddress := *masterIp + ":" + strconv.Itoa(*mport) + listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport) glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress) diff --git a/go/weed/volume.go b/go/weed/volume.go index 17d03f0c5..ce05fcdf3 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -30,7 +30,7 @@ var ( maxVolumeCounts = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...") ip = cmdVolume.Flag.String("ip", "", "ip or server name") publicIp = cmdVolume.Flag.String("publicIp", "", "Publicly accessible ") - bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") + volumeBindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") masterNode = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location") vpulse = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting") vTimeout = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds") @@ -85,7 +85,7 @@ func runVolume(cmd *Command, args []string) bool { *fixJpgOrientation, ) - listeningAddress := *bindIp + ":" + strconv.Itoa(*vport) + listeningAddress := *volumeBindIp + ":" + strconv.Itoa(*vport) glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress) From b85eb293feda0c4b2c322a3d07d17df566df02a4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 23 Oct 2014 09:55:05 -0700 Subject: [PATCH 04/23] fix doc about submit api --- docs/api.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index f26408f5a..b67463960 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -187,10 +187,11 @@ Upload File Directly .. code-block:: bash - curl -F file=@/home/chris/myphoto.jpg http://localhost:8080/submit + curl -F file=@/home/chris/myphoto.jpg http://localhost:9333/submit {"fid":"3,01fbe0dc6f1f38","fileName":"myphoto.jpg","fileUrl":"localhost:8080/3,01fbe0dc6f1f38","size":68231} -This API is a little convenient. The volume server would contact the master to get an file id and store it to the right volume server(not necessarily itself). +This API is just for convenience. The master server would get an file id and store the file to the right volume server. +It is a convenient API and does not support different parameters when assigning file id. (or you can add the support and send a push request.) Delete File *********************************** From 30bcda7136f847b59ac6e6610a40829dc10ec8cd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 25 Oct 2014 18:10:32 -0700 Subject: [PATCH 05/23] fix typo --- go/topology/topology.go | 2 +- go/weed/weed_server/master_server_handlers.go | 4 ++-- go/weed/weed_server/master_server_handlers_admin.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/topology/topology.go b/go/topology/topology.go index c90e8de0b..a81829bb6 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -109,7 +109,7 @@ func (t *Topology) NextVolumeId() storage.VolumeId { return next } -func (t *Topology) HasWriableVolume(option *VolumeGrowOption) bool { +func (t *Topology) hasWritableVolume(option *VolumeGrowOption) bool { vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 93e9e7d9a..be2a7ca7e 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -78,7 +78,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) return } - if !ms.Topo.HasWriableVolume(option) { + if !ms.Topo.hasWritableVolume(option) { if ms.Topo.FreeSpace() <= 0 { w.WriteHeader(http.StatusNotFound) writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"}) @@ -86,7 +86,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } else { ms.vgLock.Lock() defer ms.vgLock.Unlock() - if !ms.Topo.HasWriableVolume(option) { + if !ms.Topo.hasWritableVolume(option) { if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil { writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()}) return diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index c9a8020c2..080405e54 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -143,7 +143,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r * } } -func (ms *MasterServer) hasWriableVolume(option *topology.VolumeGrowOption) bool { +func (ms *MasterServer) hasWritableVolume(option *topology.VolumeGrowOption) bool { vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } From f527fc1d5ee975fd29e49bf542d893a2d433a491 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 25 Oct 2014 23:45:31 -0700 Subject: [PATCH 06/23] adjust visibility --- go/topology/topology.go | 2 +- go/weed/weed_server/master_server_handlers.go | 4 ++-- go/weed/weed_server/master_server_handlers_admin.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/topology/topology.go b/go/topology/topology.go index a81829bb6..cfce0c9a8 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -109,7 +109,7 @@ func (t *Topology) NextVolumeId() storage.VolumeId { return next } -func (t *Topology) hasWritableVolume(option *VolumeGrowOption) bool { +func (t *Topology) HasWritableVolume(option *VolumeGrowOption) bool { vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index be2a7ca7e..d7f1f4ce3 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -78,7 +78,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) return } - if !ms.Topo.hasWritableVolume(option) { + if !ms.Topo.HasWritableVolume(option) { if ms.Topo.FreeSpace() <= 0 { w.WriteHeader(http.StatusNotFound) writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"}) @@ -86,7 +86,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } else { ms.vgLock.Lock() defer ms.vgLock.Unlock() - if !ms.Topo.hasWritableVolume(option) { + if !ms.Topo.HasWritableVolume(option) { if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil { writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()}) return diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 080405e54..1a2c6b8e0 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -143,7 +143,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r * } } -func (ms *MasterServer) hasWritableVolume(option *topology.VolumeGrowOption) bool { +func (ms *MasterServer) HasWritableVolume(option *topology.VolumeGrowOption) bool { vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } From a5d6e70299559c10126f6ef248f475152303a79a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 26 Oct 2014 11:25:02 -0700 Subject: [PATCH 07/23] fix commenting error. --- go/weed/export.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/weed/export.go b/go/weed/export.go index 81bc21f6e..9f33d852f 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -3,9 +3,9 @@ package main import ( "archive/tar" "bytes" + "fmt" "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/storage" - "fmt" "os" "path" "strconv" @@ -36,7 +36,7 @@ var cmdExport = &Command{ var ( exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "input data directory to store volume data files") exportCollection = cmdExport.Flag.String("collection", "", "the volume collection name") - exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.") + exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.") dest = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout") format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}") tarFh *tar.Writer From 179d36ba0e35e4bdad86988a49828836b649e3df Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 26 Oct 2014 11:34:55 -0700 Subject: [PATCH 08/23] formatting code by: goimports -w=true . --- go/filer/client_operations.go | 6 +++--- go/filer/directory.go | 2 -- go/filer/directory_in_map.go | 3 ++- go/filer/filer.go | 2 -- go/filer/filer_embedded.go | 3 ++- go/filer/files_in_leveldb.go | 1 + go/glog/convenient_api.go | 2 -- go/images/orientation.go | 3 ++- go/images/resizing.go | 3 ++- go/operation/assign_file_id.go | 5 +++-- go/operation/data_struts.go | 2 -- go/operation/delete_content.go | 3 ++- go/operation/list_masters.go | 3 ++- go/operation/lookup.go | 3 ++- go/operation/submit.go | 3 ++- go/operation/system_message_test.go | 3 ++- go/operation/upload_content.go | 3 ++- go/sequence/sequence.go | 2 -- go/stats/disk.go | 2 -- go/stats/disk_notsupported.go | 2 -- go/stats/memory_notsupported.go | 2 -- go/storage/cdb_map.go | 5 +++-- go/storage/cdb_map_test.go | 3 ++- go/storage/compact_map.go | 2 -- go/storage/compact_map_perf_test.go | 5 +++-- go/storage/compress.go | 3 ++- go/storage/crc.go | 3 ++- go/storage/file_id.go | 5 +++-- go/storage/needle.go | 7 ++++--- go/storage/needle_map.go | 5 +++-- go/storage/needle_read_write.go | 5 +++-- go/storage/store.go | 9 +++++---- go/storage/store_vacuum.go | 3 ++- go/storage/volume.go | 3 ++- go/storage/volume_super_block.go | 3 ++- go/storage/volume_vacuum.go | 3 ++- go/storage/volume_version.go | 2 -- go/tools/read_index.go | 3 ++- go/topology/allocate_volume.go | 5 +++-- go/topology/data_center.go | 2 -- go/topology/data_node.go | 3 ++- go/topology/node.go | 5 +++-- go/topology/store_replicate.go | 5 +++-- go/topology/topology.go | 7 ++++--- go/topology/topology_event_handling.go | 5 +++-- go/topology/topology_map.go | 2 -- go/topology/topology_vacuum.go | 7 ++++--- go/topology/volume_growth.go | 5 +++-- go/topology/volume_growth_test.go | 5 +++-- go/topology/volume_layout.go | 5 +++-- go/topology/volume_location_list.go | 2 -- go/util/config.go | 3 ++- go/util/constants.go | 2 -- go/util/file_util.go | 3 ++- go/util/net_timeout.go | 3 ++- go/weed/benchmark.go | 7 ++++--- go/weed/download.go | 5 +++-- go/weed/export.go | 5 +++-- go/weed/filer.go | 7 ++++--- go/weed/fix.go | 5 +++-- go/weed/master.go | 9 +++++---- go/weed/mount.go | 2 -- go/weed/mount_std.go | 7 ++++--- go/weed/server.go | 9 +++++---- go/weed/shell.go | 3 ++- go/weed/signal_handling_notsupported.go | 2 -- go/weed/upload.go | 3 ++- go/weed/version.go | 3 ++- go/weed/volume.go | 7 ++++--- go/weed/volume_test.go | 3 ++- go/weed/weed.go | 3 ++- go/weed/weed_server/common.go | 11 ++++++----- go/weed/weed_server/filer_server.go | 5 +++-- go/weed/weed_server/filer_server_handlers.go | 9 +++++---- go/weed/weed_server/filer_server_handlers_admin.go | 3 ++- go/weed/weed_server/master_server.go | 9 +++++---- go/weed/weed_server/master_server_handlers.go | 7 ++++--- go/weed/weed_server/master_server_handlers_admin.go | 13 +++++++------ go/weed/weed_server/raft_server.go | 9 +++++---- go/weed/weed_server/raft_server_handlers.go | 7 ++++--- go/weed/weed_server/volume_server.go | 5 +++-- go/weed/weed_server/volume_server_handlers.go | 13 +++++++------ go/weed/weed_server/volume_server_handlers_admin.go | 5 +++-- .../weed_server/volume_server_handlers_vacuum.go | 3 ++- 84 files changed, 205 insertions(+), 170 deletions(-) diff --git a/go/filer/client_operations.go b/go/filer/client_operations.go index 0b006289f..b38368735 100644 --- a/go/filer/client_operations.go +++ b/go/filer/client_operations.go @@ -1,12 +1,12 @@ package filer -import () - import ( - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "fmt" + + "github.com/chrislusf/weed-fs/go/util" + "net/url" ) diff --git a/go/filer/directory.go b/go/filer/directory.go index 956a2f504..66d1aeba5 100644 --- a/go/filer/directory.go +++ b/go/filer/directory.go @@ -1,7 +1,5 @@ package filer -import () - type DirectoryId int32 type DirectoryEntry struct { diff --git a/go/filer/directory_in_map.go b/go/filer/directory_in_map.go index 1d88a78be..ee601066c 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/directory_in_map.go @@ -2,7 +2,6 @@ package filer import ( "bufio" - "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "os" @@ -10,6 +9,8 @@ import ( "strconv" "strings" "sync" + + "github.com/chrislusf/weed-fs/go/util" ) var writeLock sync.Mutex //serialize changes to dir.log diff --git a/go/filer/filer.go b/go/filer/filer.go index de877fc1f..bf4a1cb19 100644 --- a/go/filer/filer.go +++ b/go/filer/filer.go @@ -1,7 +1,5 @@ package filer -import () - type FileId string //file id on weedfs type FileEntry struct { diff --git a/go/filer/filer_embedded.go b/go/filer/filer_embedded.go index 3d3dac941..0b9b4e668 100644 --- a/go/filer/filer_embedded.go +++ b/go/filer/filer_embedded.go @@ -1,11 +1,12 @@ package filer import ( - "github.com/chrislusf/weed-fs/go/operation" "errors" "fmt" "path/filepath" "strings" + + "github.com/chrislusf/weed-fs/go/operation" ) type FilerEmbedded struct { diff --git a/go/filer/files_in_leveldb.go b/go/filer/files_in_leveldb.go index 41fbc74bd..781bb0e5f 100644 --- a/go/filer/files_in_leveldb.go +++ b/go/filer/files_in_leveldb.go @@ -2,6 +2,7 @@ package filer import ( "bytes" + "github.com/chrislusf/weed-fs/go/glog" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" diff --git a/go/glog/convenient_api.go b/go/glog/convenient_api.go index 3c378083f..cb43d60e2 100644 --- a/go/glog/convenient_api.go +++ b/go/glog/convenient_api.go @@ -1,7 +1,5 @@ package glog -import () - /* Copying the original glog because it is missing several convenient methods. 1. remove nano time in log format diff --git a/go/images/orientation.go b/go/images/orientation.go index 41ed3f0af..4bff89311 100644 --- a/go/images/orientation.go +++ b/go/images/orientation.go @@ -2,11 +2,12 @@ package images import ( "bytes" - "github.com/rwcarlsen/goexif/exif" "image" "image/draw" "image/jpeg" "log" + + "github.com/rwcarlsen/goexif/exif" ) //many code is copied from http://camlistore.org/pkg/images/images.go diff --git a/go/images/resizing.go b/go/images/resizing.go index 08a1e15d2..e9de5f7d7 100644 --- a/go/images/resizing.go +++ b/go/images/resizing.go @@ -2,11 +2,12 @@ package images import ( "bytes" - "github.com/disintegration/imaging" "image" "image/gif" "image/jpeg" "image/png" + + "github.com/disintegration/imaging" ) func Resized(ext string, data []byte, width, height int) (resized []byte, w int, h int) { diff --git a/go/operation/assign_file_id.go b/go/operation/assign_file_id.go index 4e72ad939..672bfa99c 100644 --- a/go/operation/assign_file_id.go +++ b/go/operation/assign_file_id.go @@ -1,12 +1,13 @@ package operation import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" "strconv" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type AssignResult struct { diff --git a/go/operation/data_struts.go b/go/operation/data_struts.go index 09fab05d1..4980f9913 100644 --- a/go/operation/data_struts.go +++ b/go/operation/data_struts.go @@ -1,7 +1,5 @@ package operation -import () - type JoinResult struct { VolumeSizeLimit uint64 `json:"VolumeSizeLimit,omitempty"` Error string `json:"error,omitempty"` diff --git a/go/operation/delete_content.go b/go/operation/delete_content.go index 84391b634..416a852b3 100644 --- a/go/operation/delete_content.go +++ b/go/operation/delete_content.go @@ -1,12 +1,13 @@ package operation import ( - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" "strings" "sync" + + "github.com/chrislusf/weed-fs/go/util" ) type DeleteResult struct { diff --git a/go/operation/list_masters.go b/go/operation/list_masters.go index 7d46a9ebc..7ada94243 100644 --- a/go/operation/list_masters.go +++ b/go/operation/list_masters.go @@ -1,9 +1,10 @@ package operation import ( + "encoding/json" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/util" - "encoding/json" ) type ClusterStatusResult struct { diff --git a/go/operation/lookup.go b/go/operation/lookup.go index ebf153d27..e6b6658da 100644 --- a/go/operation/lookup.go +++ b/go/operation/lookup.go @@ -1,7 +1,6 @@ package operation import ( - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" _ "fmt" @@ -9,6 +8,8 @@ import ( "net/url" "strings" "time" + + "github.com/chrislusf/weed-fs/go/util" ) type Location struct { diff --git a/go/operation/submit.go b/go/operation/submit.go index 3e09c2edf..62db46617 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -2,13 +2,14 @@ package operation import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "io" "mime" "os" "path" "strconv" "strings" + + "github.com/chrislusf/weed-fs/go/glog" ) type FilePart struct { diff --git a/go/operation/system_message_test.go b/go/operation/system_message_test.go index 2731d0b2f..b5624c258 100644 --- a/go/operation/system_message_test.go +++ b/go/operation/system_message_test.go @@ -1,10 +1,11 @@ package operation import ( - proto "code.google.com/p/goprotobuf/proto" "encoding/json" "log" "testing" + + proto "code.google.com/p/goprotobuf/proto" ) func TestSerialDeserial(t *testing.T) { diff --git a/go/operation/upload_content.go b/go/operation/upload_content.go index 38737702d..480d76dca 100644 --- a/go/operation/upload_content.go +++ b/go/operation/upload_content.go @@ -2,7 +2,6 @@ package operation import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "encoding/json" "errors" "fmt" @@ -14,6 +13,8 @@ import ( "net/textproto" "path/filepath" "strings" + + "github.com/chrislusf/weed-fs/go/glog" ) type UploadResult struct { diff --git a/go/sequence/sequence.go b/go/sequence/sequence.go index 5a1bceaaf..1aa167b6b 100644 --- a/go/sequence/sequence.go +++ b/go/sequence/sequence.go @@ -1,7 +1,5 @@ package sequence -import () - type Sequencer interface { NextFileId(count int) (uint64, int) SetMax(uint64) diff --git a/go/stats/disk.go b/go/stats/disk.go index d5275e571..46d8c465e 100644 --- a/go/stats/disk.go +++ b/go/stats/disk.go @@ -1,7 +1,5 @@ package stats -import () - type DiskStatus struct { Dir string All uint64 diff --git a/go/stats/disk_notsupported.go b/go/stats/disk_notsupported.go index 37f9bc47d..e380d27ea 100644 --- a/go/stats/disk_notsupported.go +++ b/go/stats/disk_notsupported.go @@ -2,8 +2,6 @@ package stats -import () - func (disk *DiskStatus) fillInStatus() { return } diff --git a/go/stats/memory_notsupported.go b/go/stats/memory_notsupported.go index 64c3d7c2f..ba8229364 100644 --- a/go/stats/memory_notsupported.go +++ b/go/stats/memory_notsupported.go @@ -2,8 +2,6 @@ package stats -import () - func (mem *MemStatus) fillInStatus() { return } diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go index 1869a563e..fbb59e9c0 100644 --- a/go/storage/cdb_map.go +++ b/go/storage/cdb_map.go @@ -1,13 +1,14 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "fmt" - "github.com/tgulacsi/go-cdb" "os" "path/filepath" + + "github.com/chrislusf/weed-fs/go/util" + "github.com/tgulacsi/go-cdb" ) // CDB-backed read-only needle map diff --git a/go/storage/cdb_map_test.go b/go/storage/cdb_map_test.go index cff7dfa61..ed690f44f 100644 --- a/go/storage/cdb_map_test.go +++ b/go/storage/cdb_map_test.go @@ -1,11 +1,12 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" "math/rand" "os" "runtime" "testing" + + "github.com/chrislusf/weed-fs/go/glog" ) var testIndexFilename string = "../../test/sample.idx" diff --git a/go/storage/compact_map.go b/go/storage/compact_map.go index 9cfc3e8f7..6ac18b012 100644 --- a/go/storage/compact_map.go +++ b/go/storage/compact_map.go @@ -1,7 +1,5 @@ package storage -import () - type NeedleValue struct { Key Key Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index ef43de25b..f74684225 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -1,11 +1,12 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "log" "os" "testing" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) func TestMemoryUsage(t *testing.T) { diff --git a/go/storage/compress.go b/go/storage/compress.go index a353c9d3a..5efc9e1f1 100644 --- a/go/storage/compress.go +++ b/go/storage/compress.go @@ -2,11 +2,12 @@ package storage import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "compress/flate" "compress/gzip" "io/ioutil" "strings" + + "github.com/chrislusf/weed-fs/go/glog" ) /* diff --git a/go/storage/crc.go b/go/storage/crc.go index 7aa400959..af25b9e53 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -1,9 +1,10 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/util" "fmt" "hash/crc32" + + "github.com/chrislusf/weed-fs/go/util" ) var table = crc32.MakeTable(crc32.Castagnoli) diff --git a/go/storage/file_id.go b/go/storage/file_id.go index ec566826c..f6e36a98c 100644 --- a/go/storage/file_id.go +++ b/go/storage/file_id.go @@ -1,11 +1,12 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "encoding/hex" "errors" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type FileId struct { diff --git a/go/storage/needle.go b/go/storage/needle.go index daede321b..aa3206920 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -1,9 +1,6 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/images" - "github.com/chrislusf/weed-fs/go/util" "encoding/hex" "errors" "io/ioutil" @@ -13,6 +10,10 @@ import ( "strconv" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/images" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index dca2e6c5d..504ca1552 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -1,11 +1,12 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "os" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type NeedleMapper interface { diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index bf452ba37..663b5abbd 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -1,12 +1,13 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "errors" "fmt" "io" "os" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/store.go b/go/storage/store.go index e7a9dac94..80d8a30b8 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -1,10 +1,6 @@ package storage import ( - proto "code.google.com/p/goprotobuf/proto" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "fmt" @@ -12,6 +8,11 @@ import ( "math/rand" "strconv" "strings" + + proto "code.google.com/p/goprotobuf/proto" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/store_vacuum.go b/go/storage/store_vacuum.go index 3527e4f59..209e3b4b3 100644 --- a/go/storage/store_vacuum.go +++ b/go/storage/store_vacuum.go @@ -1,9 +1,10 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" "fmt" "strconv" + + "github.com/chrislusf/weed-fs/go/glog" ) func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) { diff --git a/go/storage/volume.go b/go/storage/volume.go index 60ee1a0c4..5b0a83605 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -4,12 +4,13 @@ import ( "bytes" "errors" "fmt" - "github.com/chrislusf/weed-fs/go/glog" "io" "os" "path" "sync" "time" + + "github.com/chrislusf/weed-fs/go/glog" ) type Volume struct { diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index a7e86b1c3..57e0deea9 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -1,9 +1,10 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" "fmt" "os" + + "github.com/chrislusf/weed-fs/go/glog" ) const ( diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go index b348434d2..7e026a61d 100644 --- a/go/storage/volume_vacuum.go +++ b/go/storage/volume_vacuum.go @@ -1,10 +1,11 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" "fmt" "os" "time" + + "github.com/chrislusf/weed-fs/go/glog" ) func (v *Volume) garbageLevel() float64 { diff --git a/go/storage/volume_version.go b/go/storage/volume_version.go index 9702ae904..2e9f58aa2 100644 --- a/go/storage/volume_version.go +++ b/go/storage/volume_version.go @@ -1,7 +1,5 @@ package storage -import () - type Version uint8 const ( diff --git a/go/tools/read_index.go b/go/tools/read_index.go index b99c5b6b8..1104dc348 100644 --- a/go/tools/read_index.go +++ b/go/tools/read_index.go @@ -1,11 +1,12 @@ package main import ( - "github.com/chrislusf/weed-fs/go/storage" "flag" "fmt" "log" "os" + + "github.com/chrislusf/weed-fs/go/storage" ) var ( diff --git a/go/topology/allocate_volume.go b/go/topology/allocate_volume.go index 6562e9ac5..a791b4c1c 100644 --- a/go/topology/allocate_volume.go +++ b/go/topology/allocate_volume.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" + + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) type AllocateVolumeResult struct { diff --git a/go/topology/data_center.go b/go/topology/data_center.go index ebd07803b..bcf2dfd31 100644 --- a/go/topology/data_center.go +++ b/go/topology/data_center.go @@ -1,7 +1,5 @@ package topology -import () - type DataCenter struct { NodeImpl } diff --git a/go/topology/data_node.go b/go/topology/data_node.go index c3b90470f..109bd037f 100644 --- a/go/topology/data_node.go +++ b/go/topology/data_node.go @@ -1,9 +1,10 @@ package topology import ( + "strconv" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/storage" - "strconv" ) type DataNode struct { diff --git a/go/topology/node.go b/go/topology/node.go index 54118802e..10955fa72 100644 --- a/go/topology/node.go +++ b/go/topology/node.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "errors" "math/rand" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) type NodeId string diff --git a/go/topology/store_replicate.go b/go/topology/store_replicate.go index 6ea019bd8..0c52f9d30 100644 --- a/go/topology/store_replicate.go +++ b/go/topology/store_replicate.go @@ -2,12 +2,13 @@ package topology import ( "bytes" + "net/http" + "strconv" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/operation" "github.com/chrislusf/weed-fs/go/storage" "github.com/chrislusf/weed-fs/go/util" - "net/http" - "strconv" ) func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.VolumeId, needle *storage.Needle, r *http.Request) (size uint32, errorStatus string) { diff --git a/go/topology/topology.go b/go/topology/topology.go index cfce0c9a8..eb64d336c 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -1,14 +1,15 @@ package topology import ( + "errors" + "io/ioutil" + "math/rand" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/operation" "github.com/chrislusf/weed-fs/go/sequence" "github.com/chrislusf/weed-fs/go/storage" - "errors" "github.com/goraft/raft" - "io/ioutil" - "math/rand" ) type Topology struct { diff --git a/go/topology/topology_event_handling.go b/go/topology/topology_event_handling.go index eb4491484..7e36568b6 100644 --- a/go/topology/topology_event_handling.go +++ b/go/topology/topology_event_handling.go @@ -1,10 +1,11 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "math/rand" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func (t *Topology) StartRefreshWritableVolumes(garbageThreshold string) { diff --git a/go/topology/topology_map.go b/go/topology/topology_map.go index d6400c988..af95c6536 100644 --- a/go/topology/topology_map.go +++ b/go/topology/topology_map.go @@ -1,7 +1,5 @@ package topology -import () - func (t *Topology) ToMap() interface{} { m := make(map[string]interface{}) m["Max"] = t.GetMaxVolumeCount() diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index 72846f20b..97a76026d 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -1,13 +1,14 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList, garbageThreshold string) bool { diff --git a/go/topology/volume_growth.go b/go/topology/volume_growth.go index 2859d3992..b1f241990 100644 --- a/go/topology/volume_growth.go +++ b/go/topology/volume_growth.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "fmt" "math/rand" "sync" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) /* diff --git a/go/topology/volume_growth_test.go b/go/topology/volume_growth_test.go index 5581c87ce..267b36042 100644 --- a/go/topology/volume_growth_test.go +++ b/go/topology/volume_growth_test.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/sequence" - "github.com/chrislusf/weed-fs/go/storage" "encoding/json" "fmt" "testing" + + "github.com/chrislusf/weed-fs/go/sequence" + "github.com/chrislusf/weed-fs/go/storage" ) var topologyLayout = ` diff --git a/go/topology/volume_layout.go b/go/topology/volume_layout.go index 7bb0cf7e3..de72bf895 100644 --- a/go/topology/volume_layout.go +++ b/go/topology/volume_layout.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "errors" "math/rand" "sync" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) // mapping from volume to its locations, inverted from server to volume diff --git a/go/topology/volume_location_list.go b/go/topology/volume_location_list.go index 176f469b9..011614013 100644 --- a/go/topology/volume_location_list.go +++ b/go/topology/volume_location_list.go @@ -1,7 +1,5 @@ package topology -import () - type VolumeLocationList struct { list []*DataNode } diff --git a/go/util/config.go b/go/util/config.go index 050fd0e64..4cf1d7c64 100644 --- a/go/util/config.go +++ b/go/util/config.go @@ -10,9 +10,10 @@ package util import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "encoding/json" "os" + + "github.com/chrislusf/weed-fs/go/glog" ) type Config struct { diff --git a/go/util/constants.go b/go/util/constants.go index db1ca38e5..c407533a8 100644 --- a/go/util/constants.go +++ b/go/util/constants.go @@ -1,7 +1,5 @@ package util -import () - const ( VERSION = "0.64" ) diff --git a/go/util/file_util.go b/go/util/file_util.go index 412d98458..30e24f001 100644 --- a/go/util/file_util.go +++ b/go/util/file_util.go @@ -2,9 +2,10 @@ package util import ( "bufio" - "github.com/chrislusf/weed-fs/go/glog" "errors" "os" + + "github.com/chrislusf/weed-fs/go/glog" ) func TestFolderWritable(folder string) (err error) { diff --git a/go/util/net_timeout.go b/go/util/net_timeout.go index eb80822b5..f274e4802 100644 --- a/go/util/net_timeout.go +++ b/go/util/net_timeout.go @@ -1,9 +1,10 @@ package util import ( - "github.com/chrislusf/weed-fs/go/stats" "net" "time" + + "github.com/chrislusf/weed-fs/go/stats" ) // Listener wraps a net.Listener, and gives a place to store the timeout diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index fec8472e5..8339913cd 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -2,9 +2,6 @@ package main import ( "bufio" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "math" @@ -16,6 +13,10 @@ import ( "strings" "sync" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) type BenchmarkOptions struct { diff --git a/go/weed/download.go b/go/weed/download.go index c30d17915..c782654f5 100644 --- a/go/weed/download.go +++ b/go/weed/download.go @@ -1,14 +1,15 @@ package main import ( - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "io/ioutil" "os" "path" "strings" + + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) var ( diff --git a/go/weed/export.go b/go/weed/export.go index 9f33d852f..c9cc0e3fe 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -4,14 +4,15 @@ import ( "archive/tar" "bytes" "fmt" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "os" "path" "strconv" "strings" "text/template" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/filer.go b/go/weed/filer.go index 7dbecb4d0..b3fc9c878 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -1,13 +1,14 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" "net/http" "os" "strconv" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" ) var ( diff --git a/go/weed/fix.go b/go/weed/fix.go index ad573875a..9c8026448 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -1,11 +1,12 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "os" "path" "strconv" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/master.go b/go/weed/master.go index f96ff4c08..f88964b6d 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -1,16 +1,17 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" - "github.com/gorilla/mux" "net/http" "os" "runtime" "strconv" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/gorilla/mux" ) func init() { diff --git a/go/weed/mount.go b/go/weed/mount.go index 0290e0f53..66e645387 100644 --- a/go/weed/mount.go +++ b/go/weed/mount.go @@ -1,7 +1,5 @@ package main -import () - type MountOptions struct { filer *string dir *string diff --git a/go/weed/mount_std.go b/go/weed/mount_std.go index e5fc0986c..808c6c563 100644 --- a/go/weed/mount_std.go +++ b/go/weed/mount_std.go @@ -3,15 +3,16 @@ package main import ( + "fmt" + "os" + "runtime" + "bazil.org/fuse" "bazil.org/fuse/fs" "github.com/chrislusf/weed-fs/go/filer" "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/storage" "github.com/chrislusf/weed-fs/go/util" - "fmt" - "os" - "runtime" ) func runMount(cmd *Command, args []string) bool { diff --git a/go/weed/server.go b/go/weed/server.go index 1d854d641..6042c22e7 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -1,10 +1,6 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" - "github.com/gorilla/mux" "net/http" "os" "runtime" @@ -13,6 +9,11 @@ import ( "strings" "sync" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/gorilla/mux" ) type ServerOptions struct { diff --git a/go/weed/shell.go b/go/weed/shell.go index c8043e0dd..f2c4990ea 100644 --- a/go/weed/shell.go +++ b/go/weed/shell.go @@ -2,9 +2,10 @@ package main import ( "bufio" - "github.com/chrislusf/weed-fs/go/glog" "fmt" "os" + + "github.com/chrislusf/weed-fs/go/glog" ) func init() { diff --git a/go/weed/signal_handling_notsupported.go b/go/weed/signal_handling_notsupported.go index ad4f37b0c..343cf7de2 100644 --- a/go/weed/signal_handling_notsupported.go +++ b/go/weed/signal_handling_notsupported.go @@ -2,7 +2,5 @@ package main -import () - func OnInterrupt(fn func()) { } diff --git a/go/weed/upload.go b/go/weed/upload.go index 4eae4d274..2d67c0bd9 100644 --- a/go/weed/upload.go +++ b/go/weed/upload.go @@ -1,11 +1,12 @@ package main import ( - "github.com/chrislusf/weed-fs/go/operation" "encoding/json" "fmt" "os" "path/filepath" + + "github.com/chrislusf/weed-fs/go/operation" ) var ( diff --git a/go/weed/version.go b/go/weed/version.go index 63441509e..8d3a6fed7 100644 --- a/go/weed/version.go +++ b/go/weed/version.go @@ -1,9 +1,10 @@ package main import ( - "github.com/chrislusf/weed-fs/go/util" "fmt" "runtime" + + "github.com/chrislusf/weed-fs/go/util" ) var cmdVersion = &Command{ diff --git a/go/weed/volume.go b/go/weed/volume.go index ce05fcdf3..212cb4b33 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -1,15 +1,16 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" "net/http" "os" "runtime" "strconv" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" ) func init() { diff --git a/go/weed/volume_test.go b/go/weed/volume_test.go index 764362a2b..ef00a8c7c 100644 --- a/go/weed/volume_test.go +++ b/go/weed/volume_test.go @@ -1,10 +1,11 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" "net/http" "testing" "time" + + "github.com/chrislusf/weed-fs/go/glog" ) func TestXYZ(t *testing.T) { diff --git a/go/weed/weed.go b/go/weed/weed.go index c1f5a72de..c304b7f35 100644 --- a/go/weed/weed.go +++ b/go/weed/weed.go @@ -1,7 +1,6 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" "flag" "fmt" "io" @@ -13,6 +12,8 @@ import ( "time" "unicode" "unicode/utf8" + + "github.com/chrislusf/weed-fs/go/glog" ) var IsDebug *bool diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 816107dc5..39b0830e3 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -2,11 +2,6 @@ package weed_server import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "fmt" "net" @@ -14,6 +9,12 @@ import ( "path/filepath" "strconv" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) var serverStats *stats.ServerStats diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 5ff0ed986..9d6f7c71b 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -1,10 +1,11 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/filer" - "github.com/chrislusf/weed-fs/go/glog" "net/http" "strconv" + + "github.com/chrislusf/weed-fs/go/filer" + "github.com/chrislusf/weed-fs/go/glog" ) type FilerServer struct { diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index e36e7c310..781d7c0fc 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -1,12 +1,8 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" - "github.com/syndtr/goleveldb/leveldb" "io" "io/ioutil" "math/rand" @@ -14,6 +10,11 @@ import ( "net/url" "strconv" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" + "github.com/syndtr/goleveldb/leveldb" ) func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/filer_server_handlers_admin.go b/go/weed/weed_server/filer_server_handlers_admin.go index ff52dff24..5ba12e0b8 100644 --- a/go/weed/weed_server/filer_server_handlers_admin.go +++ b/go/weed/weed_server/filer_server_handlers_admin.go @@ -1,8 +1,9 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" "net/http" + + "github.com/chrislusf/weed-fs/go/glog" ) /* diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go index 401f6cfdb..d000cb610 100644 --- a/go/weed/weed_server/master_server.go +++ b/go/weed/weed_server/master_server.go @@ -1,16 +1,17 @@ package weed_server import ( + "net/http" + "net/http/httputil" + "net/url" + "sync" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/sequence" "github.com/chrislusf/weed-fs/go/topology" "github.com/chrislusf/weed-fs/go/util" "github.com/goraft/raft" "github.com/gorilla/mux" - "net/http" - "net/http/httputil" - "net/url" - "sync" ) type MasterServer struct { diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index d7f1f4ce3..7a7a3b70d 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -1,12 +1,13 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" "net/http" "strconv" "strings" + + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" ) func (ms *MasterServer) lookupVolumeId(vids []string, collection string) (volumeLocations map[string]operation.LookupResult) { diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 1a2c6b8e0..d7124e567 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -1,18 +1,19 @@ package weed_server import ( - proto "code.google.com/p/goprotobuf/proto" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/topology" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "io/ioutil" "net/http" "strconv" "strings" + + proto "code.google.com/p/goprotobuf/proto" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/util" ) func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index e41867076..0b049ce27 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -2,19 +2,20 @@ package weed_server import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/topology" "encoding/json" "errors" "fmt" - "github.com/goraft/raft" - "github.com/gorilla/mux" "io/ioutil" "math/rand" "net/http" "net/url" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/topology" + "github.com/goraft/raft" + "github.com/gorilla/mux" ) type RaftServer struct { diff --git a/go/weed/weed_server/raft_server_handlers.go b/go/weed/weed_server/raft_server_handlers.go index 4d51c0767..b466d9afa 100644 --- a/go/weed/weed_server/raft_server_handlers.go +++ b/go/weed/weed_server/raft_server_handlers.go @@ -1,13 +1,14 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" "encoding/json" - "github.com/goraft/raft" "io/ioutil" "net/http" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/goraft/raft" ) // Handles incoming RAFT joins. diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index 2a9085f3b..0a65fd2f6 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -1,12 +1,13 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "math/rand" "net/http" "strconv" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) type VolumeServer struct { diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index ce14f6a87..83f614941 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -1,12 +1,6 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/images" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/topology" "io" "mime" "mime/multipart" @@ -14,6 +8,13 @@ import ( "strconv" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/images" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/topology" ) var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go index 1118c8017..caf4c3be8 100644 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ b/go/weed/weed_server/volume_server_handlers_admin.go @@ -1,11 +1,12 @@ package weed_server import ( + "net/http" + "path/filepath" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/stats" "github.com/chrislusf/weed-fs/go/util" - "net/http" - "path/filepath" ) func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/volume_server_handlers_vacuum.go b/go/weed/weed_server/volume_server_handlers_vacuum.go index b0600d799..f115e3b8b 100644 --- a/go/weed/weed_server/volume_server_handlers_vacuum.go +++ b/go/weed/weed_server/volume_server_handlers_vacuum.go @@ -1,8 +1,9 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" "net/http" + + "github.com/chrislusf/weed-fs/go/glog" ) func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) { From 7ce628bf09ed949f1686a581d782cab3664511fd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 27 Oct 2014 01:09:45 -0700 Subject: [PATCH 09/23] Clean raft configurations if "peers" option is set. --- go/weed/weed_server/raft_server.go | 46 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index 0b049ce27..12148db49 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -9,6 +9,8 @@ import ( "math/rand" "net/http" "net/url" + "os" + "path" "strings" "time" @@ -46,6 +48,13 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin transporter := raft.NewHTTPTransporter("/cluster", 0) transporter.Transport.MaxIdleConnsPerHost = 1024 + // Clear old cluster configurations if peers are set + if len(s.peers) > 0 { + os.RemoveAll(path.Join(s.dataDir, "conf")) + os.RemoveAll(path.Join(s.dataDir, "log")) + os.RemoveAll(path.Join(s.dataDir, "snapshot")) + } + s.raftServer, err = raft.NewServer(s.httpAddr, s.dataDir, transporter, nil, topo, "") if err != nil { glog.V(0).Infoln(err) @@ -53,35 +62,30 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin } transporter.Install(s.raftServer, s) s.raftServer.SetHeartbeatInterval(1 * time.Second) - s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 1150 * time.Millisecond) + s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 3450 * time.Millisecond) s.raftServer.Start() s.router.HandleFunc("/cluster/join", s.joinHandler).Methods("POST") s.router.HandleFunc("/cluster/status", s.statusHandler).Methods("GET") - // Join to leader if specified. if len(s.peers) > 0 { - if !s.raftServer.IsLogEmpty() { - glog.V(0).Infoln("Starting cluster with existing logs.") - } else { - glog.V(0).Infoln("Joining cluster:", strings.Join(s.peers, ",")) - time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) - firstJoinError := s.Join(s.peers) - if firstJoinError != nil { - glog.V(0).Infoln("No existing server found. Starting as leader in the new cluster.") - _, err := s.raftServer.Do(&raft.DefaultJoinCommand{ - Name: s.raftServer.Name(), - ConnectionString: "http://" + s.httpAddr, - }) - if err != nil { - glog.V(0).Infoln(err) - return nil - } + // Join to leader if specified. + glog.V(0).Infoln("Joining cluster:", strings.Join(s.peers, ",")) + time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) + firstJoinError := s.Join(s.peers) + if firstJoinError != nil { + glog.V(0).Infoln("No existing server found. Starting as leader in the new cluster.") + _, err := s.raftServer.Do(&raft.DefaultJoinCommand{ + Name: s.raftServer.Name(), + ConnectionString: "http://" + s.httpAddr, + }) + if err != nil { + glog.V(0).Infoln(err) + return nil } } - - // Initialize the server by joining itself. } else if s.raftServer.IsLogEmpty() { + // Initialize the server by joining itself. glog.V(0).Infoln("Initializing new cluster") _, err := s.raftServer.Do(&raft.DefaultJoinCommand{ @@ -95,7 +99,7 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin } } else { - glog.V(0).Infoln("Recovered from log") + glog.V(0).Infoln("Old conf,log,snapshot should have been removed.") } return s From b0e8f8ca2a63c7be9fe0c32b68e15dc47001a672 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 27 Oct 2014 01:10:33 -0700 Subject: [PATCH 10/23] Upgraded to 0.97 goclipse. --- .project | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.project b/.project index f5551728c..4dee05c73 100644 --- a/.project +++ b/.project @@ -5,11 +5,6 @@ - - com.googlecode.goclipse.goBuilder - - - goclipse.goNature From cd1b70caf7969a5789f285cdb0c23f9b690626ab Mon Sep 17 00:00:00 2001 From: Yanyi Wu Date: Sat, 1 Nov 2014 01:48:19 +0800 Subject: [PATCH 11/23] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1243727a2..06d81ebf0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM cydev/go -RUN go get code.google.com/p/weed-fs/go/weed +RUN go get github.com/chrislusf/weed-fs/go/weed EXPOSE 8080 EXPOSE 9333 VOLUME /data From 8af475300295adf9aadc6f57216112db1e6765cd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 28 Nov 2014 16:34:03 -0800 Subject: [PATCH 12/23] Write request id to first 8 bytes of a file, instead of whole file, for better write performance. --- go/util/constants.go | 2 +- go/weed/benchmark.go | 26 ++++++++++++++++---------- go/weed/server.go | 10 +++++----- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/go/util/constants.go b/go/util/constants.go index c407533a8..61f05898f 100644 --- a/go/util/constants.go +++ b/go/util/constants.go @@ -1,5 +1,5 @@ package util const ( - VERSION = "0.64" + VERSION = "0.66" ) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index 8339913cd..e5cf831c6 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -31,6 +31,7 @@ type BenchmarkOptions struct { sequentialRead *bool collection *string cpuprofile *string + maxCpu *int vid2server map[string]string //cache for vid locations } @@ -51,7 +52,8 @@ func init() { b.read = cmdBenchmark.Flag.Bool("read", true, "enable read") b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file") b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection") - b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "write cpu profile to file") + b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file") + b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") b.vid2server = make(map[string]string) } @@ -59,25 +61,25 @@ var cmdBenchmark = &Command{ UsageLine: "benchmark -server=localhost:9333 -c=10 -n=100000", Short: "benchmark on writing millions of files and read out", Long: `benchmark on an empty weed file system. - + Two tests during benchmark: 1) write lots of small files to the system 2) read the files out - + The file content is mostly zero, but no compression is done. - + You can choose to only benchmark read or write. During write, the list of uploaded file ids is stored in "-list" specified file. You can also use your own list of file ids to run read test. - + Write speed and read speed will be collected. The numbers are used to get a sense of the system. Usually your network or the hard drive is the real bottleneck. - + Another thing to watch is whether the volumes are evenly distributed to each volume server. Because the 7 more benchmark volumes are randomly distributed to servers with free slots, it's highly possible some servers have uneven amount of - benchmark volumes. To remedy this, you can use this to grow the benchmark volumes + benchmark volumes. To remedy this, you can use this to grow the benchmark volumes before starting the benchmark command: http://localhost:9333/vol/grow?collection=benchmark&count=5 @@ -100,6 +102,10 @@ func init() { func runbenchmark(cmd *Command, args []string) bool { fmt.Printf("This is Seaweed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) + if *b.maxCpu < 1 { + *b.maxCpu = runtime.NumCPU() + } + runtime.GOMAXPROCS(*b.maxCpu) if *b.cpuprofile != "" { f, err := os.Create(*b.cpuprofile) if err != nil { @@ -497,9 +503,9 @@ func (l *FakeReader) Read(p []byte) (n int, err error) { } else { n = len(p) } - for i := 0; i < n-8; i += 8 { - for s := uint(0); s < 8; s++ { - p[i] = byte(l.id >> (s * 8)) + if n >= 8 { + for i := 0; i < 8; i++ { + p[i] = byte(l.id >> uint(i*8)) } } l.size -= int64(n) diff --git a/go/weed/server.go b/go/weed/server.go index 6042c22e7..38ccdc137 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -32,17 +32,17 @@ func init() { var cmdServer = &Command{ UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name", Short: "start a server, including volume server, and automatically elect a master server", - Long: `start both a volume server to provide storage spaces + Long: `start both a volume server to provide storage spaces and a master server to provide volume=>location mapping service and sequence number of file ids - + This is provided as a convenient way to start both volume server and master server. The servers are exactly the same as starting them separately. So other volume servers can use this embedded master server also. - + Optionally, one filer server can be started. Logically, filer servers should not be in a cluster. They run with meta data on disk, not shared. So each filer server is different. - + `, } @@ -73,7 +73,7 @@ var ( ) func init() { - serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "write cpu profile to file") + serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file") filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server") filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection") filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") From 9d03cd20ba518e2aad6db6abe8f51ee9445eaacd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 28 Nov 2014 16:55:40 -0800 Subject: [PATCH 13/23] De-support go 1.2 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a6f8b8815..79cec6d8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: go go: - - 1.2 - 1.3 - release - tip From ca67ed69a1e80bd1166105fb8a08d8c5b4f9c83f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 28 Nov 2014 17:02:10 -0800 Subject: [PATCH 14/23] Change name to Seaweed. --- docs/benchmarks.rst | 19 +++++++++++++++++-- docs/directories.rst | 2 +- docs/failover.rst | 2 +- docs/gettingstarted.rst | 6 +++--- go/weed/fix.go | 2 +- 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst index e01e6e4ba..d0b71428f 100644 --- a/docs/benchmarks.rst +++ b/docs/benchmarks.rst @@ -1,7 +1,7 @@ Benchmarks ====================== -Do we really need the benchmark? People always use benchmark to compare systems. But benchmarks are misleading. The resources, e.g., CPU, disk, memory, network, all matter a lot. And with Weed File System, single node vs multiple nodes, benchmarking on one machine vs several multiple machines, all matter a lot. +Do we really need the benchmark? People always use benchmark to compare systems. But benchmarks are misleading. The resources, e.g., CPU, disk, memory, network, all matter a lot. And with Seaweed File System, single node vs multiple nodes, benchmarking on one machine vs several multiple machines, all matter a lot. Here is the steps on how to run benchmark if you really need some numbers. @@ -38,7 +38,22 @@ Many options are options are configurable. Please check the help content: Common Problems ############################### -The most common problem is "too many open files" error. This is because the test itself starts too many network connections on one single machine. In my local macbook, if I ran "random read" following writing right away, the error happens always. I have to run "weed benchmark -write=false" to run the reading test only. Also, changing the concurrency level to "-c=16" would also help. +The most common +I start weed servers in one console for simplicity. Better run servers on different consoles. + +For more realistic tests, please start them on different machines. + +.. code-block:: bash + + # prepare directories + mkdir 3 4 5 + # start 3 servers + ./weed server -dir=./3 -master.port=9333 -volume.port=8083 & + ./weed volume -dir=./4 -port=8084 & + ./weed volume -dir=./5 -port=8085 & + ./weed benchmark -server=localhost:9333 + + problem is "too many open files" error. This is because the test itself starts too many network connections on one single machine. In my local macbook, if I ran "random read" following writing right away, the error happens always. I have to run "weed benchmark -write=false" to run the reading test only. Also, changing the concurrency level to "-c=16" would also help. My own unscientific single machine results ################################################### diff --git a/docs/directories.rst b/docs/directories.rst index 787c7a2ab..aa14da4f7 100644 --- a/docs/directories.rst +++ b/docs/directories.rst @@ -1,7 +1,7 @@ Directories and files =========================== -When talking about file systems, many people would assume directories, list files under a directory, etc. These are expected if we want to hook up Weed File System with linux by FUSE, or with Hadoop, etc. +When talking about file systems, many people would assume directories, list files under a directory, etc. These are expected if we want to hook up Seaweed File System with linux by FUSE, or with Hadoop, etc. Sample usage ##################### diff --git a/docs/failover.rst b/docs/failover.rst index f244918d8..506a33e3b 100644 --- a/docs/failover.rst +++ b/docs/failover.rst @@ -7,7 +7,7 @@ Introduction Some user will ask for no single point of failure. Although google runs its file system with a single master for years, no SPOF seems becoming a criteria for architects to pick solutions. -Luckily, it's not too difficult to enable Weed File System with failover master servers. +Luckily, it's not too difficult to enable Seaweed File System with failover master servers. Cheat Sheet: Startup multiple servers ######################################## diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index 19eb130a7..ad28319fe 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -1,6 +1,6 @@ Getting started =================================== -Installing Weed-Fs +Installing Seaweed-FS ################################### Download a proper version from `Seaweed-FS download page `_. @@ -57,7 +57,7 @@ Actually, forget about previous commands. You can setup one master server and on # use "weed server -h" to find out more ./weed server -master.port=9333 -volume.port=8080 -dir="./data" -Testing Weed-Fs +Testing Seaweed-FS ################################### With the master and volume server up, now what? Let's pump in a lot of files into the system! @@ -77,7 +77,7 @@ Then, you can simply check "du -m -s /some/big/folder" to see the actual disk us Now you can use your tools to hit weed-fs as hard as you can. -Using Weed-Fs in docker +Using Seaweed-FS in docker #################################### You can use image "cydev/weed" or build your own with `dockerfile `_ in the root of repo. diff --git a/go/weed/fix.go b/go/weed/fix.go index 9c8026448..e66075ed2 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -17,7 +17,7 @@ func init() { var cmdFix = &Command{ UsageLine: "fix -dir=/tmp -volumeId=234", Short: "run weed tool fix on index file if corrupted", - Long: `Fix runs the WeedFS fix command to re-create the index .idx file. + Long: `Fix runs the SeeweedFS fix command to re-create the index .idx file. `, } From 6c5a3d3dbfc3e801c962bd650cfd4455483cc63c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 30 Nov 2014 21:55:53 -0800 Subject: [PATCH 15/23] Increase performance by reusing []byte, reducing GC. --- go/weed/benchmark.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index e5cf831c6..f88d0d6a9 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -33,10 +33,12 @@ type BenchmarkOptions struct { cpuprofile *string maxCpu *int vid2server map[string]string //cache for vid locations + } var ( - b BenchmarkOptions + b BenchmarkOptions + sharedBytes []byte ) func init() { @@ -55,6 +57,7 @@ func init() { b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file") b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") b.vid2server = make(map[string]string) + sharedBytes = make([]byte, 1024) } var cmdBenchmark = &Command{ @@ -512,6 +515,23 @@ func (l *FakeReader) Read(p []byte) (n int, err error) { return } +func (l *FakeReader) WriteTo(w io.Writer) (n int64, err error) { + size := int(l.size) + bufferSize := len(sharedBytes) + for size > 0 { + tempBuffer := sharedBytes + if size < bufferSize { + tempBuffer = sharedBytes[0:size] + } + count, e := w.Write(tempBuffer) + if e != nil { + return int64(size), e + } + size -= count + } + return l.size, nil +} + func Readln(r *bufio.Reader) ([]byte, error) { var ( isPrefix bool = true @@ -524,3 +544,4 @@ func Readln(r *bufio.Reader) ([]byte, error) { } return ln, err } + From 74b76a2c41694a74920230afe5fa2559eeb885b3 Mon Sep 17 00:00:00 2001 From: Shaoshan Liu Date: Wed, 3 Dec 2014 21:04:02 -0800 Subject: [PATCH 16/23] add instructions for users who are not familiar with golang --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index ab07683dd..b22d07569 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,31 @@ Seaweed File System go get github.com/chrislusf/weed-fs/go/weed ``` +## Installation guide for users who are not familiar with golang + +step 1: install go on your machine and setup the environment by following the instructions from the following link: + +https://golang.org/doc/install + +make sure you set up your $GOPATH + + +step 2: also you may need to install Mercurial by following the instructions below + +http://mercurial.selenic.com/downloads + + +step 3: download, compile, and install the project by executing the following command + +go get github.com/chrislusf/weed-fs/go/weed + +once this is down, you should see the executable "weed" under $GOPATH/bin + +step 4: after you modify your code locally, you could start a local build by calling "go build" under $GOPATH/src/github.com/chrislusf/weed-fs/go/weed + + + + ## Reference For pre-compiled releases, From ed7b00bf0207749cb555d7d77bc35ddf5849ec37 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Wed, 3 Dec 2014 21:37:00 -0800 Subject: [PATCH 17/23] Update README.md --- README.md | 67 ++++++++++++++++++++++--------------------------------- 1 file changed, 27 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index b22d07569..95d2a5ade 100644 --- a/README.md +++ b/README.md @@ -5,41 +5,6 @@ Seaweed File System [![GoDoc](https://godoc.org/github.com/chrislusf/weed-fs/go?status.svg)](https://godoc.org/github.com/chrislusf/weed-fs/go) [![RTD](https://readthedocs.org/projects/weed-fs/badge/?version=latest)](http://weed-fs.readthedocs.org/en/latest/) -## Usage - -``` -go get github.com/chrislusf/weed-fs/go/weed -``` - -## Installation guide for users who are not familiar with golang - -step 1: install go on your machine and setup the environment by following the instructions from the following link: - -https://golang.org/doc/install - -make sure you set up your $GOPATH - - -step 2: also you may need to install Mercurial by following the instructions below - -http://mercurial.selenic.com/downloads - - -step 3: download, compile, and install the project by executing the following command - -go get github.com/chrislusf/weed-fs/go/weed - -once this is down, you should see the executable "weed" under $GOPATH/bin - -step 4: after you modify your code locally, you could start a local build by calling "go build" under $GOPATH/src/github.com/chrislusf/weed-fs/go/weed - - - - -## Reference - -For pre-compiled releases, - https://bintray.com/chrislusf/Weed-FS/seaweed ## Introduction @@ -271,12 +236,34 @@ More tools and documentation, on how to maintain and scale the system. For examp This is a super exciting project! And I need helpers! -## Contributions ## -To make contributions easier, I have mirrored a repo in github.com -``` - https://github.com/chrislusf/weed-fs.git -``` +## Installation guide for users who are not familiar with golang + +step 1: install go on your machine and setup the environment by following the instructions from the following link: + +https://golang.org/doc/install + +make sure you set up your $GOPATH + + +step 2: also you may need to install Mercurial by following the instructions below + +http://mercurial.selenic.com/downloads + + +step 3: download, compile, and install the project by executing the following command + +go get github.com/chrislusf/weed-fs/go/weed + +once this is done, you should see the executable "weed" under $GOPATH/bin + +step 4: after you modify your code locally, you could start a local build by calling "go install" under $GOPATH/src/github.com/chrislusf/weed-fs/go/weed + +## Reference + +For pre-compiled releases, + https://bintray.com/chrislusf/Weed-FS/seaweed + ## Disk Related topics ## ### Hard Drive Performance ### From 89fd1e4b6e586412f0692ac479b566a323296595 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Thu, 4 Dec 2014 18:30:44 -0800 Subject: [PATCH 18/23] Add more thread safe counters. Tighten thread synchronization. --- go/weed/benchmark.go | 79 +++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index f88d0d6a9..47b400fcd 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -132,12 +132,12 @@ func runbenchmark(cmd *Command, args []string) bool { func bench_write() { fileIdLineChan := make(chan string) finishChan := make(chan bool) - writeStats = newStats() + writeStats = newStats(*b.concurrency) idChan := make(chan int) - wait.Add(*b.concurrency) go writeFileIds(*b.idListFile, fileIdLineChan, finishChan) for i := 0; i < *b.concurrency; i++ { - go writeFiles(idChan, fileIdLineChan, writeStats) + wait.Add(1) + go writeFiles(idChan, fileIdLineChan, &writeStats.localStats[i]) } writeStats.start = time.Now() writeStats.total = *b.numberOfFiles @@ -150,7 +150,6 @@ func bench_write() { writeStats.end = time.Now() wait.Add(1) finishChan <- true - finishChan <- true close(finishChan) wait.Wait() writeStats.printStats() @@ -159,14 +158,14 @@ func bench_write() { func bench_read() { fileIdLineChan := make(chan string) finishChan := make(chan bool) - readStats = newStats() - wait.Add(*b.concurrency) + readStats = newStats(*b.concurrency) go readFileIds(*b.idListFile, fileIdLineChan) readStats.start = time.Now() readStats.total = *b.numberOfFiles go readStats.checkProgress("Randomly Reading Benchmark", finishChan) for i := 0; i < *b.concurrency; i++ { - go readFiles(fileIdLineChan, readStats) + wait.Add(1) + go readFiles(fileIdLineChan, &readStats.localStats[i]) } wait.Wait() finishChan <- true @@ -180,12 +179,13 @@ type delayedFile struct { fp *operation.FilePart } -func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) { +func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { + defer wait.Done() delayedDeleteChan := make(chan *delayedFile, 100) var waitForDeletions sync.WaitGroup for i := 0; i < 7; i++ { + waitForDeletions.Add(1) go func() { - waitForDeletions.Add(1) for df := range delayedDeleteChan { if df == nil { break @@ -228,6 +228,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) { s.transferred += fileSize } else { s.failed++ + fmt.Printf("Failed to write with error:%v\n", err) } writeStats.addSample(time.Now().Sub(start)) <-serverLimitChan[fp.Server] @@ -244,10 +245,10 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) { } close(delayedDeleteChan) waitForDeletions.Wait() - wait.Done() } -func readFiles(fileIdLineChan chan string, s *stats) { +func readFiles(fileIdLineChan chan string, s *stat) { + defer wait.Done() serverLimitChan := make(map[string]chan bool) masterLimitChan := make(chan bool, 1) for { @@ -288,7 +289,7 @@ func readFiles(fileIdLineChan chan string, s *stats) { readStats.addSample(time.Now().Sub(start)) } else { s.failed++ - println("!!!! Failed to read from ", url, " !!!!!") + fmt.Printf("Failed to read %s error:%v\n", url, err) } <-serverLimitChan[server] } else { @@ -299,7 +300,6 @@ func readFiles(fileIdLineChan chan string, s *stats) { break } } - wait.Done() } func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan bool) { @@ -363,20 +363,28 @@ const ( // An efficient statics collecting and rendering type stats struct { - data []int - overflow []int + data []int + overflow []int + localStats []stat + start time.Time + end time.Time + total int +} +type stat struct { completed int failed int total int transferred int64 - start time.Time - end time.Time } var percentages = []int{50, 66, 75, 80, 90, 95, 98, 99, 100} -func newStats() *stats { - return &stats{data: make([]int, benchResolution), overflow: make([]int, 0)} +func newStats(n int) *stats { + return &stats{ + data: make([]int, benchResolution), + overflow: make([]int, 0), + localStats: make([]stat, n), + } } func (s *stats) addSample(d time.Duration) { @@ -399,26 +407,38 @@ func (s *stats) checkProgress(testName string, finishChan chan bool) { case <-finishChan: return case t := <-ticker: - completed, transferred, taken := s.completed-lastCompleted, s.transferred-lastTransferred, t.Sub(lastTime) + completed, transferred, taken, total := 0, int64(0), t.Sub(lastTime), s.total + for _, localStat := range s.localStats { + completed += localStat.completed + transferred += localStat.transferred + total += localStat.total + } fmt.Printf("Completed %d of %d requests, %3.1f%% %3.1f/s %3.1fMB/s\n", - s.completed, s.total, float64(s.completed)*100/float64(s.total), - float64(completed)*float64(int64(time.Second))/float64(int64(taken)), - float64(transferred)*float64(int64(time.Second))/float64(int64(taken))/float64(1024*1024), + completed, total, float64(completed)*100/float64(total), + float64(completed-lastCompleted)*float64(int64(time.Second))/float64(int64(taken)), + float64(transferred-lastTransferred)*float64(int64(time.Second))/float64(int64(taken))/float64(1024*1024), ) - lastCompleted, lastTransferred, lastTime = s.completed, s.transferred, t + lastCompleted, lastTransferred, lastTime = completed, transferred, t } } } func (s *stats) printStats() { + completed, failed, transferred, total := 0, 0, int64(0), s.total + for _, localStat := range s.localStats { + completed += localStat.completed + failed += localStat.failed + transferred += localStat.transferred + total += localStat.total + } timeTaken := float64(int64(s.end.Sub(s.start))) / 1000000000 fmt.Printf("\nConcurrency Level: %d\n", *b.concurrency) fmt.Printf("Time taken for tests: %.3f seconds\n", timeTaken) - fmt.Printf("Complete requests: %d\n", s.completed) - fmt.Printf("Failed requests: %d\n", s.failed) - fmt.Printf("Total transferred: %d bytes\n", s.transferred) - fmt.Printf("Requests per second: %.2f [#/sec]\n", float64(s.completed)/timeTaken) - fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", float64(s.transferred)/1024/timeTaken) + fmt.Printf("Complete requests: %d\n", completed) + fmt.Printf("Failed requests: %d\n", failed) + fmt.Printf("Total transferred: %d bytes\n", transferred) + fmt.Printf("Requests per second: %.2f [#/sec]\n", float64(completed)/timeTaken) + fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", float64(transferred)/1024/timeTaken) n, sum := 0, 0 min, max := 10000000, 0 for i := 0; i < len(s.data); i++ { @@ -544,4 +564,3 @@ func Readln(r *bufio.Reader) ([]byte, error) { } return ln, err } - From 7a6394378cfd3a09bb273211e9bad1ec19e01e84 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Thu, 4 Dec 2014 21:22:09 -0800 Subject: [PATCH 19/23] Remove a volume server concurrent connection limit. --- go/weed/benchmark.go | 150 ++++++++++++++++++------------------------- 1 file changed, 62 insertions(+), 88 deletions(-) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index 47b400fcd..f4f0b1874 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -93,16 +93,11 @@ var cmdBenchmark = &Command{ } var ( - wait sync.WaitGroup - writeStats *stats - readStats *stats - serverLimitChan map[string]chan bool + wait sync.WaitGroup + writeStats *stats + readStats *stats ) -func init() { - serverLimitChan = make(map[string]chan bool) -} - func runbenchmark(cmd *Command, args []string) bool { fmt.Printf("This is Seaweed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) if *b.maxCpu < 1 { @@ -148,10 +143,11 @@ func bench_write() { close(idChan) wait.Wait() writeStats.end = time.Now() - wait.Add(1) + wait.Add(2) + finishChan <- true finishChan <- true - close(finishChan) wait.Wait() + close(finishChan) writeStats.printStats() } @@ -168,7 +164,9 @@ func bench_read() { go readFiles(fileIdLineChan, &readStats.localStats[i]) } wait.Wait() + wait.Add(1) finishChan <- true + wait.Wait() close(finishChan) readStats.end = time.Now() readStats.printStats() @@ -186,61 +184,46 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { for i := 0; i < 7; i++ { waitForDeletions.Add(1) go func() { + defer waitForDeletions.Done() for df := range delayedDeleteChan { - if df == nil { - break - } if df.enterTime.After(time.Now()) { time.Sleep(df.enterTime.Sub(time.Now())) } - fp := df.fp - serverLimitChan[fp.Server] <- true - if e := util.Delete("http://" + fp.Server + "/" + fp.Fid); e == nil { + if e := util.Delete("http://" + df.fp.Server + "/" + df.fp.Fid); e == nil { s.completed++ } else { s.failed++ } - <-serverLimitChan[fp.Server] } - waitForDeletions.Done() }() } - for { - if id, ok := <-idChan; ok { - start := time.Now() - fileSize := int64(*b.fileSize + rand.Intn(64)) - fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize} - if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil { - fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection - if _, ok := serverLimitChan[fp.Server]; !ok { - serverLimitChan[fp.Server] = make(chan bool, 7) - } - serverLimitChan[fp.Server] <- true - if _, err := fp.Upload(0, *b.server); err == nil { - if rand.Intn(100) < *b.deletePercentage { - s.total++ - delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp} - } else { - fileIdLineChan <- fp.Fid - } - s.completed++ - s.transferred += fileSize + for id := range idChan { + start := time.Now() + fileSize := int64(*b.fileSize + rand.Intn(64)) + fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize} + if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil { + fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection + if _, err := fp.Upload(0, *b.server); err == nil { + if rand.Intn(100) < *b.deletePercentage { + s.total++ + delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp} } else { - s.failed++ - fmt.Printf("Failed to write with error:%v\n", err) - } - writeStats.addSample(time.Now().Sub(start)) - <-serverLimitChan[fp.Server] - if *cmdBenchmark.IsDebug { - fmt.Printf("writing %d file %s\n", id, fp.Fid) + fileIdLineChan <- fp.Fid } + s.completed++ + s.transferred += fileSize } else { s.failed++ - println("writing file error:", err.Error()) + fmt.Printf("Failed to write with error:%v\n", err) + } + writeStats.addSample(time.Now().Sub(start)) + if *cmdBenchmark.IsDebug { + fmt.Printf("writing %d file %s\n", id, fp.Fid) } } else { - break + s.failed++ + println("writing file error:", err.Error()) } } close(delayedDeleteChan) @@ -249,55 +232,45 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { func readFiles(fileIdLineChan chan string, s *stat) { defer wait.Done() - serverLimitChan := make(map[string]chan bool) masterLimitChan := make(chan bool, 1) - for { - if fid, ok := <-fileIdLineChan; ok { - if len(fid) == 0 { - continue - } - if fid[0] == '#' { - continue - } - if *cmdBenchmark.IsDebug { - fmt.Printf("reading file %s\n", fid) - } - parts := strings.SplitN(fid, ",", 2) - vid := parts[0] - start := time.Now() - if server, ok := b.vid2server[vid]; !ok { - masterLimitChan <- true - if _, now_ok := b.vid2server[vid]; !now_ok { - if ret, err := operation.Lookup(*b.server, vid); err == nil { - if len(ret.Locations) > 0 { - server = ret.Locations[0].PublicUrl - b.vid2server[vid] = server - } + for fid := range fileIdLineChan { + if len(fid) == 0 { + continue + } + if fid[0] == '#' { + continue + } + if *cmdBenchmark.IsDebug { + fmt.Printf("reading file %s\n", fid) + } + parts := strings.SplitN(fid, ",", 2) + vid := parts[0] + start := time.Now() + if server, ok := b.vid2server[vid]; !ok { + masterLimitChan <- true + if _, now_ok := b.vid2server[vid]; !now_ok { + if ret, err := operation.Lookup(*b.server, vid); err == nil { + if len(ret.Locations) > 0 { + server = ret.Locations[0].PublicUrl + b.vid2server[vid] = server } } - <-masterLimitChan } - if server, ok := b.vid2server[vid]; ok { - if _, ok := serverLimitChan[server]; !ok { - serverLimitChan[server] = make(chan bool, 7) - } - serverLimitChan[server] <- true - url := "http://" + server + "/" + fid - if bytesRead, err := util.Get(url); err == nil { - s.completed++ - s.transferred += int64(len(bytesRead)) - readStats.addSample(time.Now().Sub(start)) - } else { - s.failed++ - fmt.Printf("Failed to read %s error:%v\n", url, err) - } - <-serverLimitChan[server] + <-masterLimitChan + } + if server, ok := b.vid2server[vid]; ok { + url := "http://" + server + "/" + fid + if bytesRead, err := util.Get(url); err == nil { + s.completed++ + s.transferred += int64(len(bytesRead)) + readStats.addSample(time.Now().Sub(start)) } else { s.failed++ - println("!!!! volume id ", vid, " location not found!!!!!") + fmt.Printf("Failed to read %s error:%v\n", url, err) } } else { - break + s.failed++ + println("!!!! volume id ", vid, " location not found!!!!!") } } } @@ -405,6 +378,7 @@ func (s *stats) checkProgress(testName string, finishChan chan bool) { for { select { case <-finishChan: + wait.Done() return case t := <-ticker: completed, transferred, taken, total := 0, int64(0), t.Sub(lastTime), s.total From 482e3fb97325ae073f19b8293bb25d210beed721 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 5 Dec 2014 12:00:13 -0800 Subject: [PATCH 20/23] Improve Benchmark tool's performance. --- go/util/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/util/constants.go b/go/util/constants.go index 61f05898f..d677ba44f 100644 --- a/go/util/constants.go +++ b/go/util/constants.go @@ -1,5 +1,5 @@ package util const ( - VERSION = "0.66" + VERSION = "0.67" ) From ba972694c730429889c696bd9853a38843f64f65 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Mon, 8 Dec 2014 20:27:26 -0800 Subject: [PATCH 21/23] Add filer option to redirect instead of proxying to volume server on file GET requests. --- go/weed/filer.go | 8 +++++-- go/weed/server.go | 6 +++++- go/weed/weed_server/filer_server.go | 22 +++++++++++++------- go/weed/weed_server/filer_server_handlers.go | 12 +++++++++-- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/go/weed/filer.go b/go/weed/filer.go index b3fc9c878..5ae3a3496 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -21,6 +21,7 @@ type FilerOptions struct { collection *string defaultReplicaPlacement *string dir *string + redirectOnRead *bool } func init() { @@ -29,7 +30,8 @@ func init() { f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection") f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port") f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data") - f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "Default replication type if not specified.") + f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified") + f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") } var cmdFiler = &Command{ @@ -60,7 +62,9 @@ func runFiler(cmd *Command, args []string) bool { } r := http.NewServeMux() - _, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection) + _, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection, + *f.defaultReplicaPlacement, *f.redirectOnRead, + ) if nfs_err != nil { glog.Fatalf(nfs_err.Error()) } diff --git a/go/weed/server.go b/go/weed/server.go index 38ccdc137..22222d699 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -79,6 +79,8 @@ func init() { filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified") filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") + filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") + } func runServer(cmd *Command, args []string) bool { @@ -150,7 +152,9 @@ func runServer(cmd *Command, args []string) bool { if *isStartingFiler { go func() { r := http.NewServeMux() - _, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection) + _, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, + *filerOptions.defaultReplicaPlacement, *filerOptions.redirectOnRead, + ) if nfs_err != nil { glog.Fatalf(nfs_err.Error()) } diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 9d6f7c71b..0bda58d06 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -9,17 +9,23 @@ import ( ) type FilerServer struct { - port string - master string - collection string - filer filer.Filer + port string + master string + collection string + defaultReplication string + redirectOnRead bool + filer filer.Filer } -func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string) (fs *FilerServer, err error) { +func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string, + replication string, redirectOnRead bool, +) (fs *FilerServer, err error) { fs = &FilerServer{ - master: master, - collection: collection, - port: ":" + strconv.Itoa(port), + master: master, + collection: collection, + defaultReplication: replication, + redirectOnRead: redirectOnRead, + port: ":" + strconv.Itoa(port), } if fs.filer, err = filer.NewFilerEmbedded(master, dir); err != nil { diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 781d7c0fc..9d8481ac9 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -81,7 +81,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, return } urlLocation := lookup.Locations[rand.Intn(len(lookup.Locations))].PublicUrl - u, _ := url.Parse("http://" + urlLocation + "/" + fileId) + urlString := "http://" + urlLocation + "/" + fileId + if fs.redirectOnRead { + + } + u, _ := url.Parse(urlString) request := &http.Request{ Method: r.Method, URL: u, @@ -110,7 +114,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() - assignResult, ae := operation.Assign(fs.master, 1, query.Get("replication"), fs.collection, query.Get("ttl")) + replication := query.Get("replication") + if replication == "" { + replication = fs.defaultReplication + } + assignResult, ae := operation.Assign(fs.master, 1, replication, fs.collection, query.Get("ttl")) if ae != nil { glog.V(0).Infoln("failing to assign a file id", ae.Error()) writeJsonError(w, r, ae) From 52180f386b044af7a6daba3e33ff04b5e9da25ba Mon Sep 17 00:00:00 2001 From: chrislusf Date: Mon, 8 Dec 2014 20:29:25 -0800 Subject: [PATCH 22/23] Add read-write lock to guard topology changes on new collections and ttls. --- go/topology/collection.go | 21 ++++++++++--------- go/topology/topology.go | 29 +++++++++++++------------- go/topology/topology_map.go | 7 ++++--- go/topology/topology_vacuum.go | 14 +++++++------ go/util/concurrent_read_map.go | 37 ++++++++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 34 deletions(-) create mode 100644 go/util/concurrent_read_map.go diff --git a/go/topology/collection.go b/go/topology/collection.go index 506f43fbf..4b47ae88a 100644 --- a/go/topology/collection.go +++ b/go/topology/collection.go @@ -2,17 +2,18 @@ package topology import ( "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) type Collection struct { Name string volumeSizeLimit uint64 - storageType2VolumeLayout map[string]*VolumeLayout + storageType2VolumeLayout *util.ConcurrentReadMap } func NewCollection(name string, volumeSizeLimit uint64) *Collection { c := &Collection{Name: name, volumeSizeLimit: volumeSizeLimit} - c.storageType2VolumeLayout = make(map[string]*VolumeLayout) + c.storageType2VolumeLayout = util.NewConcurrentReadMap() return c } @@ -21,16 +22,16 @@ func (c *Collection) GetOrCreateVolumeLayout(rp *storage.ReplicaPlacement, ttl * if ttl != nil { keyString += ttl.String() } - if c.storageType2VolumeLayout[keyString] == nil { - c.storageType2VolumeLayout[keyString] = NewVolumeLayout(rp, ttl, c.volumeSizeLimit) - } - return c.storageType2VolumeLayout[keyString] + vl := c.storageType2VolumeLayout.Get(keyString, func() interface{} { + return NewVolumeLayout(rp, ttl, c.volumeSizeLimit) + }) + return vl.(*VolumeLayout) } func (c *Collection) Lookup(vid storage.VolumeId) []*DataNode { - for _, vl := range c.storageType2VolumeLayout { + for _, vl := range c.storageType2VolumeLayout.Items { if vl != nil { - if list := vl.Lookup(vid); list != nil { + if list := vl.(*VolumeLayout).Lookup(vid); list != nil { return list } } @@ -39,9 +40,9 @@ func (c *Collection) Lookup(vid storage.VolumeId) []*DataNode { } func (c *Collection) ListVolumeServers() (nodes []*DataNode) { - for _, vl := range c.storageType2VolumeLayout { + for _, vl := range c.storageType2VolumeLayout.Items { if vl != nil { - if list := vl.ListVolumeServers(); list != nil { + if list := vl.(*VolumeLayout).ListVolumeServers(); list != nil { nodes = append(nodes, list...) } } diff --git a/go/topology/topology.go b/go/topology/topology.go index eb64d336c..c2073ed2f 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -9,13 +9,14 @@ import ( "github.com/chrislusf/weed-fs/go/operation" "github.com/chrislusf/weed-fs/go/sequence" "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" "github.com/goraft/raft" ) type Topology struct { NodeImpl - collectionMap map[string]*Collection + collectionMap *util.ConcurrentReadMap pulse int64 @@ -38,7 +39,7 @@ func NewTopology(id string, confFile string, seq sequence.Sequencer, volumeSizeL t.nodeType = "Topology" t.NodeImpl.value = t t.children = make(map[NodeId]Node) - t.collectionMap = make(map[string]*Collection) + t.collectionMap = util.NewConcurrentReadMap() t.pulse = int64(pulse) t.volumeSizeLimit = volumeSizeLimit @@ -90,14 +91,14 @@ func (t *Topology) loadConfiguration(configurationFile string) error { func (t *Topology) Lookup(collection string, vid storage.VolumeId) []*DataNode { //maybe an issue if lots of collections? if collection == "" { - for _, c := range t.collectionMap { - if list := c.Lookup(vid); list != nil { + for _, c := range t.collectionMap.Items { + if list := c.(*Collection).Lookup(vid); list != nil { return list } } } else { - if c, ok := t.collectionMap[collection]; ok { - return c.Lookup(vid) + if c, ok := t.collectionMap.Items[collection]; ok { + return c.(*Collection).Lookup(vid) } } return nil @@ -125,20 +126,18 @@ func (t *Topology) PickForWrite(count int, option *VolumeGrowOption) (string, in } func (t *Topology) GetVolumeLayout(collectionName string, rp *storage.ReplicaPlacement, ttl *storage.TTL) *VolumeLayout { - _, ok := t.collectionMap[collectionName] - if !ok { - t.collectionMap[collectionName] = NewCollection(collectionName, t.volumeSizeLimit) - } - return t.collectionMap[collectionName].GetOrCreateVolumeLayout(rp, ttl) + return t.collectionMap.Get(collectionName, func() interface{} { + return NewCollection(collectionName, t.volumeSizeLimit) + }).(*Collection).GetOrCreateVolumeLayout(rp, ttl) } -func (t *Topology) GetCollection(collectionName string) (collection *Collection, ok bool) { - collection, ok = t.collectionMap[collectionName] - return +func (t *Topology) GetCollection(collectionName string) (*Collection, bool) { + c, hasCollection := t.collectionMap.Items[collectionName] + return c.(*Collection), hasCollection } func (t *Topology) DeleteCollection(collectionName string) { - delete(t.collectionMap, collectionName) + delete(t.collectionMap.Items, collectionName) } func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { diff --git a/go/topology/topology_map.go b/go/topology/topology_map.go index af95c6536..6a1423ca8 100644 --- a/go/topology/topology_map.go +++ b/go/topology/topology_map.go @@ -11,10 +11,11 @@ func (t *Topology) ToMap() interface{} { } m["DataCenters"] = dcs var layouts []interface{} - for _, c := range t.collectionMap { - for _, layout := range c.storageType2VolumeLayout { + for _, col := range t.collectionMap.Items { + c := col.(*Collection) + for _, layout := range c.storageType2VolumeLayout.Items { if layout != nil { - tmp := layout.ToMap() + tmp := layout.(*VolumeLayout).ToMap() tmp["collection"] = c.Name layouts = append(layouts, tmp) } diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index 97a76026d..d6fa2213e 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -80,13 +80,15 @@ func batchVacuumVolumeCommit(vl *VolumeLayout, vid storage.VolumeId, locationlis return isCommitSuccess } func (t *Topology) Vacuum(garbageThreshold string) int { - for _, c := range t.collectionMap { - for _, vl := range c.storageType2VolumeLayout { + for _, col := range t.collectionMap.Items { + c := col.(*Collection) + for _, vl := range c.storageType2VolumeLayout.Items { if vl != nil { - for vid, locationlist := range vl.vid2location { - if batchVacuumVolumeCheck(vl, vid, locationlist, garbageThreshold) { - if batchVacuumVolumeCompact(vl, vid, locationlist) { - batchVacuumVolumeCommit(vl, vid, locationlist) + volumeLayout := vl.(*VolumeLayout) + for vid, locationlist := range volumeLayout.vid2location { + if batchVacuumVolumeCheck(volumeLayout, vid, locationlist, garbageThreshold) { + if batchVacuumVolumeCompact(volumeLayout, vid, locationlist) { + batchVacuumVolumeCommit(volumeLayout, vid, locationlist) } } } diff --git a/go/util/concurrent_read_map.go b/go/util/concurrent_read_map.go new file mode 100644 index 000000000..d16fdbcaf --- /dev/null +++ b/go/util/concurrent_read_map.go @@ -0,0 +1,37 @@ +package util + +import "sync" + +// A mostly for read map, which can thread-safely +// initialize the map entries. +type ConcurrentReadMap struct { + rmutex sync.RWMutex + mutex sync.Mutex + Items map[string]interface{} +} + +func NewConcurrentReadMap() *ConcurrentReadMap { + return &ConcurrentReadMap{Items: make(map[string]interface{})} +} + +func (m *ConcurrentReadMap) initMapEntry(key string, newEntry func() interface{}) (value interface{}) { + m.mutex.Lock() + defer m.mutex.Unlock() + if value, ok := m.Items[key]; ok { + return value + } + value = newEntry() + m.Items[key] = value + return value +} + +func (m *ConcurrentReadMap) Get(key string, newEntry func() interface{}) interface{} { + m.rmutex.RLock() + if value, ok := m.Items[key]; ok { + m.rmutex.RUnlock() + return value + } else { + m.rmutex.RUnlock() + return m.initMapEntry(key, newEntry) + } +} From e431d4121e8da8d7fc243b29b780c2cd535a4210 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Mon, 8 Dec 2014 20:34:27 -0800 Subject: [PATCH 23/23] Add optional http redirect for filer GET requests. --- go/weed/weed_server/filer_server_handlers.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 9d8481ac9..6f22912a7 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -83,7 +83,8 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, urlLocation := lookup.Locations[rand.Intn(len(lookup.Locations))].PublicUrl urlString := "http://" + urlLocation + "/" + fileId if fs.redirectOnRead { - + http.Redirect(w, r, urlString, http.StatusFound) + return } u, _ := url.Parse(urlString) request := &http.Request{