From df78466a12d9eea524cb31d32f17b127bcc76b0b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 5 Aug 2014 18:14:12 -0700 Subject: [PATCH 01/58] remove uploading stated from volume server. --- go/weed/weed_server/volume_server.go | 1 - go/weed/weed_server/volume_server_handlers_admin.go | 4 ---- 2 files changed, 5 deletions(-) diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index b118e8c35..9cada91ac 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -35,7 +35,6 @@ func NewVolumeServer(r *http.ServeMux, ip string, port int, publicIp string, fol } vs.store = storage.NewStore(port, ip, publicUrl, folders, maxCounts) - r.HandleFunc("/submit", secure(vs.whiteList, vs.submitFromVolumeServerHandler)) r.HandleFunc("/status", secure(vs.whiteList, vs.statusHandler)) r.HandleFunc("/admin/assign_volume", secure(vs.whiteList, vs.assignVolumeHandler)) r.HandleFunc("/admin/vacuum_volume_check", secure(vs.whiteList, vs.vacuumVolumeCheckHandler)) diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go index 5921a3b96..6d285524a 100644 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ b/go/weed/weed_server/volume_server_handlers_admin.go @@ -50,10 +50,6 @@ func (vs *VolumeServer) freezeVolumeHandler(w http.ResponseWriter, r *http.Reque glog.V(2).Infoln("freeze volume =", r.FormValue("volume"), ", error =", err) } -func (vs *VolumeServer) submitFromVolumeServerHandler(w http.ResponseWriter, r *http.Request) { - submitForClientHandler(w, r, vs.masterNode) -} - func (vs *VolumeServer) statsDiskHandler(w http.ResponseWriter, r *http.Request) { m := make(map[string]interface{}) m["Version"] = util.VERSION From aac5fa49de28504f12e283f5fb22f22a1044c676 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 11 Aug 2014 07:41:00 -0700 Subject: [PATCH 02/58] remove the submit via volume server api. --- docs/api.rst | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 4f3806be9..cb66d5627 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -69,17 +69,6 @@ One volume servers one write a time. If you need to increase concurrency, you ca This generates 4 empty volumes. -Upload File Directly -*********************************** - -.. code-block:: bash - - curl -F file=@/home/chris/myphoto.jpg http://localhost:9333/submit - {"fid":"3,01fbe0dc6f1f38","fileName":"myphoto.jpg", - "fileUrl":"localhost:8080/3,01fbe0dc6f1f38","size":68231} - -This API is a little convenient. The master server would contact itself via HTTP to get an file id and store it to the right volume server. It is a convenient API and does not support different parameters when assigning file id. - Check System Status *********************************** From 4c58cef24aa81c08f3df0b3d8bbaede9e54bfa0c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 25 Aug 2014 11:37:00 -0700 Subject: [PATCH 03/58] a bit refactoring to prepare for volume format change and backward compatibility. --- go/storage/volume.go | 56 +-------------------------- go/storage/volume_super_block.go | 65 ++++++++++++++++++++++++++++++++ go/weed/export.go | 4 +- 3 files changed, 68 insertions(+), 57 deletions(-) create mode 100644 go/storage/volume_super_block.go diff --git a/go/storage/volume.go b/go/storage/volume.go index 7bd8e7467..dec560545 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -12,22 +12,6 @@ import ( "time" ) -const ( - SuperBlockSize = 8 -) - -type SuperBlock struct { - Version Version - ReplicaPlacement *ReplicaPlacement -} - -func (s *SuperBlock) Bytes() []byte { - header := make([]byte, SuperBlockSize) - header[0] = byte(s.Version) - header[1] = s.ReplicaPlacement.Byte() - return header -} - type Volume struct { Id VolumeId dir string @@ -122,7 +106,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error { return e } func (v *Volume) Version() Version { - return v.SuperBlock.Version + return v.SuperBlock.Version() } func (v *Volume) Size() int64 { stat, e := v.dataFile.Stat() @@ -138,44 +122,6 @@ func (v *Volume) Close() { v.nm.Close() _ = v.dataFile.Close() } -func (v *Volume) maybeWriteSuperBlock() error { - stat, e := v.dataFile.Stat() - if e != nil { - glog.V(0).Infof("failed to stat datafile %s: %s", v.dataFile, e.Error()) - return e - } - if stat.Size() == 0 { - v.SuperBlock.Version = CurrentVersion - _, e = v.dataFile.Write(v.SuperBlock.Bytes()) - if e != nil && os.IsPermission(e) { - //read-only, but zero length - recreate it! - if v.dataFile, e = os.Create(v.dataFile.Name()); e == nil { - if _, e = v.dataFile.Write(v.SuperBlock.Bytes()); e == nil { - v.readOnly = false - } - } - } - } - return e -} -func (v *Volume) readSuperBlock() (err error) { - if _, err = v.dataFile.Seek(0, 0); err != nil { - return fmt.Errorf("cannot seek to the beginning of %s: %s", v.dataFile.Name(), err.Error()) - } - header := make([]byte, SuperBlockSize) - if _, e := v.dataFile.Read(header); e != nil { - return fmt.Errorf("cannot read superblock: %s", e.Error()) - } - v.SuperBlock, err = ParseSuperBlock(header) - return err -} -func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) { - superBlock.Version = Version(header[0]) - if superBlock.ReplicaPlacement, err = NewReplicaPlacementFromByte(header[1]); err != nil { - err = fmt.Errorf("cannot read replica type: %s", err.Error()) - } - return -} func (v *Volume) NeedToReplicate() bool { return v.ReplicaPlacement.GetCopyCount() > 1 } diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go new file mode 100644 index 000000000..95f7c30bb --- /dev/null +++ b/go/storage/volume_super_block.go @@ -0,0 +1,65 @@ +package storage + +import ( + "code.google.com/p/weed-fs/go/glog" + "fmt" + "os" +) + +const ( + SuperBlockSize = 8 +) + +type SuperBlock struct { + version Version + ReplicaPlacement *ReplicaPlacement +} + +func (s *SuperBlock) Version() Version { + return s.version +} +func (s *SuperBlock) Bytes() []byte { + header := make([]byte, SuperBlockSize) + header[0] = byte(s.version) + header[1] = s.ReplicaPlacement.Byte() + return header +} + +func (v *Volume) maybeWriteSuperBlock() error { + stat, e := v.dataFile.Stat() + if e != nil { + glog.V(0).Infof("failed to stat datafile %s: %s", v.dataFile, e.Error()) + return e + } + if stat.Size() == 0 { + v.SuperBlock.version = CurrentVersion + _, e = v.dataFile.Write(v.SuperBlock.Bytes()) + if e != nil && os.IsPermission(e) { + //read-only, but zero length - recreate it! + if v.dataFile, e = os.Create(v.dataFile.Name()); e == nil { + if _, e = v.dataFile.Write(v.SuperBlock.Bytes()); e == nil { + v.readOnly = false + } + } + } + } + return e +} +func (v *Volume) readSuperBlock() (err error) { + if _, err = v.dataFile.Seek(0, 0); err != nil { + return fmt.Errorf("cannot seek to the beginning of %s: %s", v.dataFile.Name(), err.Error()) + } + header := make([]byte, SuperBlockSize) + if _, e := v.dataFile.Read(header); e != nil { + return fmt.Errorf("cannot read superblock: %s", e.Error()) + } + v.SuperBlock, err = ParseSuperBlock(header) + return err +} +func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) { + superBlock.version = Version(header[0]) + if superBlock.ReplicaPlacement, err = NewReplicaPlacementFromByte(header[1]); err != nil { + err = fmt.Errorf("cannot read replica type: %s", err.Error()) + } + return +} diff --git a/go/weed/export.go b/go/weed/export.go index 3f8ff85bb..f79ab1a5f 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -27,7 +27,7 @@ var cmdExport = &Command{ UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}}", Short: "list or export files from one volume data file", Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified. - + The format of file name in the tar file can be customized. Default is {{.Mime}}/{{.Id}}:{{.Name}}. Also available is {{.Key}}. `, @@ -100,7 +100,7 @@ func runExport(cmd *Command, args []string) bool { var version storage.Version err = storage.ScanVolumeFile(*exportVolumePath, *exportCollection, vid, func(superBlock storage.SuperBlock) error { - version = superBlock.Version + version = superBlock.Version() return nil }, true, func(n *storage.Needle, offset int64) error { nv, ok := nm.Get(n.Id) From ce4acecaa8fa9461cbfae1fadea5b320161d2acd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 25 Aug 2014 12:02:04 -0700 Subject: [PATCH 04/58] Fix filer proxing http status code Double quote etag value. --- go/weed/weed_server/filer_server_handlers.go | 1 + go/weed/weed_server/volume_server_handlers.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index ab30aaaed..0f83352a9 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -103,6 +103,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, for k, v := range resp.Header { w.Header()[k] = v } + w.WriteHeader(resp.StatusCode) io.Copy(w, resp.Body) } diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index eed198e4a..e7571ccc7 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -95,7 +95,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNotModified) return } - w.Header().Set("Etag", etag) + w.Header().Set("Etag", "\""+etag+"\"") if n.NameSize > 0 && filename == "" { filename = string(n.Name) dotIndex := strings.LastIndex(filename, ".") From 57a4549d8678bb08f4ecbdede1939830471e3091 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 26 Aug 2014 10:15:12 -0700 Subject: [PATCH 05/58] wrap etag value with double quotes --- go/storage/crc.go | 2 +- go/weed/weed_server/volume_server_handlers.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/storage/crc.go b/go/storage/crc.go index 41f7f6d00..43e65757a 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -25,5 +25,5 @@ func (c CRC) Value() uint32 { func (n *Needle) Etag() string { bits := make([]byte, 4) util.Uint32toBytes(bits, uint32(n.Checksum)) - return fmt.Sprintf("%x", bits) + return fmt.Sprintf("\"%x\"", bits) } diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index e7571ccc7..eed198e4a 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -95,7 +95,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNotModified) return } - w.Header().Set("Etag", "\""+etag+"\"") + w.Header().Set("Etag", etag) if n.NameSize > 0 && filename == "" { filename = string(n.Name) dotIndex := strings.LastIndex(filename, ".") From 69343c5951a7e6a6e272393c1946c12d635b51b8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 31 Aug 2014 23:25:54 -0700 Subject: [PATCH 06/58] adding ttl field to volume super block --- go/storage/volume_super_block.go | 10 ++++++++++ go/storage/volume_super_block_test.go | 22 ++++++++++++++++++++++ go/util/bytes.go | 12 ++++++++++++ go/util/constants.go | 2 +- 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 go/storage/volume_super_block_test.go diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index 95f7c30bb..35030b93e 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -2,6 +2,7 @@ package storage import ( "code.google.com/p/weed-fs/go/glog" + "code.google.com/p/weed-fs/go/util" "fmt" "os" ) @@ -10,9 +11,16 @@ const ( SuperBlockSize = 8 ) +/* +* Super block currently has 8 bytes allocated for each volume. +* Byte 0: version, 1 or 2 +* Byte 1: Replica Placement strategy, 000, 001, 002, 010, etc +* Byte 2 and byte 3: Time to live in minutes + */ type SuperBlock struct { version Version ReplicaPlacement *ReplicaPlacement + Ttl uint16 } func (s *SuperBlock) Version() Version { @@ -22,6 +30,7 @@ func (s *SuperBlock) Bytes() []byte { header := make([]byte, SuperBlockSize) header[0] = byte(s.version) header[1] = s.ReplicaPlacement.Byte() + util.Uint16toBytes(header[2:4], s.Ttl) return header } @@ -61,5 +70,6 @@ func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) { if superBlock.ReplicaPlacement, err = NewReplicaPlacementFromByte(header[1]); err != nil { err = fmt.Errorf("cannot read replica type: %s", err.Error()) } + superBlock.Ttl = util.BytesToUint16(header[2:4]) return } diff --git a/go/storage/volume_super_block_test.go b/go/storage/volume_super_block_test.go new file mode 100644 index 000000000..19a1bb757 --- /dev/null +++ b/go/storage/volume_super_block_test.go @@ -0,0 +1,22 @@ +package storage + +import ( + "testing" +) + +func TestSuperBlockReadWrite(t *testing.T) { + rp, _ := NewReplicaPlacementFromByte(byte(001)) + s := &SuperBlock{ + version: CurrentVersion, + ReplicaPlacement: rp, + Ttl: uint16(35), + } + + bytes := s.Bytes() + + if !(bytes[2] == 0 && bytes[3] == 35) { + println("byte[2]:", bytes[2], "byte[3]:", bytes[3]) + t.Fail() + } + +} diff --git a/go/util/bytes.go b/go/util/bytes.go index 6cc3d7018..dfa4ae665 100644 --- a/go/util/bytes.go +++ b/go/util/bytes.go @@ -1,5 +1,7 @@ package util +// big endian + func BytesToUint64(b []byte) (v uint64) { length := uint(len(b)) for i := uint(0); i < length-1; i++ { @@ -18,6 +20,12 @@ func BytesToUint32(b []byte) (v uint32) { v += uint32(b[length-1]) return } +func BytesToUint16(b []byte) (v uint16) { + v += uint16(b[0]) + v <<= 8 + v += uint16(b[1]) + return +} func Uint64toBytes(b []byte, v uint64) { for i := uint(0); i < 8; i++ { b[7-i] = byte(v >> (i * 8)) @@ -28,6 +36,10 @@ func Uint32toBytes(b []byte, v uint32) { b[3-i] = byte(v >> (i * 8)) } } +func Uint16toBytes(b []byte, v uint16) { + b[0] = byte(v >> 8) + b[1] = byte(v) +} func Uint8toBytes(b []byte, v uint8) { b[0] = byte(v) } diff --git a/go/util/constants.go b/go/util/constants.go index 196b4a405..f66930d62 100644 --- a/go/util/constants.go +++ b/go/util/constants.go @@ -3,5 +3,5 @@ package util import () const ( - VERSION = "0.63 beta" + VERSION = "0.63" ) From 4be5ccd0c85b82f0faae2905dd51340b2d84f98b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 4 Sep 2014 19:26:31 -0700 Subject: [PATCH 07/58] resolve directory log file error avoid possible race condition --- go/filer/directory_in_map.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/go/filer/directory_in_map.go b/go/filer/directory_in_map.go index 9c2ecdf80..051ff31fe 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/directory_in_map.go @@ -9,8 +9,11 @@ import ( "path/filepath" "strconv" "strings" + "sync" ) +var writeLock sync.Mutex //serialize changes to dir.log + type DirectoryEntryInMap struct { Name string Parent *DirectoryEntryInMap @@ -26,9 +29,9 @@ type DirectoryManagerInMap struct { } func (dm *DirectoryManagerInMap) NewDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap) { + writeLock.Lock() + defer writeLock.Unlock() d = &DirectoryEntryInMap{Name: name, Parent: parent, SubDirectories: make(map[string]*DirectoryEntryInMap)} - dm.max++ - d.Id = dm.max parts := make([]string, 0) for p := d; p != nil && p.Name != ""; p = p.Parent { parts = append(parts, p.Name) @@ -40,6 +43,8 @@ func (dm *DirectoryManagerInMap) NewDirectoryEntryInMap(parent *DirectoryEntryIn for i := 0; i < n/2; i++ { parts[i], parts[n-1-i] = parts[n-1-i], parts[i] } + dm.max++ + d.Id = dm.max dm.log("add", "/"+strings.Join(parts, "/"), strconv.Itoa(int(d.Id))) return d } @@ -193,6 +198,8 @@ func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (DirectoryId, err } func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParentDirPath string, newName string) error { + writeLock.Lock() + defer writeLock.Unlock() oldDir, oe := dm.findDirectory(oldDirPath) if oe != nil { return oe @@ -223,6 +230,8 @@ func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []Dir return dirNames, nil } func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error { + writeLock.Lock() + defer writeLock.Unlock() if dirPath == "/" { return fmt.Errorf("Can not delete %s", dirPath) } From a092794804b2f7cbd656e439305d29bfa96ad2b9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 4 Sep 2014 19:34:43 -0700 Subject: [PATCH 08/58] use error to report error --- go/filer/directory_in_map.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/go/filer/directory_in_map.go b/go/filer/directory_in_map.go index 051ff31fe..46a626f77 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/directory_in_map.go @@ -28,7 +28,7 @@ type DirectoryManagerInMap struct { isLoading bool } -func (dm *DirectoryManagerInMap) NewDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap) { +func (dm *DirectoryManagerInMap) NewDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap, err error) { writeLock.Lock() defer writeLock.Unlock() d = &DirectoryEntryInMap{Name: name, Parent: parent, SubDirectories: make(map[string]*DirectoryEntryInMap)} @@ -38,7 +38,7 @@ func (dm *DirectoryManagerInMap) NewDirectoryEntryInMap(parent *DirectoryEntryIn } n := len(parts) if n <= 0 { - return d + return nil, fmt.Errorf("Failed to create folder %s/%s", parent.Name, name) } for i := 0; i < n/2; i++ { parts[i], parts[n-1-i] = parts[n-1-i], parts[i] @@ -46,7 +46,7 @@ func (dm *DirectoryManagerInMap) NewDirectoryEntryInMap(parent *DirectoryEntryIn dm.max++ d.Id = dm.max dm.log("add", "/"+strings.Join(parts, "/"), strconv.Itoa(int(d.Id))) - return d + return d, nil } func (dm *DirectoryManagerInMap) log(words ...string) { @@ -162,7 +162,11 @@ func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId DirectoryId if i != len(parts)-1 { return fmt.Errorf("%s should be created after parent %s!", dirPath, parts[i]) } - sub = dm.NewDirectoryEntryInMap(dir, parts[i]) + var err error + sub, err = dm.NewDirectoryEntryInMap(dir, parts[i]) + if err != nil { + return err + } if sub.Id != dirId { return fmt.Errorf("%s should be have id %v instead of %v!", dirPath, sub.Id, dirId) } @@ -183,7 +187,11 @@ func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEn for i := 1; i < len(parts); i++ { sub, ok := dir.SubDirectories[parts[i]] if !ok { - sub = dm.NewDirectoryEntryInMap(dir, parts[i]) + var err error + sub, err = dm.NewDirectoryEntryInMap(dir, parts[i]) + if err != nil { + return nil, false + } dir.SubDirectories[parts[i]] = sub created = true } From b9aee2defbc2f5aafbc3ea049fbe2ab5f3320999 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 20 Sep 2014 12:38:59 -0700 Subject: [PATCH 09/58] add TTL support The volume TTL and file TTL are not necessarily the same. as long as file TTL is smaller than volume TTL, it'll be fine. volume TTL is used when assigning file id, e.g. http://.../dir/assign?ttl=3h file TTL is used when uploading --- go/filer/directory_in_map.go | 4 +- go/operation/assign_file_id.go | 5 +- go/operation/submit.go | 11 +- go/operation/system_message.pb.go | 12 +- go/proto/system_message.proto | 1 + go/storage/needle.go | 9 +- go/storage/needle_read_write.go | 21 +++ go/storage/replica_placement.go | 8 - go/storage/store.go | 67 +++++--- go/storage/volume.go | 74 ++++++++- go/storage/volume_info.go | 2 + go/storage/volume_super_block.go | 10 +- go/storage/volume_super_block_test.go | 7 +- go/storage/volume_ttl.go | 147 ++++++++++++++++++ go/storage/volume_vacuum.go | 14 +- go/topology/allocate_volume.go | 7 +- go/topology/collection.go | 23 +-- go/topology/data_node.go | 6 +- go/topology/topology.go | 21 ++- go/topology/topology_event_handling.go | 6 +- go/topology/topology_map.go | 2 +- go/topology/topology_vacuum.go | 2 +- go/topology/volume_growth.go | 12 +- go/topology/volume_layout.go | 13 +- go/weed/benchmark.go | 2 +- go/weed/compact.go | 2 +- go/weed/upload.go | 6 +- go/weed/weed_server/common.go | 4 +- go/weed/weed_server/filer_server_handlers.go | 12 +- .../master_server_handlers_admin.go | 9 +- .../volume_server_handlers_admin.go | 2 +- 31 files changed, 416 insertions(+), 105 deletions(-) create mode 100644 go/storage/volume_ttl.go diff --git a/go/filer/directory_in_map.go b/go/filer/directory_in_map.go index 46a626f77..35b4e53c1 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/directory_in_map.go @@ -187,10 +187,10 @@ func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEn for i := 1; i < len(parts); i++ { sub, ok := dir.SubDirectories[parts[i]] if !ok { - var err error + var err error sub, err = dm.NewDirectoryEntryInMap(dir, parts[i]) if err != nil { - return nil, false + return nil, false } dir.SubDirectories[parts[i]] = sub created = true diff --git a/go/operation/assign_file_id.go b/go/operation/assign_file_id.go index 018e1d763..34d371f37 100644 --- a/go/operation/assign_file_id.go +++ b/go/operation/assign_file_id.go @@ -17,7 +17,7 @@ type AssignResult struct { Error string `json:"error,omitempty"` } -func Assign(server string, count int, replication string, collection string) (*AssignResult, error) { +func Assign(server string, count int, replication string, collection string, ttl string) (*AssignResult, error) { values := make(url.Values) values.Add("count", strconv.Itoa(count)) if replication != "" { @@ -26,6 +26,9 @@ func Assign(server string, count int, replication string, collection string) (*A if collection != "" { values.Add("collection", collection) } + if ttl != "" { + values.Add("ttl", ttl) + } jsonBlob, err := util.Post("http://"+server+"/dir/assign", values) glog.V(2).Info("assign result :", string(jsonBlob)) if err != nil { diff --git a/go/operation/submit.go b/go/operation/submit.go index 9191f7d9a..ec45cc320 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -20,6 +20,7 @@ type FilePart struct { ModTime int64 //in seconds Replication string Collection string + Ttl string Server string //this comes from assign result Fid string //this comes from assign result, but customizable } @@ -32,12 +33,12 @@ type SubmitResult struct { Error string `json:"error,omitempty"` } -func SubmitFiles(master string, files []FilePart, replication string, collection string, maxMB int) ([]SubmitResult, error) { +func SubmitFiles(master string, files []FilePart, replication string, collection string, ttl string, maxMB int) ([]SubmitResult, error) { results := make([]SubmitResult, len(files)) for index, file := range files { results[index].FileName = file.FileName } - ret, err := Assign(master, len(files), replication, collection) + ret, err := Assign(master, len(files), replication, collection, ttl) if err != nil { for index, _ := range files { results[index].Error = err.Error() @@ -112,7 +113,7 @@ func (fi FilePart) Upload(maxMB int, master string) (retSize uint32, err error) chunks := fi.FileSize/chunkSize + 1 fids := make([]string, 0) for i := int64(0); i < chunks; i++ { - id, count, e := upload_one_chunk(fi.FileName+"-"+strconv.FormatInt(i+1, 10), io.LimitReader(fi.Reader, chunkSize), master, fi.Replication, fi.Collection) + id, count, e := upload_one_chunk(fi.FileName+"-"+strconv.FormatInt(i+1, 10), io.LimitReader(fi.Reader, chunkSize), master, fi.Replication, fi.Collection, fi.Ttl) if e != nil { return 0, e } @@ -130,8 +131,8 @@ func (fi FilePart) Upload(maxMB int, master string) (retSize uint32, err error) return } -func upload_one_chunk(filename string, reader io.Reader, master, replication string, collection string) (fid string, size uint32, e error) { - ret, err := Assign(master, 1, replication, collection) +func upload_one_chunk(filename string, reader io.Reader, master, replication string, collection string, ttl string) (fid string, size uint32, e error) { + ret, err := Assign(master, 1, replication, collection, ttl) if err != nil { return "", 0, err } diff --git a/go/operation/system_message.pb.go b/go/operation/system_message.pb.go index 45ae8a648..9f00dd74d 100644 --- a/go/operation/system_message.pb.go +++ b/go/operation/system_message.pb.go @@ -15,12 +15,10 @@ It has these top-level messages: package operation import proto "code.google.com/p/goprotobuf/proto" -import json "encoding/json" import math "math" -// Reference proto, json, and math imports to suppress error if they are not otherwise used. +// Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal -var _ = &json.SyntaxError{} var _ = math.Inf type VolumeInformationMessage struct { @@ -33,6 +31,7 @@ type VolumeInformationMessage struct { ReadOnly *bool `protobuf:"varint,7,opt,name=read_only" json:"read_only,omitempty"` ReplicaPlacement *uint32 `protobuf:"varint,8,req,name=replica_placement" json:"replica_placement,omitempty"` Version *uint32 `protobuf:"varint,9,opt,name=version,def=2" json:"version,omitempty"` + Ttl *uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -105,6 +104,13 @@ func (m *VolumeInformationMessage) GetVersion() uint32 { return Default_VolumeInformationMessage_Version } +func (m *VolumeInformationMessage) GetTtl() uint32 { + if m != nil && m.Ttl != nil { + return *m.Ttl + } + return 0 +} + type JoinMessage struct { IsInit *bool `protobuf:"varint,1,opt,name=is_init" json:"is_init,omitempty"` Ip *string `protobuf:"bytes,2,req,name=ip" json:"ip,omitempty"` diff --git a/go/proto/system_message.proto b/go/proto/system_message.proto index 15574ad56..ecd4973f7 100644 --- a/go/proto/system_message.proto +++ b/go/proto/system_message.proto @@ -10,6 +10,7 @@ message VolumeInformationMessage { optional bool read_only = 7; required uint32 replica_placement = 8; optional uint32 version = 9 [default=2]; + optional uint32 ttl = 10; } message JoinMessage { diff --git a/go/storage/needle.go b/go/storage/needle.go index 77aa70169..3bf627141 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -38,12 +38,13 @@ type Needle struct { MimeSize uint8 //version2 Mime []byte `comment:"maximum 256 characters"` //version2 LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk + Ttl *TTL Checksum CRC `comment:"CRC32 to check integrity"` Padding []byte `comment:"Aligned to 8 bytes"` } -func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string, isGzipped bool, modifiedTime uint64, e error) { +func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string, isGzipped bool, modifiedTime uint64, ttl *TTL, e error) { form, fe := r.MultipartReader() if fe != nil { glog.V(0).Infoln("MultipartReader [ERROR]", fe) @@ -92,12 +93,13 @@ func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string fileName = fileName[:len(fileName)-3] } modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64) + ttl, _ = ReadTTL(r.FormValue("ttl")) return } func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) { fname, mimeType, isGzipped := "", "", false n = new(Needle) - fname, n.Data, mimeType, isGzipped, n.LastModified, e = ParseUpload(r) + fname, n.Data, mimeType, isGzipped, n.LastModified, n.Ttl, e = ParseUpload(r) if e != nil { return } @@ -116,6 +118,9 @@ func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) { n.LastModified = uint64(time.Now().Unix()) } n.SetHasLastModifiedDate() + if n.Ttl != nil { + n.SetHasTtl() + } if fixJpgOrientation { loweredName := strings.ToLower(fname) diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index 835d7c270..848121ff2 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -14,7 +14,9 @@ const ( FlagHasName = 0x02 FlagHasMime = 0x04 FlagHasLastModifiedDate = 0x08 + FlagHasTtl = 0x10 LastModifiedBytesLength = 5 + TtlBytesLength = 2 ) func (n *Needle) DiskSize() int64 { @@ -70,6 +72,9 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { if n.HasLastModifiedDate() { n.Size = n.Size + LastModifiedBytesLength } + if n.HasTtl() { + n.Size = n.Size + TtlBytesLength + } } size = n.DataSize util.Uint32toBytes(header[12:16], n.Size) @@ -112,6 +117,12 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { return } } + if n.HasTtl() { + n.Ttl.ToBytes(header[0:TtlBytesLength]) + if _, err = w.Write(header[0:TtlBytesLength]); err != nil { + return + } + } } padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize) util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) @@ -194,6 +205,10 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) { n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength]) index = index + LastModifiedBytesLength } + if index < lenBytes && n.HasTtl() { + n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength]) + index = index + TtlBytesLength + } } func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bodyLength uint32, err error) { @@ -263,3 +278,9 @@ func (n *Needle) HasLastModifiedDate() bool { func (n *Needle) SetHasLastModifiedDate() { n.Flags = n.Flags | FlagHasLastModifiedDate } +func (n *Needle) HasTtl() bool { + return n.Flags&FlagHasTtl > 0 +} +func (n *Needle) SetHasTtl() { + n.Flags = n.Flags | FlagHasTtl +} diff --git a/go/storage/replica_placement.go b/go/storage/replica_placement.go index 696888cd8..c1aca52eb 100644 --- a/go/storage/replica_placement.go +++ b/go/storage/replica_placement.go @@ -5,10 +5,6 @@ import ( "fmt" ) -const ( - ReplicaPlacementCount = 9 -) - type ReplicaPlacement struct { SameRackCount int DiffRackCount int @@ -55,7 +51,3 @@ func (rp *ReplicaPlacement) String() string { func (rp *ReplicaPlacement) GetCopyCount() int { return rp.DiffDataCenterCount + rp.DiffRackCount + rp.SameRackCount + 1 } - -func (rp *ReplicaPlacement) GetReplicationLevelIndex() int { - return rp.DiffDataCenterCount*3 + rp.DiffRackCount*3 + rp.SameRackCount -} diff --git a/go/storage/store.go b/go/storage/store.go index a6a4f399e..ef38ade98 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -14,6 +14,10 @@ import ( "strings" ) +const ( + MAX_TTL_VOLUME_REMOVAL_DELAY = 10 // 10 minutes +) + type DiskLocation struct { Directory string MaxVolumeCount int @@ -83,11 +87,15 @@ func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts } return } -func (s *Store) AddVolume(volumeListString string, collection string, replicaPlacement string) error { +func (s *Store) AddVolume(volumeListString string, collection string, replicaPlacement string, ttlString string) error { rt, e := NewReplicaPlacementFromString(replicaPlacement) if e != nil { return e } + ttl, e := ReadTTL(ttlString) + if e != nil { + return e + } for _, range_string := range strings.Split(volumeListString, ",") { if strings.Index(range_string, "-") < 0 { id_string := range_string @@ -95,7 +103,7 @@ func (s *Store) AddVolume(volumeListString string, collection string, replicaPla if err != nil { return fmt.Errorf("Volume Id %s is not a valid unsigned integer!", id_string) } - e = s.addVolume(VolumeId(id), collection, rt) + e = s.addVolume(VolumeId(id), collection, rt, ttl) } else { pair := strings.Split(range_string, "-") start, start_err := strconv.ParseUint(pair[0], 10, 64) @@ -107,7 +115,7 @@ func (s *Store) AddVolume(volumeListString string, collection string, replicaPla return fmt.Errorf("Volume End Id %s is not a valid unsigned integer!", pair[1]) } for id := start; id <= end; id++ { - if err := s.addVolume(VolumeId(id), collection, rt); err != nil { + if err := s.addVolume(VolumeId(id), collection, rt, ttl); err != nil { e = err } } @@ -129,6 +137,14 @@ func (s *Store) DeleteCollection(collection string) (e error) { } return } +func (s *Store) DeleteVolume(volumes map[VolumeId]*Volume, v *Volume) (e error) { + e = v.Destroy() + if e != nil { + return + } + delete(volumes, v.Id) + return +} func (s *Store) findVolume(vid VolumeId) *Volume { for _, location := range s.Locations { if v, found := location.volumes[vid]; found { @@ -148,13 +164,14 @@ func (s *Store) findFreeLocation() (ret *DiskLocation) { } return ret } -func (s *Store) addVolume(vid VolumeId, collection string, replicaPlacement *ReplicaPlacement) error { +func (s *Store) addVolume(vid VolumeId, collection string, replicaPlacement *ReplicaPlacement, ttl *TTL) error { if s.findVolume(vid) != nil { return fmt.Errorf("Volume Id %d already exists!", vid) } if location := s.findFreeLocation(); location != nil { - glog.V(0).Infoln("In dir", location.Directory, "adds volume =", vid, ", collection =", collection, ", replicaPlacement =", replicaPlacement) - if volume, err := NewVolume(location.Directory, collection, vid, replicaPlacement); err == nil { + glog.V(0).Infof("In dir %s adds volume:%v collection:%s replicaPlacement:%v ttl:%v", + location.Directory, vid, collection, replicaPlacement, ttl) + if volume, err := NewVolume(location.Directory, collection, vid, replicaPlacement, ttl); err == nil { location.volumes[vid] = volume return nil } else { @@ -190,9 +207,9 @@ func (l *DiskLocation) loadExistingVolumes() { } if vid, err := NewVolumeId(base); err == nil { if l.volumes[vid] == nil { - if v, e := NewVolume(l.Directory, collection, vid, nil); e == nil { + if v, e := NewVolume(l.Directory, collection, vid, nil, nil); e == nil { l.volumes[vid] = v - glog.V(0).Infoln("data file", l.Directory+"/"+name, "replicaPlacement =", v.ReplicaPlacement, "version =", v.Version(), "size =", v.Size()) + glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s", l.Directory+"/"+name, v.ReplicaPlacement, v.Version(), v.Size(), v.Ttl.String()) } } } @@ -240,21 +257,31 @@ func (s *Store) Join() (masterNode string, e error) { for _, location := range s.Locations { maxVolumeCount = maxVolumeCount + location.MaxVolumeCount for k, v := range location.volumes { - volumeMessage := &operation.VolumeInformationMessage{ - Id: proto.Uint32(uint32(k)), - Size: proto.Uint64(uint64(v.Size())), - Collection: proto.String(v.Collection), - FileCount: proto.Uint64(uint64(v.nm.FileCount())), - DeleteCount: proto.Uint64(uint64(v.nm.DeletedCount())), - DeletedByteCount: proto.Uint64(v.nm.DeletedSize()), - ReadOnly: proto.Bool(v.readOnly), - ReplicaPlacement: proto.Uint32(uint32(v.ReplicaPlacement.Byte())), - Version: proto.Uint32(uint32(v.Version())), - } - volumeMessages = append(volumeMessages, volumeMessage) if maxFileKey < v.nm.MaxFileKey() { maxFileKey = v.nm.MaxFileKey() } + if !v.expired(s.volumeSizeLimit) { + volumeMessage := &operation.VolumeInformationMessage{ + Id: proto.Uint32(uint32(k)), + Size: proto.Uint64(uint64(v.Size())), + Collection: proto.String(v.Collection), + FileCount: proto.Uint64(uint64(v.nm.FileCount())), + DeleteCount: proto.Uint64(uint64(v.nm.DeletedCount())), + DeletedByteCount: proto.Uint64(v.nm.DeletedSize()), + ReadOnly: proto.Bool(v.readOnly), + ReplicaPlacement: proto.Uint32(uint32(v.ReplicaPlacement.Byte())), + Version: proto.Uint32(uint32(v.Version())), + Ttl: proto.Uint32(v.Ttl.ToUint32()), + } + volumeMessages = append(volumeMessages, volumeMessage) + } else { + if v.exiredLongEnough(MAX_TTL_VOLUME_REMOVAL_DELAY) { + s.DeleteVolume(location.volumes, v) + glog.V(0).Infoln("volume", v.Id, "is deleted.") + } else { + glog.V(0).Infoln("volume", v.Id, "is expired.") + } + } } } diff --git a/go/storage/volume.go b/go/storage/volume.go index dec560545..34ae7e386 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -22,12 +22,13 @@ type Volume struct { SuperBlock - accessLock sync.Mutex + accessLock sync.Mutex + lastModifiedTime uint64 //unix time in seconds } -func NewVolume(dirname string, collection string, id VolumeId, replicaPlacement *ReplicaPlacement) (v *Volume, e error) { +func NewVolume(dirname string, collection string, id VolumeId, replicaPlacement *ReplicaPlacement, ttl *TTL) (v *Volume, e error) { v = &Volume{dir: dirname, Collection: collection, Id: id} - v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement} + v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl} e = v.load(true, true) return } @@ -49,12 +50,13 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error { var e error fileName := v.FileName() - if exists, canRead, canWrite, _ := checkFile(fileName + ".dat"); exists { + if exists, canRead, canWrite, modifiedTime := checkFile(fileName + ".dat"); exists { if !canRead { return fmt.Errorf("cannot read Volume Data file %s.dat", fileName) } if canWrite { v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644) + v.lastModifiedTime = uint64(modifiedTime.Unix()) } else { glog.V(0).Infoln("opening " + fileName + ".dat in READONLY mode") v.dataFile, e = os.Open(fileName + ".dat") @@ -192,6 +194,9 @@ func (v *Volume) write(n *Needle) (size uint32, err error) { glog.V(4).Infof("failed to save in needle map %d: %s", n.Id, err.Error()) } } + if v.lastModifiedTime < n.LastModified { + v.lastModifiedTime = n.LastModified + } return } @@ -221,8 +226,25 @@ func (v *Volume) delete(n *Needle) (uint32, error) { func (v *Volume) read(n *Needle) (int, error) { nv, ok := v.nm.Get(n.Id) - if ok && nv.Offset > 0 { - return n.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version()) + if !ok || nv.Offset == 0 { + return -1, errors.New("Not Found") + } + bytesRead, err := n.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version()) + if err != nil { + return bytesRead, err + } + if !n.HasTtl() { + return bytesRead, err + } + ttlMinutes := n.Ttl.Minutes() + if ttlMinutes == 0 { + return bytesRead, nil + } + if !n.HasLastModifiedDate() { + return bytesRead, nil + } + if uint64(time.Now().Unix()) < n.LastModified+uint64(ttlMinutes*60) { + return bytesRead, nil } return -1, errors.New("Not Found") } @@ -343,3 +365,43 @@ func (v *Volume) ensureConvertIdxToCdb(fileName string) (cdbCanRead bool) { } return true } + +// volume is expired if modified time + volume ttl < now +// except when volume is empty +// or when the volume does not have a ttl +// or when volumeSizeLimit is 0 when server just starts +func (v *Volume) expired(volumeSizeLimit uint64) bool { + if volumeSizeLimit == 0 { + //skip if we don't know size limit + return false + } + if v.ContentSize() == 0 { + return false + } + if v.Ttl == nil || v.Ttl.Minutes() == 0 { + return false + } + glog.V(0).Infof("now:%v lastModified:%v", time.Now().Unix(), v.lastModifiedTime) + livedMinutes := (time.Now().Unix() - int64(v.lastModifiedTime)) / 60 + glog.V(0).Infof("ttl:%v lived:%v", v.Ttl, livedMinutes) + if int64(v.Ttl.Minutes()) < livedMinutes { + return true + } + return false +} + +// wait either maxDelayMinutes or 10% of ttl minutes +func (v *Volume) exiredLongEnough(maxDelayMinutes uint32) bool { + if v.Ttl == nil || v.Ttl.Minutes() == 0 { + return false + } + removalDelay := v.Ttl.Minutes() / 10 + if removalDelay > maxDelayMinutes { + removalDelay = maxDelayMinutes + } + + if uint64(v.Ttl.Minutes()+removalDelay)*60+v.lastModifiedTime < uint64(time.Now().Unix()) { + return true + } + return false +} diff --git a/go/storage/volume_info.go b/go/storage/volume_info.go index 165af1a19..6a954f743 100644 --- a/go/storage/volume_info.go +++ b/go/storage/volume_info.go @@ -8,6 +8,7 @@ type VolumeInfo struct { Id VolumeId Size uint64 ReplicaPlacement *ReplicaPlacement + Ttl *TTL Collection string Version Version FileCount int @@ -32,5 +33,6 @@ func NewVolumeInfo(m *operation.VolumeInformationMessage) (vi VolumeInfo, err er return vi, e } vi.ReplicaPlacement = rp + vi.Ttl = LoadTTLFromUint32(*m.Ttl) return vi, nil } diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index 35030b93e..3fbef44d6 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -2,7 +2,6 @@ package storage import ( "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/util" "fmt" "os" ) @@ -15,12 +14,13 @@ const ( * Super block currently has 8 bytes allocated for each volume. * Byte 0: version, 1 or 2 * Byte 1: Replica Placement strategy, 000, 001, 002, 010, etc -* Byte 2 and byte 3: Time to live in minutes +* Byte 2 and byte 3: Time to live. See TTL for definition +* Rest bytes: Reserved */ type SuperBlock struct { version Version ReplicaPlacement *ReplicaPlacement - Ttl uint16 + Ttl *TTL } func (s *SuperBlock) Version() Version { @@ -30,7 +30,7 @@ func (s *SuperBlock) Bytes() []byte { header := make([]byte, SuperBlockSize) header[0] = byte(s.version) header[1] = s.ReplicaPlacement.Byte() - util.Uint16toBytes(header[2:4], s.Ttl) + s.Ttl.ToBytes(header[2:4]) return header } @@ -70,6 +70,6 @@ func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) { if superBlock.ReplicaPlacement, err = NewReplicaPlacementFromByte(header[1]); err != nil { err = fmt.Errorf("cannot read replica type: %s", err.Error()) } - superBlock.Ttl = util.BytesToUint16(header[2:4]) + superBlock.Ttl = LoadTTLFromBytes(header[2:4]) return } diff --git a/go/storage/volume_super_block_test.go b/go/storage/volume_super_block_test.go index 19a1bb757..13db4b194 100644 --- a/go/storage/volume_super_block_test.go +++ b/go/storage/volume_super_block_test.go @@ -6,16 +6,17 @@ import ( func TestSuperBlockReadWrite(t *testing.T) { rp, _ := NewReplicaPlacementFromByte(byte(001)) + ttl, _ := ReadTTL("15d") s := &SuperBlock{ version: CurrentVersion, ReplicaPlacement: rp, - Ttl: uint16(35), + Ttl: ttl, } bytes := s.Bytes() - if !(bytes[2] == 0 && bytes[3] == 35) { - println("byte[2]:", bytes[2], "byte[3]:", bytes[3]) + if !(bytes[2] == 15 && bytes[3] == Day) { + println("byte[2]:", bytes[2], "byte[3]:", bytes[3]) t.Fail() } diff --git a/go/storage/volume_ttl.go b/go/storage/volume_ttl.go new file mode 100644 index 000000000..5ff43e24e --- /dev/null +++ b/go/storage/volume_ttl.go @@ -0,0 +1,147 @@ +package storage + +import ( + "strconv" +) + +var ( + TtlRange []uint16 +) + +const ( + //stored unit types + Empty byte = iota + Minute + Hour + Day + Week + Month + Year +) + +func init() { + TtlRange = []uint16{3, 10, 30, + 60 /*1 hour*/, 60 * 3, 60 * 6, 60 * 12, + 1440 /*1 day*/, 1440 * 3, 1440 * 7, 1440 * 15, 1440 * 31, + 44888 /*1 month*/, 65535, + } +} + +type TTL struct { + count byte + unit byte +} + +var EMPTY_TTL = &TTL{} + +// translate a readable ttl to internal ttl +// Supports format example: +// 3m: 3 minutes +// 4h: 4 hours +// 5d: 5 days +// 6w: 6 weeks +// 7M: 7 months +// 8y: 8 years +func ReadTTL(ttlString string) (*TTL, error) { + if ttlString == "" { + return EMPTY_TTL, nil + } + ttlBytes := []byte(ttlString) + unitByte := ttlBytes[len(ttlBytes)-1] + countBytes := ttlBytes[0 : len(ttlBytes)-1] + if '0' <= unitByte && unitByte <= '9' { + countBytes = ttlBytes + unitByte = 'm' + } + count, err := strconv.Atoi(string(countBytes)) + unit := toStoredByte(unitByte) + return &TTL{count: byte(count), unit: unit}, err +} + +// read stored bytes to a ttl +func LoadTTLFromBytes(input []byte) (t *TTL) { + return &TTL{count: input[0], unit: input[1]} +} + +// read stored bytes to a ttl +func LoadTTLFromUint32(ttl uint32) (t *TTL) { + input := make([]byte, 2) + input[1] = byte(ttl) + input[0] = byte(ttl >> 8) + return LoadTTLFromBytes(input) +} + +// save stored bytes to an output with 2 bytes +func (t TTL) ToBytes(output []byte) { + output[0] = t.count + output[1] = t.unit +} + +func (t TTL) ToUint32() (output uint32) { + output = uint32(t.count) << 8 + output += uint32(t.unit) + return output +} + +func (t TTL) String() string { + if t.count == 0 { + return "" + } + if t.unit == Empty { + return "" + } + countString := strconv.Itoa(int(t.count)) + switch t.unit { + case Minute: + return countString + "m" + case Hour: + return countString + "h" + case Day: + return countString + "d" + case Week: + return countString + "w" + case Month: + return countString + "M" + case Year: + return countString + "y" + } + return "" +} + +func toStoredByte(readableUnitByte byte) byte { + switch readableUnitByte { + case 'm': + return Minute + case 'h': + return Hour + case 'd': + return Day + case 'w': + return Week + case 'M': + return Month + case 'Y': + return Year + } + return 0 +} + +func (t TTL) Minutes() uint32 { + switch t.unit { + case Empty: + return 0 + case Minute: + return uint32(t.count) + case Hour: + return uint32(t.count) * 60 + case Day: + return uint32(t.count) * 60 * 24 + case Week: + return uint32(t.count) * 60 * 24 * 7 + case Month: + return uint32(t.count) * 60 * 24 * 31 + case Year: + return uint32(t.count) * 60 * 24 * 31 * 365 + } + return 0 +} diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go index 7d2a38cb8..706a1f951 100644 --- a/go/storage/volume_vacuum.go +++ b/go/storage/volume_vacuum.go @@ -4,7 +4,7 @@ import ( "code.google.com/p/weed-fs/go/glog" "fmt" "os" - _ "time" + "time" ) func (v *Volume) garbageLevel() float64 { @@ -13,9 +13,10 @@ func (v *Volume) garbageLevel() float64 { func (v *Volume) Compact() error { glog.V(3).Infof("Compacting ...") - v.accessLock.Lock() - defer v.accessLock.Unlock() - glog.V(3).Infof("Got Compaction lock...") + //no need to lock for copy on write + //v.accessLock.Lock() + //defer v.accessLock.Unlock() + //glog.V(3).Infof("Got Compaction lock...") filePath := v.FileName() glog.V(3).Infof("creating copies for volume %d ...", v.Id) @@ -59,10 +60,15 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err erro nm := NewNeedleMap(idx) new_offset := int64(SuperBlockSize) + now := uint64(time.Now().Unix()) + err = ScanVolumeFile(v.dir, v.Collection, v.Id, func(superBlock SuperBlock) error { _, err = dst.Write(superBlock.Bytes()) return err }, true, func(n *Needle, offset int64) error { + if n.HasTtl() && now >= n.LastModified+uint64(v.Ttl.Minutes()*60) { + return nil + } nv, ok := v.nm.Get(n.Id) glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv) if ok && int64(nv.Offset)*NeedlePaddingSize == offset && nv.Size > 0 { diff --git a/go/topology/allocate_volume.go b/go/topology/allocate_volume.go index 77b4ac508..4aeef35f7 100644 --- a/go/topology/allocate_volume.go +++ b/go/topology/allocate_volume.go @@ -12,11 +12,12 @@ type AllocateVolumeResult struct { Error string } -func AllocateVolume(dn *DataNode, vid storage.VolumeId, collection string, rp *storage.ReplicaPlacement) error { +func AllocateVolume(dn *DataNode, vid storage.VolumeId, option *VolumeGrowOption) error { values := make(url.Values) values.Add("volume", vid.String()) - values.Add("collection", collection) - values.Add("replication", rp.String()) + values.Add("collection", option.Collection) + values.Add("replication", option.ReplicaPlacement.String()) + values.Add("ttl", option.Ttl.String()) jsonBlob, err := util.Post("http://"+dn.PublicUrl+"/admin/assign_volume", values) if err != nil { return err diff --git a/go/topology/collection.go b/go/topology/collection.go index b21122d22..c014231af 100644 --- a/go/topology/collection.go +++ b/go/topology/collection.go @@ -1,33 +1,34 @@ package topology import ( - "code.google.com/p/weed-fs/go/glog" "code.google.com/p/weed-fs/go/storage" ) type Collection struct { Name string volumeSizeLimit uint64 - replicaType2VolumeLayout []*VolumeLayout + storageType2VolumeLayout map[string]*VolumeLayout } func NewCollection(name string, volumeSizeLimit uint64) *Collection { c := &Collection{Name: name, volumeSizeLimit: volumeSizeLimit} - c.replicaType2VolumeLayout = make([]*VolumeLayout, storage.ReplicaPlacementCount) + c.storageType2VolumeLayout = make(map[string]*VolumeLayout) return c } -func (c *Collection) GetOrCreateVolumeLayout(rp *storage.ReplicaPlacement) *VolumeLayout { - replicaPlacementIndex := rp.GetReplicationLevelIndex() - if c.replicaType2VolumeLayout[replicaPlacementIndex] == nil { - glog.V(0).Infoln("collection", c.Name, "adding replication type", rp) - c.replicaType2VolumeLayout[replicaPlacementIndex] = NewVolumeLayout(rp, c.volumeSizeLimit) +func (c *Collection) GetOrCreateVolumeLayout(rp *storage.ReplicaPlacement, ttl *storage.TTL) *VolumeLayout { + keyString := rp.String() + if ttl != nil { + keyString += ttl.String() } - return c.replicaType2VolumeLayout[replicaPlacementIndex] + if c.storageType2VolumeLayout[keyString] == nil { + c.storageType2VolumeLayout[keyString] = NewVolumeLayout(rp, ttl, c.volumeSizeLimit) + } + return c.storageType2VolumeLayout[keyString] } func (c *Collection) Lookup(vid storage.VolumeId) []*DataNode { - for _, vl := range c.replicaType2VolumeLayout { + for _, vl := range c.storageType2VolumeLayout { if vl != nil { if list := vl.Lookup(vid); list != nil { return list @@ -38,7 +39,7 @@ func (c *Collection) Lookup(vid storage.VolumeId) []*DataNode { } func (c *Collection) ListVolumeServers() (nodes []*DataNode) { - for _, vl := range c.replicaType2VolumeLayout { + for _, vl := range c.storageType2VolumeLayout { if vl != nil { if list := vl.ListVolumeServers(); list != nil { nodes = append(nodes, list...) diff --git a/go/topology/data_node.go b/go/topology/data_node.go index ae80e08bb..c67c5c1c1 100644 --- a/go/topology/data_node.go +++ b/go/topology/data_node.go @@ -38,15 +38,16 @@ func (dn *DataNode) AddOrUpdateVolume(v storage.VolumeInfo) { } } -func (dn *DataNode) UpdateVolumes(actualVolumes []storage.VolumeInfo) { +func (dn *DataNode) UpdateVolumes(actualVolumes []storage.VolumeInfo) (deletedVolumes []storage.VolumeInfo) { actualVolumeMap := make(map[storage.VolumeId]storage.VolumeInfo) for _, v := range actualVolumes { actualVolumeMap[v.Id] = v } - for vid, _ := range dn.volumes { + for vid, v := range dn.volumes { if _, ok := actualVolumeMap[vid]; !ok { glog.V(0).Infoln("Deleting volume id:", vid) delete(dn.volumes, vid) + deletedVolumes = append(deletedVolumes, v) dn.UpAdjustVolumeCountDelta(-1) dn.UpAdjustActiveVolumeCountDelta(-1) } @@ -54,6 +55,7 @@ func (dn *DataNode) UpdateVolumes(actualVolumes []storage.VolumeInfo) { for _, v := range actualVolumes { dn.AddOrUpdateVolume(v) } + return } func (dn *DataNode) GetDataCenter() *DataCenter { diff --git a/go/topology/topology.go b/go/topology/topology.go index f1daffb53..acdef5e36 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -110,12 +110,12 @@ func (t *Topology) NextVolumeId() storage.VolumeId { } func (t *Topology) HasWriableVolume(option *VolumeGrowOption) bool { - vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement) + vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } func (t *Topology) PickForWrite(count int, option *VolumeGrowOption) (string, int, *DataNode, error) { - vid, count, datanodes, err := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement).PickForWrite(count, option) + vid, count, datanodes, err := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl).PickForWrite(count, option) if err != nil || datanodes.Length() == 0 { return "", 0, nil, errors.New("No writable volumes avalable!") } @@ -123,12 +123,12 @@ func (t *Topology) PickForWrite(count int, option *VolumeGrowOption) (string, in return storage.NewFileId(*vid, fileId, rand.Uint32()).String(), count, datanodes.Head(), nil } -func (t *Topology) GetVolumeLayout(collectionName string, rp *storage.ReplicaPlacement) *VolumeLayout { +func (t *Topology) GetVolumeLayout(collectionName string, rp *storage.ReplicaPlacement, ttl *storage.TTL) *VolumeLayout { _, ok := t.collectionMap[collectionName] if !ok { t.collectionMap[collectionName] = NewCollection(collectionName, t.volumeSizeLimit) } - return t.collectionMap[collectionName].GetOrCreateVolumeLayout(rp) + return t.collectionMap[collectionName].GetOrCreateVolumeLayout(rp, ttl) } func (t *Topology) GetCollection(collectionName string) (collection *Collection, ok bool) { @@ -141,10 +141,14 @@ func (t *Topology) DeleteCollection(collectionName string) { } func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { - t.GetVolumeLayout(v.Collection, v.ReplicaPlacement).RegisterVolume(&v, dn) + t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl).RegisterVolume(&v, dn) +} +func (t *Topology) UnRegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { + glog.Infof("removing volume info:%+v", v) + t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl).UnRegisterVolume(&v, dn) } -func (t *Topology) RegisterVolumes(joinMessage *operation.JoinMessage) { +func (t *Topology) ProcessJoinMessage(joinMessage *operation.JoinMessage) { t.Sequence.SetMax(*joinMessage.MaxFileKey) dcName, rackName := t.configuration.Locate(*joinMessage.Ip, *joinMessage.DataCenter, *joinMessage.Rack) dc := t.GetOrCreateDataCenter(dcName) @@ -162,10 +166,13 @@ func (t *Topology) RegisterVolumes(joinMessage *operation.JoinMessage) { glog.V(0).Infoln("Fail to convert joined volume information:", err.Error()) } } - dn.UpdateVolumes(volumeInfos) + deletedVolumes := dn.UpdateVolumes(volumeInfos) for _, v := range volumeInfos { t.RegisterVolumeLayout(v, dn) } + for _, v := range deletedVolumes { + t.UnRegisterVolumeLayout(v, dn) + } } func (t *Topology) GetOrCreateDataCenter(dcName string) *DataCenter { diff --git a/go/topology/topology_event_handling.go b/go/topology/topology_event_handling.go index 7398ff9bf..1e630e149 100644 --- a/go/topology/topology_event_handling.go +++ b/go/topology/topology_event_handling.go @@ -41,7 +41,7 @@ func (t *Topology) StartRefreshWritableVolumes(garbageThreshold string) { }() } func (t *Topology) SetVolumeCapacityFull(volumeInfo storage.VolumeInfo) bool { - vl := t.GetVolumeLayout(volumeInfo.Collection, volumeInfo.ReplicaPlacement) + vl := t.GetVolumeLayout(volumeInfo.Collection, volumeInfo.ReplicaPlacement, volumeInfo.Ttl) if !vl.SetVolumeCapacityFull(volumeInfo.Id) { return false } @@ -55,7 +55,7 @@ func (t *Topology) SetVolumeCapacityFull(volumeInfo storage.VolumeInfo) bool { func (t *Topology) UnRegisterDataNode(dn *DataNode) { for _, v := range dn.volumes { glog.V(0).Infoln("Removing Volume", v.Id, "from the dead volume server", dn) - vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement) + vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl) vl.SetVolumeUnavailable(dn, v.Id) } dn.UpAdjustVolumeCountDelta(-dn.GetVolumeCount()) @@ -65,7 +65,7 @@ func (t *Topology) UnRegisterDataNode(dn *DataNode) { } func (t *Topology) RegisterRecoveredDataNode(dn *DataNode) { for _, v := range dn.volumes { - vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement) + vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl) if vl.isWritable(&v) { vl.SetVolumeAvailable(dn, v.Id) } diff --git a/go/topology/topology_map.go b/go/topology/topology_map.go index f66d4c251..d6400c988 100644 --- a/go/topology/topology_map.go +++ b/go/topology/topology_map.go @@ -14,7 +14,7 @@ func (t *Topology) ToMap() interface{} { m["DataCenters"] = dcs var layouts []interface{} for _, c := range t.collectionMap { - for _, layout := range c.replicaType2VolumeLayout { + for _, layout := range c.storageType2VolumeLayout { if layout != nil { tmp := layout.ToMap() tmp["collection"] = c.Name diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index a1d6d2564..9eaca37d4 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -80,7 +80,7 @@ func batchVacuumVolumeCommit(vl *VolumeLayout, vid storage.VolumeId, locationlis } func (t *Topology) Vacuum(garbageThreshold string) int { for _, c := range t.collectionMap { - for _, vl := range c.replicaType2VolumeLayout { + for _, vl := range c.storageType2VolumeLayout { if vl != nil { for vid, locationlist := range vl.vid2location { if batchVacuumVolumeCheck(vl, vid, locationlist, garbageThreshold) { diff --git a/go/topology/volume_growth.go b/go/topology/volume_growth.go index 4965e3ba0..778aa038a 100644 --- a/go/topology/volume_growth.go +++ b/go/topology/volume_growth.go @@ -19,6 +19,7 @@ This package is created to resolve these replica placement issues: type VolumeGrowOption struct { Collection string ReplicaPlacement *storage.ReplicaPlacement + Ttl *storage.TTL DataCenter string Rack string DataNode string @@ -184,8 +185,15 @@ func (vg *VolumeGrowth) findEmptySlotsForOneVolume(topo *Topology, option *Volum func (vg *VolumeGrowth) grow(topo *Topology, vid storage.VolumeId, option *VolumeGrowOption, servers ...*DataNode) error { for _, server := range servers { - if err := AllocateVolume(server, vid, option.Collection, option.ReplicaPlacement); err == nil { - vi := storage.VolumeInfo{Id: vid, Size: 0, Collection: option.Collection, ReplicaPlacement: option.ReplicaPlacement, Version: storage.CurrentVersion} + if err := AllocateVolume(server, vid, option); err == nil { + vi := storage.VolumeInfo{ + Id: vid, + Size: 0, + Collection: option.Collection, + ReplicaPlacement: option.ReplicaPlacement, + Ttl: option.Ttl, + Version: storage.CurrentVersion, + } server.AddOrUpdateVolume(vi) topo.RegisterVolumeLayout(vi, server) glog.V(0).Infoln("Created Volume", vid, "on", server) diff --git a/go/topology/volume_layout.go b/go/topology/volume_layout.go index 538acb54c..1e55072a3 100644 --- a/go/topology/volume_layout.go +++ b/go/topology/volume_layout.go @@ -11,15 +11,17 @@ import ( // mapping from volume to its locations, inverted from server to volume type VolumeLayout struct { rp *storage.ReplicaPlacement + ttl *storage.TTL vid2location map[storage.VolumeId]*VolumeLocationList writables []storage.VolumeId // transient array of writable volume id volumeSizeLimit uint64 accessLock sync.Mutex } -func NewVolumeLayout(rp *storage.ReplicaPlacement, volumeSizeLimit uint64) *VolumeLayout { +func NewVolumeLayout(rp *storage.ReplicaPlacement, ttl *storage.TTL, volumeSizeLimit uint64) *VolumeLayout { return &VolumeLayout{ rp: rp, + ttl: ttl, vid2location: make(map[storage.VolumeId]*VolumeLocationList), writables: *new([]storage.VolumeId), volumeSizeLimit: volumeSizeLimit, @@ -42,6 +44,14 @@ func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) { } } +func (vl *VolumeLayout) UnRegisterVolume(v *storage.VolumeInfo, dn *DataNode) { + vl.accessLock.Lock() + defer vl.accessLock.Unlock() + + vl.removeFromWritable(v.Id) + delete(vl.vid2location, v.Id) +} + func (vl *VolumeLayout) AddToWritable(vid storage.VolumeId) { for _, id := range vl.writables { if vid == id { @@ -192,6 +202,7 @@ func (vl *VolumeLayout) SetVolumeCapacityFull(vid storage.VolumeId) bool { func (vl *VolumeLayout) ToMap() map[string]interface{} { m := make(map[string]interface{}) m["replication"] = vl.rp.String() + m["ttl"] = vl.ttl.String() m["writables"] = vl.writables //m["locations"] = vl.vid2location return m diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index eab923751..27aebaef0 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -201,7 +201,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) { start := time.Now() fileSize := int64(*b.fileSize + rand.Intn(64)) fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize} - if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection); err == nil { + if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil { fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection if _, ok := serverLimitChan[fp.Server]; !ok { serverLimitChan[fp.Server] = make(chan bool, 7) diff --git a/go/weed/compact.go b/go/weed/compact.go index 580f3f98d..57a02261f 100644 --- a/go/weed/compact.go +++ b/go/weed/compact.go @@ -33,7 +33,7 @@ func runCompact(cmd *Command, args []string) bool { } vid := storage.VolumeId(*compactVolumeId) - v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid, nil) + v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid, nil, nil) if err != nil { glog.Fatalf("Load Volume [ERROR] %s\n", err) } diff --git a/go/weed/upload.go b/go/weed/upload.go index b59313a2a..c21499dd0 100644 --- a/go/weed/upload.go +++ b/go/weed/upload.go @@ -12,6 +12,7 @@ var ( uploadReplication *string uploadCollection *string uploadDir *string + uploadTtl *string include *string maxMB *int ) @@ -24,6 +25,7 @@ func init() { include = cmdUpload.Flag.String("include", "", "pattens of files to upload, e.g., *.pdf, *.html, ab?d.txt, works together with -dir") uploadReplication = cmdUpload.Flag.String("replication", "", "replication type") uploadCollection = cmdUpload.Flag.String("collection", "", "optional collection name") + uploadTtl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y") maxMB = cmdUpload.Flag.Int("maxMB", 0, "split files larger than the limit") } @@ -67,7 +69,7 @@ func runUpload(cmd *Command, args []string) bool { if e != nil { return e } - results, e := operation.SubmitFiles(*server, parts, *uploadReplication, *uploadCollection, *maxMB) + results, e := operation.SubmitFiles(*server, parts, *uploadReplication, *uploadCollection, *uploadTtl, *maxMB) bytes, _ := json.Marshal(results) fmt.Println(string(bytes)) if e != nil { @@ -84,7 +86,7 @@ func runUpload(cmd *Command, args []string) bool { if e != nil { fmt.Println(e.Error()) } - results, _ := operation.SubmitFiles(*server, parts, *uploadReplication, *uploadCollection, *maxMB) + results, _ := operation.SubmitFiles(*server, parts, *uploadReplication, *uploadCollection, *uploadTtl, *maxMB) bytes, _ := json.Marshal(results) fmt.Println(string(bytes)) } diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index a547d7462..49e84378c 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -99,14 +99,14 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st } debug("parsing upload file...") - fname, data, mimeType, isGzipped, lastModified, pe := storage.ParseUpload(r) + fname, data, mimeType, isGzipped, lastModified, _, pe := storage.ParseUpload(r) if pe != nil { writeJsonError(w, r, pe) return } debug("assigning file id for", fname) - assignResult, ae := operation.Assign(masterUrl, 1, r.FormValue("replication"), r.FormValue("collection")) + assignResult, ae := operation.Assign(masterUrl, 1, r.FormValue("replication"), r.FormValue("collection"), r.FormValue("ttl")) if ae != nil { writeJsonError(w, r, ae) return diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 0f83352a9..f760030f3 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -109,7 +109,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() - assignResult, ae := operation.Assign(fs.master, 1, query.Get("replication"), fs.collection) + assignResult, ae := operation.Assign(fs.master, 1, query.Get("replication"), fs.collection, query.Get("ttl")) if ae != nil { glog.V(0).Infoln("failing to assign a file id", ae.Error()) writeJsonError(w, r, ae) @@ -131,14 +131,14 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { } resp, do_err := util.Do(request) if do_err != nil { - glog.V(0).Infoln("failing to connect to volume server", do_err.Error()) + glog.V(0).Infoln("failing to connect to volume server", r.RequestURI, do_err.Error()) writeJsonError(w, r, do_err) return } defer resp.Body.Close() resp_body, ra_err := ioutil.ReadAll(resp.Body) if ra_err != nil { - glog.V(0).Infoln("failing to upload to volume server", ra_err.Error()) + glog.V(0).Infoln("failing to upload to volume server", r.RequestURI, ra_err.Error()) writeJsonError(w, r, ra_err) return } @@ -146,12 +146,12 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { var ret operation.UploadResult unmarshal_err := json.Unmarshal(resp_body, &ret) if unmarshal_err != nil { - glog.V(0).Infoln("failing to read upload resonse", string(resp_body)) + glog.V(0).Infoln("failing to read upload resonse", r.RequestURI, string(resp_body)) writeJsonError(w, r, unmarshal_err) return } if ret.Error != "" { - glog.V(0).Infoln("failing to post to volume server", ret.Error) + glog.V(0).Infoln("failing to post to volume server", r.RequestURI, ret.Error) writeJsonError(w, r, errors.New(ret.Error)) return } @@ -169,7 +169,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { glog.V(4).Infoln("saving", path, "=>", assignResult.Fid) if db_err := fs.filer.CreateFile(path, assignResult.Fid); db_err != nil { operation.DeleteFile(fs.master, assignResult.Fid) //clean up - glog.V(0).Infoln("failing to write to filer server", db_err.Error()) + glog.V(0).Infoln("failing to write to filer server", r.RequestURI, db_err.Error()) writeJsonError(w, r, db_err) return } diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index d50075fd5..1458bf3e6 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -55,7 +55,7 @@ func (ms *MasterServer) dirJoinHandler(w http.ResponseWriter, r *http.Request) { } } - ms.Topo.RegisterVolumes(joinMessage) + ms.Topo.ProcessJoinMessage(joinMessage) writeJsonQuiet(w, r, operation.JoinResult{VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024}) } @@ -144,7 +144,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r * } func (ms *MasterServer) hasWriableVolume(option *topology.VolumeGrowOption) bool { - vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement) + vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } @@ -157,9 +157,14 @@ func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGr if err != nil { return nil, err } + ttl, err := storage.ReadTTL(r.FormValue("ttl")) + if err != nil { + return nil, err + } volumeGrowOption := &topology.VolumeGrowOption{ Collection: r.FormValue("collection"), ReplicaPlacement: replicaPlacement, + Ttl: ttl, DataCenter: r.FormValue("dataCenter"), Rack: r.FormValue("rack"), DataNode: r.FormValue("dataNode"), diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go index 6d285524a..1c01fdfd0 100644 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ b/go/weed/weed_server/volume_server_handlers_admin.go @@ -16,7 +16,7 @@ func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { } func (vs *VolumeServer) assignVolumeHandler(w http.ResponseWriter, r *http.Request) { - err := vs.store.AddVolume(r.FormValue("volume"), r.FormValue("collection"), r.FormValue("replication")) + err := vs.store.AddVolume(r.FormValue("volume"), r.FormValue("collection"), r.FormValue("replication"), r.FormValue("ttl")) if err == nil { writeJsonQuiet(w, r, map[string]string{"error": ""}) } else { From f7094d7a99f73db855f59afafbf6e1dd276ba394 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 20 Sep 2014 12:39:42 -0700 Subject: [PATCH 10/58] version 0.64 --- go/util/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/util/constants.go b/go/util/constants.go index f66930d62..db1ca38e5 100644 --- a/go/util/constants.go +++ b/go/util/constants.go @@ -3,5 +3,5 @@ package util import () const ( - VERSION = "0.63" + VERSION = "0.64" ) From 7920b4685e41407c3bcc64fd22579cfe0f27bda8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 20 Sep 2014 20:51:24 -0700 Subject: [PATCH 11/58] Adding unit tests for volume ttl. --- go/storage/needle.go | 2 +- go/storage/volume_ttl.go | 16 ++-------- go/storage/volume_ttl_test.go | 60 +++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 go/storage/volume_ttl_test.go diff --git a/go/storage/needle.go b/go/storage/needle.go index 3bf627141..51079376a 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -118,7 +118,7 @@ func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) { n.LastModified = uint64(time.Now().Unix()) } n.SetHasLastModifiedDate() - if n.Ttl != nil { + if n.Ttl != EMPTY_TTL { n.SetHasTtl() } diff --git a/go/storage/volume_ttl.go b/go/storage/volume_ttl.go index 5ff43e24e..459ee55ba 100644 --- a/go/storage/volume_ttl.go +++ b/go/storage/volume_ttl.go @@ -4,10 +4,6 @@ import ( "strconv" ) -var ( - TtlRange []uint16 -) - const ( //stored unit types Empty byte = iota @@ -19,14 +15,6 @@ const ( Year ) -func init() { - TtlRange = []uint16{3, 10, 30, - 60 /*1 hour*/, 60 * 3, 60 * 6, 60 * 12, - 1440 /*1 day*/, 1440 * 3, 1440 * 7, 1440 * 15, 1440 * 31, - 44888 /*1 month*/, 65535, - } -} - type TTL struct { count byte unit byte @@ -120,7 +108,7 @@ func toStoredByte(readableUnitByte byte) byte { return Week case 'M': return Month - case 'Y': + case 'y': return Year } return 0 @@ -141,7 +129,7 @@ func (t TTL) Minutes() uint32 { case Month: return uint32(t.count) * 60 * 24 * 31 case Year: - return uint32(t.count) * 60 * 24 * 31 * 365 + return uint32(t.count) * 60 * 24 * 365 } return 0 } diff --git a/go/storage/volume_ttl_test.go b/go/storage/volume_ttl_test.go new file mode 100644 index 000000000..216469a4c --- /dev/null +++ b/go/storage/volume_ttl_test.go @@ -0,0 +1,60 @@ +package storage + +import ( + "testing" +) + +func TestTTLReadWrite(t *testing.T) { + ttl, _ := ReadTTL("") + if ttl.Minutes() != 0 { + t.Errorf("empty ttl:%v", ttl) + } + + ttl, _ = ReadTTL("9") + if ttl.Minutes() != 9 { + t.Errorf("9 ttl:%v", ttl) + } + + ttl, _ = ReadTTL("8m") + if ttl.Minutes() != 8 { + t.Errorf("8m ttl:%v", ttl) + } + + ttl, _ = ReadTTL("5h") + if ttl.Minutes() != 300 { + t.Errorf("5h ttl:%v", ttl) + } + + ttl, _ = ReadTTL("5d") + if ttl.Minutes() != 5*24*60 { + t.Errorf("5d ttl:%v", ttl) + } + + ttl, _ = ReadTTL("5w") + if ttl.Minutes() != 5*7*24*60 { + t.Errorf("5w ttl:%v", ttl) + } + + ttl, _ = ReadTTL("5M") + if ttl.Minutes() != 5*31*24*60 { + t.Errorf("5M ttl:%v", ttl) + } + + ttl, _ = ReadTTL("5y") + if ttl.Minutes() != 5*365*24*60 { + t.Errorf("5y ttl:%v", ttl) + } + + output := make([]byte, 2) + ttl.ToBytes(output) + ttl2 := LoadTTLFromBytes(output) + if ttl.Minutes() != ttl2.Minutes() { + t.Errorf("ttl:%v ttl2:%v", ttl, ttl2) + } + + ttl3 := LoadTTLFromUint32(ttl.ToUint32()) + if ttl.Minutes() != ttl3.Minutes() { + t.Errorf("ttl:%v ttl3:%v", ttl, ttl3) + } + +} From 794b755f1d6b26e044627135d91f0d49388a36d5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 20 Sep 2014 21:18:26 -0700 Subject: [PATCH 12/58] rename to Seaweed File System --- go/weed/benchmark.go | 2 +- go/weed/filer.go | 2 +- go/weed/mount_std.go | 2 +- go/weed/server.go | 2 +- go/weed/version.go | 4 ++-- go/weed/weed.go | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index 27aebaef0..7e98fd177 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -98,7 +98,7 @@ func init() { } func runbenchmark(cmd *Command, args []string) bool { - fmt.Printf("This is Weed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) + fmt.Printf("This is Seaweed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) if *b.cpuprofile != "" { f, err := os.Create(*b.cpuprofile) if err != nil { diff --git a/go/weed/filer.go b/go/weed/filer.go index d7d028d50..da54a91fb 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -63,7 +63,7 @@ func runFiler(cmd *Command, args []string) bool { if nfs_err != nil { glog.Fatalf(nfs_err.Error()) } - glog.V(0).Infoln("Start Weed Filer", util.VERSION, "at port", strconv.Itoa(*f.port)) + glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*f.port)) filerListener, e := util.NewListener( ":"+strconv.Itoa(*f.port), time.Duration(10)*time.Second, diff --git a/go/weed/mount_std.go b/go/weed/mount_std.go index 9376b3f2e..5b058fe89 100644 --- a/go/weed/mount_std.go +++ b/go/weed/mount_std.go @@ -15,7 +15,7 @@ import ( ) func runMount(cmd *Command, args []string) bool { - fmt.Printf("This is Weed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) + fmt.Printf("This is Seaweed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) if *mountOptions.dir == "" { fmt.Printf("Please specify the mount directory via \"-dir\"") return false diff --git a/go/weed/server.go b/go/weed/server.go index 8b0372159..66fbaa120 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -152,7 +152,7 @@ func runServer(cmd *Command, args []string) bool { if nfs_err != nil { glog.Fatalf(nfs_err.Error()) } - glog.V(0).Infoln("Start Weed Filer", util.VERSION, "at port", strconv.Itoa(*filerOptions.port)) + glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*filerOptions.port)) filerListener, e := util.NewListener( ":"+strconv.Itoa(*filerOptions.port), time.Duration(10)*time.Second, diff --git a/go/weed/version.go b/go/weed/version.go index 2e0a59822..a7c24ea3b 100644 --- a/go/weed/version.go +++ b/go/weed/version.go @@ -9,8 +9,8 @@ import ( var cmdVersion = &Command{ Run: runVersion, UsageLine: "version", - Short: "print Weed File System version", - Long: `Version prints the Weed File System version`, + Short: "print Seaweed File System version", + Long: `Version prints the Seaweed File System version`, } func runVersion(cmd *Command, args []string) bool { diff --git a/go/weed/weed.go b/go/weed/weed.go index c739d8e93..03cf29179 100644 --- a/go/weed/weed.go +++ b/go/weed/weed.go @@ -90,7 +90,7 @@ func main() { } var usageTemplate = ` -Weed File System : store billions of files and serve them fast! +Seaweed File System : store billions of files and serve them fast! Usage: From ca16cfa14715b80a6a41e8bf047eea41fabe7818 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 20 Sep 2014 22:10:58 -0700 Subject: [PATCH 13/58] adjust for external API changes --- go/images/orientation.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/go/images/orientation.go b/go/images/orientation.go index a7c126fc1..41ed3f0af 100644 --- a/go/images/orientation.go +++ b/go/images/orientation.go @@ -21,7 +21,10 @@ func FixJpgOrientation(data []byte) (oriented []byte) { } angle := 0 flipMode := FlipDirection(0) - orient := tag.Int(0) + orient, err := tag.Int(0) + if err != nil { + return data + } switch orient { case topLeftSide: // do nothing From ba179b154259ef2790e30f068c4368658de7c512 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 20 Sep 2014 23:30:35 -0700 Subject: [PATCH 14/58] Add "-ip.bind" option when starting volume servers. Also some Weed->Seaweed changes. --- go/weed/master.go | 2 +- go/weed/server.go | 9 +++++---- go/weed/volume.go | 5 +++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/go/weed/master.go b/go/weed/master.go index b95ca5cb1..ef2c062b5 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -63,7 +63,7 @@ func runMaster(cmd *Command, args []string) bool { listeningAddress := *masterIp + ":" + strconv.Itoa(*mport) - glog.V(0).Infoln("Start Weed Master", util.VERSION, "at", listeningAddress) + glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress) listener, e := util.NewListener(listeningAddress, time.Duration(*mTimeout)*time.Second) if e != nil { diff --git a/go/weed/server.go b/go/weed/server.go index 66fbaa120..32bad976a 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -48,6 +48,7 @@ var cmdServer = &Command{ var ( serverIp = cmdServer.Flag.String("ip", "", "ip or server name") serverPublicIp = cmdServer.Flag.String("publicIp", "", "ip or server name") + serverBindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") serverMaxCpu = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") serverTimeout = cmdServer.Flag.Int("idleTimeout", 10, "connection idle seconds") serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name") @@ -178,8 +179,8 @@ func runServer(cmd *Command, args []string) bool { *masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold, serverWhiteList, ) - glog.V(0).Infoln("Start Weed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort)) - masterListener, e := util.NewListener(*serverIp+":"+strconv.Itoa(*masterPort), time.Duration(*serverTimeout)*time.Second) + glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort)) + masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), time.Duration(*serverTimeout)*time.Second) if e != nil { glog.Fatalf(e.Error()) } @@ -211,9 +212,9 @@ func runServer(cmd *Command, args []string) bool { *volumeFixJpgOrientation, ) - glog.V(0).Infoln("Start Weed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort)) + glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort)) volumeListener, e := util.NewListener( - *serverIp+":"+strconv.Itoa(*volumePort), + *serverBindIp+":"+strconv.Itoa(*volumePort), time.Duration(*serverTimeout)*time.Second, ) if e != nil { diff --git a/go/weed/volume.go b/go/weed/volume.go index 8aa52c43d..7a57730cd 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -30,6 +30,7 @@ var ( maxVolumeCounts = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...") ip = cmdVolume.Flag.String("ip", "", "ip or server name") publicIp = cmdVolume.Flag.String("publicIp", "", "Publicly accessible ") + bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") masterNode = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location") vpulse = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting") vTimeout = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds") @@ -84,9 +85,9 @@ func runVolume(cmd *Command, args []string) bool { *fixJpgOrientation, ) - listeningAddress := *ip + ":" + strconv.Itoa(*vport) + listeningAddress := *bindIp + ":" + strconv.Itoa(*vport) - glog.V(0).Infoln("Start Weed volume server", util.VERSION, "at", listeningAddress) + glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress) listener, e := util.NewListener(listeningAddress, time.Duration(*vTimeout)*time.Second) if e != nil { From a2f8d985fbcddf9722015dfafe8a9ddffcf55fc0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 20 Sep 2014 23:34:13 -0700 Subject: [PATCH 15/58] Fix typo. --- go/weed/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/weed/server.go b/go/weed/server.go index 32bad976a..57fde6757 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -48,7 +48,7 @@ var cmdServer = &Command{ var ( serverIp = cmdServer.Flag.String("ip", "", "ip or server name") serverPublicIp = cmdServer.Flag.String("publicIp", "", "ip or server name") - serverBindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") + serverBindIp = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") serverMaxCpu = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") serverTimeout = cmdServer.Flag.Int("idleTimeout", 10, "connection idle seconds") serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name") From 1cd19447e328b01427ad5b4d5d29b3323432c4af Mon Sep 17 00:00:00 2001 From: wyy Date: Thu, 25 Sep 2014 00:47:09 +0800 Subject: [PATCH 16/58] use github.com/aszxqw instead of code.google.com/p --- go/filer/client_operations.go | 2 +- go/filer/directory_in_map.go | 2 +- go/filer/filer_embedded.go | 2 +- go/filer/files_in_leveldb.go | 2 +- go/operation/assign_file_id.go | 4 ++-- go/operation/delete_content.go | 2 +- go/operation/list_masters.go | 4 ++-- go/operation/lookup.go | 2 +- go/operation/submit.go | 2 +- go/operation/upload_content.go | 2 +- go/storage/cdb_map.go | 2 +- go/storage/cdb_map_test.go | 2 +- go/storage/compact_map_perf_test.go | 4 ++-- go/storage/compress.go | 2 +- go/storage/crc.go | 2 +- go/storage/file_id.go | 4 ++-- go/storage/needle.go | 6 +++--- go/storage/needle_map.go | 4 ++-- go/storage/needle_read_write.go | 4 ++-- go/storage/store.go | 6 +++--- go/storage/store_vacuum.go | 2 +- go/storage/volume.go | 2 +- go/storage/volume_info.go | 2 +- go/storage/volume_super_block.go | 2 +- go/storage/volume_vacuum.go | 2 +- go/tools/read_index.go | 2 +- go/topology/allocate_volume.go | 4 ++-- go/topology/cluster_commands.go | 4 ++-- go/topology/collection.go | 2 +- go/topology/data_node.go | 4 ++-- go/topology/node.go | 4 ++-- go/topology/store_replicate.go | 8 ++++---- go/topology/topology.go | 8 ++++---- go/topology/topology_event_handling.go | 4 ++-- go/topology/topology_vacuum.go | 6 +++--- go/topology/volume_growth.go | 4 ++-- go/topology/volume_growth_test.go | 4 ++-- go/topology/volume_layout.go | 4 ++-- go/util/config.go | 2 +- go/util/file_util.go | 2 +- go/util/net_timeout.go | 2 +- go/weed/benchmark.go | 6 +++--- go/weed/compact.go | 4 ++-- go/weed/download.go | 4 ++-- go/weed/export.go | 4 ++-- go/weed/filer.go | 6 +++--- go/weed/fix.go | 4 ++-- go/weed/master.go | 6 +++--- go/weed/mount_std.go | 8 ++++---- go/weed/server.go | 6 +++--- go/weed/shell.go | 2 +- go/weed/upload.go | 2 +- go/weed/version.go | 2 +- go/weed/volume.go | 6 +++--- go/weed/volume_test.go | 2 +- go/weed/weed.go | 2 +- go/weed/weed_server/common.go | 10 +++++----- go/weed/weed_server/filer_server.go | 4 ++-- go/weed/weed_server/filer_server_handlers.go | 6 +++--- go/weed/weed_server/filer_server_handlers_admin.go | 2 +- go/weed/weed_server/master_server.go | 8 ++++---- go/weed/weed_server/master_server_handlers.go | 6 +++--- go/weed/weed_server/master_server_handlers_admin.go | 10 +++++----- go/weed/weed_server/raft_server.go | 4 ++-- go/weed/weed_server/raft_server_handlers.go | 4 ++-- go/weed/weed_server/volume_server.go | 4 ++-- go/weed/weed_server/volume_server_handlers.go | 12 ++++++------ go/weed/weed_server/volume_server_handlers_admin.go | 6 +++--- go/weed/weed_server/volume_server_handlers_vacuum.go | 2 +- 69 files changed, 138 insertions(+), 138 deletions(-) diff --git a/go/filer/client_operations.go b/go/filer/client_operations.go index 5d89fc865..d9a9501a5 100644 --- a/go/filer/client_operations.go +++ b/go/filer/client_operations.go @@ -3,7 +3,7 @@ package filer import () import ( - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/util" "encoding/json" "errors" "fmt" diff --git a/go/filer/directory_in_map.go b/go/filer/directory_in_map.go index 35b4e53c1..85cde765d 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/directory_in_map.go @@ -2,7 +2,7 @@ package filer import ( "bufio" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/util" "fmt" "io" "os" diff --git a/go/filer/filer_embedded.go b/go/filer/filer_embedded.go index a3b64d37b..d716802ac 100644 --- a/go/filer/filer_embedded.go +++ b/go/filer/filer_embedded.go @@ -1,7 +1,7 @@ package filer import ( - "code.google.com/p/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/operation" "errors" "fmt" "path/filepath" diff --git a/go/filer/files_in_leveldb.go b/go/filer/files_in_leveldb.go index dbf2e6c52..17d726dd7 100644 --- a/go/filer/files_in_leveldb.go +++ b/go/filer/files_in_leveldb.go @@ -2,7 +2,7 @@ package filer import ( "bytes" - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" ) diff --git a/go/operation/assign_file_id.go b/go/operation/assign_file_id.go index 34d371f37..891653278 100644 --- a/go/operation/assign_file_id.go +++ b/go/operation/assign_file_id.go @@ -1,8 +1,8 @@ package operation import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/util" "encoding/json" "errors" "net/url" diff --git a/go/operation/delete_content.go b/go/operation/delete_content.go index e4f9d39bb..a1485a2db 100644 --- a/go/operation/delete_content.go +++ b/go/operation/delete_content.go @@ -1,7 +1,7 @@ package operation import ( - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/util" "encoding/json" "errors" "net/url" diff --git a/go/operation/list_masters.go b/go/operation/list_masters.go index 491c79f20..120ba58a1 100644 --- a/go/operation/list_masters.go +++ b/go/operation/list_masters.go @@ -1,8 +1,8 @@ package operation import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/util" "encoding/json" ) diff --git a/go/operation/lookup.go b/go/operation/lookup.go index f191bfdbf..1b5d57e2f 100644 --- a/go/operation/lookup.go +++ b/go/operation/lookup.go @@ -1,7 +1,7 @@ package operation import ( - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/util" "encoding/json" "errors" _ "fmt" diff --git a/go/operation/submit.go b/go/operation/submit.go index ec45cc320..18bc593a3 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -2,7 +2,7 @@ package operation import ( "bytes" - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "io" "mime" "os" diff --git a/go/operation/upload_content.go b/go/operation/upload_content.go index b89e65ce8..3812a7d86 100644 --- a/go/operation/upload_content.go +++ b/go/operation/upload_content.go @@ -2,7 +2,7 @@ package operation import ( "bytes" - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "encoding/json" "errors" "fmt" diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go index d09a87e2a..07cbca1a9 100644 --- a/go/storage/cdb_map.go +++ b/go/storage/cdb_map.go @@ -1,7 +1,7 @@ package storage import ( - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/util" "encoding/json" "errors" "fmt" diff --git a/go/storage/cdb_map_test.go b/go/storage/cdb_map_test.go index f6a7d42ad..25f6067cb 100644 --- a/go/storage/cdb_map_test.go +++ b/go/storage/cdb_map_test.go @@ -1,7 +1,7 @@ package storage import ( - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "math/rand" "os" "runtime" diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index 37b23a59f..4795ead5a 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -1,8 +1,8 @@ package storage import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/util" "log" "os" "testing" diff --git a/go/storage/compress.go b/go/storage/compress.go index 846fd0714..7390b85dd 100644 --- a/go/storage/compress.go +++ b/go/storage/compress.go @@ -2,7 +2,7 @@ package storage import ( "bytes" - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "compress/flate" "compress/gzip" "io/ioutil" diff --git a/go/storage/crc.go b/go/storage/crc.go index 43e65757a..9b77c30c6 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -1,7 +1,7 @@ package storage import ( - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/util" "fmt" "hash/crc32" ) diff --git a/go/storage/file_id.go b/go/storage/file_id.go index 5fcd8c387..435a54202 100644 --- a/go/storage/file_id.go +++ b/go/storage/file_id.go @@ -1,8 +1,8 @@ package storage import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/util" "encoding/hex" "errors" "strings" diff --git a/go/storage/needle.go b/go/storage/needle.go index 51079376a..803bc624e 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -1,9 +1,9 @@ package storage import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/images" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/images" + "github.com/aszxqw/weed-fs/go/util" "encoding/hex" "errors" "io/ioutil" diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index 6d94ee1ca..90f3eff7d 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -1,8 +1,8 @@ package storage import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/util" "fmt" "io" "os" diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index 848121ff2..d298c806b 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -1,8 +1,8 @@ package storage import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/util" "errors" "fmt" "io" diff --git a/go/storage/store.go b/go/storage/store.go index ef38ade98..f5b252ad3 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -2,9 +2,9 @@ package storage import ( proto "code.google.com/p/goprotobuf/proto" - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/operation" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/util" "encoding/json" "errors" "fmt" diff --git a/go/storage/store_vacuum.go b/go/storage/store_vacuum.go index 5adaa7561..b3f13fd2d 100644 --- a/go/storage/store_vacuum.go +++ b/go/storage/store_vacuum.go @@ -1,7 +1,7 @@ package storage import ( - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "fmt" "strconv" ) diff --git a/go/storage/volume.go b/go/storage/volume.go index 34ae7e386..10fd145e9 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -2,7 +2,7 @@ package storage import ( "bytes" - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "errors" "fmt" "io" diff --git a/go/storage/volume_info.go b/go/storage/volume_info.go index 6a954f743..81dff4a11 100644 --- a/go/storage/volume_info.go +++ b/go/storage/volume_info.go @@ -1,7 +1,7 @@ package storage import ( - "code.google.com/p/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/operation" ) type VolumeInfo struct { diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index 3fbef44d6..da510ee30 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -1,7 +1,7 @@ package storage import ( - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "fmt" "os" ) diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go index 706a1f951..7feb1c256 100644 --- a/go/storage/volume_vacuum.go +++ b/go/storage/volume_vacuum.go @@ -1,7 +1,7 @@ package storage import ( - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "fmt" "os" "time" diff --git a/go/tools/read_index.go b/go/tools/read_index.go index a958de410..36d39a381 100644 --- a/go/tools/read_index.go +++ b/go/tools/read_index.go @@ -1,7 +1,7 @@ package main import ( - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/storage" "flag" "fmt" "log" diff --git a/go/topology/allocate_volume.go b/go/topology/allocate_volume.go index 4aeef35f7..eaeae4427 100644 --- a/go/topology/allocate_volume.go +++ b/go/topology/allocate_volume.go @@ -1,8 +1,8 @@ package topology import ( - "code.google.com/p/weed-fs/go/storage" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/util" "encoding/json" "errors" "net/url" diff --git a/go/topology/cluster_commands.go b/go/topology/cluster_commands.go index 703435173..369c0b9b9 100644 --- a/go/topology/cluster_commands.go +++ b/go/topology/cluster_commands.go @@ -1,8 +1,8 @@ package topology import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/storage" "github.com/goraft/raft" ) diff --git a/go/topology/collection.go b/go/topology/collection.go index c014231af..72ac86522 100644 --- a/go/topology/collection.go +++ b/go/topology/collection.go @@ -1,7 +1,7 @@ package topology import ( - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/storage" ) type Collection struct { diff --git a/go/topology/data_node.go b/go/topology/data_node.go index c67c5c1c1..5c16cf85d 100644 --- a/go/topology/data_node.go +++ b/go/topology/data_node.go @@ -1,8 +1,8 @@ package topology import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/storage" "strconv" ) diff --git a/go/topology/node.go b/go/topology/node.go index c52414008..12e9a709a 100644 --- a/go/topology/node.go +++ b/go/topology/node.go @@ -1,8 +1,8 @@ package topology import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/storage" "errors" "math/rand" "strings" diff --git a/go/topology/store_replicate.go b/go/topology/store_replicate.go index a982cebe5..fefaea8a6 100644 --- a/go/topology/store_replicate.go +++ b/go/topology/store_replicate.go @@ -2,10 +2,10 @@ package topology import ( "bytes" - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/operation" - "code.google.com/p/weed-fs/go/storage" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/util" "net/http" "strconv" ) diff --git a/go/topology/topology.go b/go/topology/topology.go index acdef5e36..77c9558bb 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -1,10 +1,10 @@ package topology import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/operation" - "code.google.com/p/weed-fs/go/sequence" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/sequence" + "github.com/aszxqw/weed-fs/go/storage" "errors" "github.com/goraft/raft" "io/ioutil" diff --git a/go/topology/topology_event_handling.go b/go/topology/topology_event_handling.go index 1e630e149..61e6f5ffa 100644 --- a/go/topology/topology_event_handling.go +++ b/go/topology/topology_event_handling.go @@ -1,8 +1,8 @@ package topology import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/storage" "math/rand" "time" ) diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index 9eaca37d4..bdc0c38f9 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -1,9 +1,9 @@ package topology import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/storage" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/util" "encoding/json" "errors" "net/url" diff --git a/go/topology/volume_growth.go b/go/topology/volume_growth.go index 778aa038a..e083a8531 100644 --- a/go/topology/volume_growth.go +++ b/go/topology/volume_growth.go @@ -1,8 +1,8 @@ package topology import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/storage" "fmt" "math/rand" "sync" diff --git a/go/topology/volume_growth_test.go b/go/topology/volume_growth_test.go index d2913ef5c..ba1e43ff2 100644 --- a/go/topology/volume_growth_test.go +++ b/go/topology/volume_growth_test.go @@ -1,8 +1,8 @@ package topology import ( - "code.google.com/p/weed-fs/go/sequence" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/sequence" + "github.com/aszxqw/weed-fs/go/storage" "encoding/json" "fmt" "testing" diff --git a/go/topology/volume_layout.go b/go/topology/volume_layout.go index 1e55072a3..35f45c08d 100644 --- a/go/topology/volume_layout.go +++ b/go/topology/volume_layout.go @@ -1,8 +1,8 @@ package topology import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/storage" "errors" "math/rand" "sync" diff --git a/go/util/config.go b/go/util/config.go index 9a1ac680b..1287ed881 100644 --- a/go/util/config.go +++ b/go/util/config.go @@ -10,7 +10,7 @@ package util import ( "bytes" - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "encoding/json" "os" ) diff --git a/go/util/file_util.go b/go/util/file_util.go index 9f3354011..82f6bffe3 100644 --- a/go/util/file_util.go +++ b/go/util/file_util.go @@ -2,7 +2,7 @@ package util import ( "bufio" - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "errors" "os" ) diff --git a/go/util/net_timeout.go b/go/util/net_timeout.go index a6cc81c99..ec9796e63 100644 --- a/go/util/net_timeout.go +++ b/go/util/net_timeout.go @@ -1,7 +1,7 @@ package util import ( - "code.google.com/p/weed-fs/go/stats" + "github.com/aszxqw/weed-fs/go/stats" "net" "time" ) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index 7e98fd177..e33d37816 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -2,9 +2,9 @@ package main import ( "bufio" - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/operation" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/util" "fmt" "io" "math" diff --git a/go/weed/compact.go b/go/weed/compact.go index 57a02261f..1006701aa 100644 --- a/go/weed/compact.go +++ b/go/weed/compact.go @@ -1,8 +1,8 @@ package main import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/storage" ) func init() { diff --git a/go/weed/download.go b/go/weed/download.go index 4309fe5e0..3b3df6d61 100644 --- a/go/weed/download.go +++ b/go/weed/download.go @@ -1,8 +1,8 @@ package main import ( - "code.google.com/p/weed-fs/go/operation" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/util" "fmt" "io" "io/ioutil" diff --git a/go/weed/export.go b/go/weed/export.go index f79ab1a5f..e24108220 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -3,8 +3,8 @@ package main import ( "archive/tar" "bytes" - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/storage" "fmt" "os" "path" diff --git a/go/weed/filer.go b/go/weed/filer.go index da54a91fb..0301d1e5c 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -1,9 +1,9 @@ package main import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/util" - "code.google.com/p/weed-fs/go/weed/weed_server" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/weed/weed_server" "net/http" "os" "strconv" diff --git a/go/weed/fix.go b/go/weed/fix.go index 02147d796..f07d9269f 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -1,8 +1,8 @@ package main import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/storage" "os" "path" "strconv" diff --git a/go/weed/master.go b/go/weed/master.go index ef2c062b5..046cbff74 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -1,9 +1,9 @@ package main import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/util" - "code.google.com/p/weed-fs/go/weed/weed_server" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/weed/weed_server" "github.com/gorilla/mux" "net/http" "os" diff --git a/go/weed/mount_std.go b/go/weed/mount_std.go index 5b058fe89..2df47ad62 100644 --- a/go/weed/mount_std.go +++ b/go/weed/mount_std.go @@ -5,10 +5,10 @@ package main import ( "bazil.org/fuse" "bazil.org/fuse/fs" - "code.google.com/p/weed-fs/go/filer" - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/storage" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/filer" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/util" "fmt" "os" "runtime" diff --git a/go/weed/server.go b/go/weed/server.go index 57fde6757..624c1ca84 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -1,9 +1,9 @@ package main import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/util" - "code.google.com/p/weed-fs/go/weed/weed_server" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/weed/weed_server" "github.com/gorilla/mux" "net/http" "os" diff --git a/go/weed/shell.go b/go/weed/shell.go index 6a3331284..16a414deb 100644 --- a/go/weed/shell.go +++ b/go/weed/shell.go @@ -2,7 +2,7 @@ package main import ( "bufio" - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "fmt" "os" ) diff --git a/go/weed/upload.go b/go/weed/upload.go index c21499dd0..dee013645 100644 --- a/go/weed/upload.go +++ b/go/weed/upload.go @@ -1,7 +1,7 @@ package main import ( - "code.google.com/p/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/operation" "encoding/json" "fmt" "os" diff --git a/go/weed/version.go b/go/weed/version.go index a7c24ea3b..92eb4548d 100644 --- a/go/weed/version.go +++ b/go/weed/version.go @@ -1,7 +1,7 @@ package main import ( - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/util" "fmt" "runtime" ) diff --git a/go/weed/volume.go b/go/weed/volume.go index 7a57730cd..ba6df265b 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -1,9 +1,9 @@ package main import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/util" - "code.google.com/p/weed-fs/go/weed/weed_server" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/weed/weed_server" "net/http" "os" "runtime" diff --git a/go/weed/volume_test.go b/go/weed/volume_test.go index 2499d8543..c5231cf5f 100644 --- a/go/weed/volume_test.go +++ b/go/weed/volume_test.go @@ -1,7 +1,7 @@ package main import ( - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "net/http" "testing" "time" diff --git a/go/weed/weed.go b/go/weed/weed.go index 03cf29179..642de6b4a 100644 --- a/go/weed/weed.go +++ b/go/weed/weed.go @@ -1,7 +1,7 @@ package main import ( - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "flag" "fmt" "io" diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 49e84378c..4af150225 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -2,11 +2,11 @@ package weed_server import ( "bytes" - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/operation" - "code.google.com/p/weed-fs/go/stats" - "code.google.com/p/weed-fs/go/storage" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/stats" + "github.com/aszxqw/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/util" "encoding/json" "fmt" "net" diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index e56cb5964..0e889fe9a 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -1,8 +1,8 @@ package weed_server import ( - "code.google.com/p/weed-fs/go/filer" - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/filer" + "github.com/aszxqw/weed-fs/go/glog" "net/http" "strconv" ) diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index f760030f3..af667666e 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -1,9 +1,9 @@ package weed_server import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/operation" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/util" "encoding/json" "errors" "github.com/syndtr/goleveldb/leveldb" diff --git a/go/weed/weed_server/filer_server_handlers_admin.go b/go/weed/weed_server/filer_server_handlers_admin.go index b4b8b7221..d693769f2 100644 --- a/go/weed/weed_server/filer_server_handlers_admin.go +++ b/go/weed/weed_server/filer_server_handlers_admin.go @@ -1,7 +1,7 @@ package weed_server import ( - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "net/http" ) diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go index a89959fa4..b3dbe2045 100644 --- a/go/weed/weed_server/master_server.go +++ b/go/weed/weed_server/master_server.go @@ -1,10 +1,10 @@ package weed_server import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/sequence" - "code.google.com/p/weed-fs/go/topology" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/sequence" + "github.com/aszxqw/weed-fs/go/topology" + "github.com/aszxqw/weed-fs/go/util" "github.com/goraft/raft" "github.com/gorilla/mux" "net/http" diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 1b58e6e73..5a5ebc225 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -1,9 +1,9 @@ package weed_server import ( - "code.google.com/p/weed-fs/go/operation" - "code.google.com/p/weed-fs/go/stats" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/stats" + "github.com/aszxqw/weed-fs/go/storage" "net/http" "strconv" "strings" diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 1458bf3e6..80cabf093 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -2,11 +2,11 @@ package weed_server import ( proto "code.google.com/p/goprotobuf/proto" - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/operation" - "code.google.com/p/weed-fs/go/storage" - "code.google.com/p/weed-fs/go/topology" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/topology" + "github.com/aszxqw/weed-fs/go/util" "encoding/json" "errors" "io/ioutil" diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index f67caaebd..83c55c333 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -2,8 +2,8 @@ package weed_server import ( "bytes" - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/topology" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/topology" "encoding/json" "errors" "fmt" diff --git a/go/weed/weed_server/raft_server_handlers.go b/go/weed/weed_server/raft_server_handlers.go index 1ce24a963..6eba0cb62 100644 --- a/go/weed/weed_server/raft_server_handlers.go +++ b/go/weed/weed_server/raft_server_handlers.go @@ -1,8 +1,8 @@ package weed_server import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/operation" "encoding/json" "github.com/goraft/raft" "io/ioutil" diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index 9cada91ac..b480e0742 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -1,8 +1,8 @@ package weed_server import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/storage" "math/rand" "net/http" "strconv" diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index eed198e4a..0f0e8012d 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -1,12 +1,12 @@ package weed_server import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/images" - "code.google.com/p/weed-fs/go/operation" - "code.google.com/p/weed-fs/go/stats" - "code.google.com/p/weed-fs/go/storage" - "code.google.com/p/weed-fs/go/topology" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/images" + "github.com/aszxqw/weed-fs/go/operation" + "github.com/aszxqw/weed-fs/go/stats" + "github.com/aszxqw/weed-fs/go/storage" + "github.com/aszxqw/weed-fs/go/topology" "io" "mime" "mime/multipart" diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go index 1c01fdfd0..181de6017 100644 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ b/go/weed/weed_server/volume_server_handlers_admin.go @@ -1,9 +1,9 @@ package weed_server import ( - "code.google.com/p/weed-fs/go/glog" - "code.google.com/p/weed-fs/go/stats" - "code.google.com/p/weed-fs/go/util" + "github.com/aszxqw/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/stats" + "github.com/aszxqw/weed-fs/go/util" "net/http" "path/filepath" ) diff --git a/go/weed/weed_server/volume_server_handlers_vacuum.go b/go/weed/weed_server/volume_server_handlers_vacuum.go index 60a5e9742..4cbb1ba11 100644 --- a/go/weed/weed_server/volume_server_handlers_vacuum.go +++ b/go/weed/weed_server/volume_server_handlers_vacuum.go @@ -1,7 +1,7 @@ package weed_server import ( - "code.google.com/p/weed-fs/go/glog" + "github.com/aszxqw/weed-fs/go/glog" "net/http" ) From 2f1cf164b92f29db4dd0452572a948ae37b28add Mon Sep 17 00:00:00 2001 From: wyy Date: Thu, 25 Sep 2014 16:28:32 +0800 Subject: [PATCH 17/58] update readme.md --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 1808a0277..75fb07b36 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,19 @@ weed-fs [![GoDoc](https://godoc.org/github.com/chrislusf/weed-fs/go?status.svg)](https://godoc.org/github.com/chrislusf/weed-fs/go) [![RTD](https://readthedocs.org/projects/weed-fs/badge/?version=latest)](http://weed-fs.readthedocs.org/en/latest/) +## Introduction + An official mirror of code.google.com/p/weed-fs. Moving to github.com to make cooperations easier. This repo and the google code repo will be kept synchronized. +## Usage + +``` +go get github.com/aszxqw/weed-fs/go/weed +``` + +## Reference For documents and bug reporting, Please visit http://weed-fs.googlecode.com From d39c62bbed9b56164ab92101cf7ec46d4a1bfeee Mon Sep 17 00:00:00 2001 From: wyy Date: Thu, 25 Sep 2014 16:34:26 +0800 Subject: [PATCH 18/58] add .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..46f3c0430 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +weed From 4126280d5559a364c6e0f31ab7a57bc3b1d15d56 Mon Sep 17 00:00:00 2001 From: wyy Date: Thu, 25 Sep 2014 16:57:22 +0800 Subject: [PATCH 19/58] use github.com/chrislusf instead of github.com/aszxqw --- go/filer/client_operations.go | 2 +- go/filer/directory_in_map.go | 2 +- go/filer/filer_embedded.go | 2 +- go/filer/files_in_leveldb.go | 2 +- go/operation/assign_file_id.go | 4 ++-- go/operation/delete_content.go | 2 +- go/operation/list_masters.go | 4 ++-- go/operation/lookup.go | 2 +- go/operation/submit.go | 2 +- go/operation/upload_content.go | 2 +- go/storage/cdb_map.go | 2 +- go/storage/cdb_map_test.go | 2 +- go/storage/compact_map_perf_test.go | 4 ++-- go/storage/compress.go | 2 +- go/storage/crc.go | 2 +- go/storage/file_id.go | 4 ++-- go/storage/needle.go | 6 +++--- go/storage/needle_map.go | 4 ++-- go/storage/needle_read_write.go | 4 ++-- go/storage/store.go | 6 +++--- go/storage/store_vacuum.go | 2 +- go/storage/volume.go | 2 +- go/storage/volume_info.go | 2 +- go/storage/volume_super_block.go | 2 +- go/storage/volume_vacuum.go | 2 +- go/tools/read_index.go | 2 +- go/topology/allocate_volume.go | 4 ++-- go/topology/cluster_commands.go | 4 ++-- go/topology/collection.go | 2 +- go/topology/data_node.go | 4 ++-- go/topology/node.go | 4 ++-- go/topology/store_replicate.go | 8 ++++---- go/topology/topology.go | 8 ++++---- go/topology/topology_event_handling.go | 4 ++-- go/topology/topology_vacuum.go | 6 +++--- go/topology/volume_growth.go | 4 ++-- go/topology/volume_growth_test.go | 4 ++-- go/topology/volume_layout.go | 4 ++-- go/util/config.go | 2 +- go/util/file_util.go | 2 +- go/util/net_timeout.go | 2 +- go/weed/benchmark.go | 6 +++--- go/weed/compact.go | 4 ++-- go/weed/download.go | 4 ++-- go/weed/export.go | 4 ++-- go/weed/filer.go | 6 +++--- go/weed/fix.go | 4 ++-- go/weed/master.go | 6 +++--- go/weed/mount_std.go | 8 ++++---- go/weed/server.go | 6 +++--- go/weed/shell.go | 2 +- go/weed/upload.go | 2 +- go/weed/version.go | 2 +- go/weed/volume.go | 6 +++--- go/weed/volume_test.go | 2 +- go/weed/weed.go | 2 +- go/weed/weed_server/common.go | 10 +++++----- go/weed/weed_server/filer_server.go | 4 ++-- go/weed/weed_server/filer_server_handlers.go | 6 +++--- go/weed/weed_server/filer_server_handlers_admin.go | 2 +- go/weed/weed_server/master_server.go | 8 ++++---- go/weed/weed_server/master_server_handlers.go | 6 +++--- go/weed/weed_server/master_server_handlers_admin.go | 10 +++++----- go/weed/weed_server/raft_server.go | 4 ++-- go/weed/weed_server/raft_server_handlers.go | 4 ++-- go/weed/weed_server/volume_server.go | 4 ++-- go/weed/weed_server/volume_server_handlers.go | 12 ++++++------ go/weed/weed_server/volume_server_handlers_admin.go | 6 +++--- go/weed/weed_server/volume_server_handlers_vacuum.go | 2 +- 69 files changed, 138 insertions(+), 138 deletions(-) diff --git a/go/filer/client_operations.go b/go/filer/client_operations.go index d9a9501a5..0b006289f 100644 --- a/go/filer/client_operations.go +++ b/go/filer/client_operations.go @@ -3,7 +3,7 @@ package filer import () import ( - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "fmt" diff --git a/go/filer/directory_in_map.go b/go/filer/directory_in_map.go index 85cde765d..1d88a78be 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/directory_in_map.go @@ -2,7 +2,7 @@ package filer import ( "bufio" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "os" diff --git a/go/filer/filer_embedded.go b/go/filer/filer_embedded.go index d716802ac..3d3dac941 100644 --- a/go/filer/filer_embedded.go +++ b/go/filer/filer_embedded.go @@ -1,7 +1,7 @@ package filer import ( - "github.com/aszxqw/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/operation" "errors" "fmt" "path/filepath" diff --git a/go/filer/files_in_leveldb.go b/go/filer/files_in_leveldb.go index 17d726dd7..41fbc74bd 100644 --- a/go/filer/files_in_leveldb.go +++ b/go/filer/files_in_leveldb.go @@ -2,7 +2,7 @@ package filer import ( "bytes" - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" ) diff --git a/go/operation/assign_file_id.go b/go/operation/assign_file_id.go index 891653278..4e72ad939 100644 --- a/go/operation/assign_file_id.go +++ b/go/operation/assign_file_id.go @@ -1,8 +1,8 @@ package operation import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" diff --git a/go/operation/delete_content.go b/go/operation/delete_content.go index a1485a2db..84391b634 100644 --- a/go/operation/delete_content.go +++ b/go/operation/delete_content.go @@ -1,7 +1,7 @@ package operation import ( - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" diff --git a/go/operation/list_masters.go b/go/operation/list_masters.go index 120ba58a1..7d46a9ebc 100644 --- a/go/operation/list_masters.go +++ b/go/operation/list_masters.go @@ -1,8 +1,8 @@ package operation import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" "encoding/json" ) diff --git a/go/operation/lookup.go b/go/operation/lookup.go index 1b5d57e2f..ebf153d27 100644 --- a/go/operation/lookup.go +++ b/go/operation/lookup.go @@ -1,7 +1,7 @@ package operation import ( - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" _ "fmt" diff --git a/go/operation/submit.go b/go/operation/submit.go index 18bc593a3..3e09c2edf 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -2,7 +2,7 @@ package operation import ( "bytes" - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "io" "mime" "os" diff --git a/go/operation/upload_content.go b/go/operation/upload_content.go index 3812a7d86..38737702d 100644 --- a/go/operation/upload_content.go +++ b/go/operation/upload_content.go @@ -2,7 +2,7 @@ package operation import ( "bytes" - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "encoding/json" "errors" "fmt" diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go index 07cbca1a9..1869a563e 100644 --- a/go/storage/cdb_map.go +++ b/go/storage/cdb_map.go @@ -1,7 +1,7 @@ package storage import ( - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "fmt" diff --git a/go/storage/cdb_map_test.go b/go/storage/cdb_map_test.go index 25f6067cb..cff7dfa61 100644 --- a/go/storage/cdb_map_test.go +++ b/go/storage/cdb_map_test.go @@ -1,7 +1,7 @@ package storage import ( - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "math/rand" "os" "runtime" diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index 4795ead5a..ef43de25b 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -1,8 +1,8 @@ package storage import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" "log" "os" "testing" diff --git a/go/storage/compress.go b/go/storage/compress.go index 7390b85dd..a353c9d3a 100644 --- a/go/storage/compress.go +++ b/go/storage/compress.go @@ -2,7 +2,7 @@ package storage import ( "bytes" - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "compress/flate" "compress/gzip" "io/ioutil" diff --git a/go/storage/crc.go b/go/storage/crc.go index 9b77c30c6..7aa400959 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -1,7 +1,7 @@ package storage import ( - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" "fmt" "hash/crc32" ) diff --git a/go/storage/file_id.go b/go/storage/file_id.go index 435a54202..ec566826c 100644 --- a/go/storage/file_id.go +++ b/go/storage/file_id.go @@ -1,8 +1,8 @@ package storage import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" "encoding/hex" "errors" "strings" diff --git a/go/storage/needle.go b/go/storage/needle.go index 803bc624e..daede321b 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -1,9 +1,9 @@ package storage import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/images" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/images" + "github.com/chrislusf/weed-fs/go/util" "encoding/hex" "errors" "io/ioutil" diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index 90f3eff7d..dca2e6c5d 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -1,8 +1,8 @@ package storage import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "os" diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index d298c806b..bf452ba37 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -1,8 +1,8 @@ package storage import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" "errors" "fmt" "io" diff --git a/go/storage/store.go b/go/storage/store.go index f5b252ad3..e7a9dac94 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -2,9 +2,9 @@ package storage import ( proto "code.google.com/p/goprotobuf/proto" - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/operation" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "fmt" diff --git a/go/storage/store_vacuum.go b/go/storage/store_vacuum.go index b3f13fd2d..3527e4f59 100644 --- a/go/storage/store_vacuum.go +++ b/go/storage/store_vacuum.go @@ -1,7 +1,7 @@ package storage import ( - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "fmt" "strconv" ) diff --git a/go/storage/volume.go b/go/storage/volume.go index 10fd145e9..de79e9107 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -2,7 +2,7 @@ package storage import ( "bytes" - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "errors" "fmt" "io" diff --git a/go/storage/volume_info.go b/go/storage/volume_info.go index 81dff4a11..6410c1784 100644 --- a/go/storage/volume_info.go +++ b/go/storage/volume_info.go @@ -1,7 +1,7 @@ package storage import ( - "github.com/aszxqw/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/operation" ) type VolumeInfo struct { diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index da510ee30..a7e86b1c3 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -1,7 +1,7 @@ package storage import ( - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "fmt" "os" ) diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go index 7feb1c256..b348434d2 100644 --- a/go/storage/volume_vacuum.go +++ b/go/storage/volume_vacuum.go @@ -1,7 +1,7 @@ package storage import ( - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "fmt" "os" "time" diff --git a/go/tools/read_index.go b/go/tools/read_index.go index 36d39a381..b99c5b6b8 100644 --- a/go/tools/read_index.go +++ b/go/tools/read_index.go @@ -1,7 +1,7 @@ package main import ( - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/storage" "flag" "fmt" "log" diff --git a/go/topology/allocate_volume.go b/go/topology/allocate_volume.go index eaeae4427..6562e9ac5 100644 --- a/go/topology/allocate_volume.go +++ b/go/topology/allocate_volume.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/aszxqw/weed-fs/go/storage" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" diff --git a/go/topology/cluster_commands.go b/go/topology/cluster_commands.go index 369c0b9b9..cafc52c76 100644 --- a/go/topology/cluster_commands.go +++ b/go/topology/cluster_commands.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" "github.com/goraft/raft" ) diff --git a/go/topology/collection.go b/go/topology/collection.go index 72ac86522..506f43fbf 100644 --- a/go/topology/collection.go +++ b/go/topology/collection.go @@ -1,7 +1,7 @@ package topology import ( - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/storage" ) type Collection struct { diff --git a/go/topology/data_node.go b/go/topology/data_node.go index 5c16cf85d..c3b90470f 100644 --- a/go/topology/data_node.go +++ b/go/topology/data_node.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" "strconv" ) diff --git a/go/topology/node.go b/go/topology/node.go index 12e9a709a..54118802e 100644 --- a/go/topology/node.go +++ b/go/topology/node.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" "errors" "math/rand" "strings" diff --git a/go/topology/store_replicate.go b/go/topology/store_replicate.go index fefaea8a6..6ea019bd8 100644 --- a/go/topology/store_replicate.go +++ b/go/topology/store_replicate.go @@ -2,10 +2,10 @@ package topology import ( "bytes" - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/operation" - "github.com/aszxqw/weed-fs/go/storage" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" "net/http" "strconv" ) diff --git a/go/topology/topology.go b/go/topology/topology.go index 77c9558bb..c90e8de0b 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -1,10 +1,10 @@ package topology import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/operation" - "github.com/aszxqw/weed-fs/go/sequence" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/sequence" + "github.com/chrislusf/weed-fs/go/storage" "errors" "github.com/goraft/raft" "io/ioutil" diff --git a/go/topology/topology_event_handling.go b/go/topology/topology_event_handling.go index 61e6f5ffa..eb4491484 100644 --- a/go/topology/topology_event_handling.go +++ b/go/topology/topology_event_handling.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" "math/rand" "time" ) diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index bdc0c38f9..72846f20b 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -1,9 +1,9 @@ package topology import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/storage" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" diff --git a/go/topology/volume_growth.go b/go/topology/volume_growth.go index e083a8531..2859d3992 100644 --- a/go/topology/volume_growth.go +++ b/go/topology/volume_growth.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" "fmt" "math/rand" "sync" diff --git a/go/topology/volume_growth_test.go b/go/topology/volume_growth_test.go index ba1e43ff2..5581c87ce 100644 --- a/go/topology/volume_growth_test.go +++ b/go/topology/volume_growth_test.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/aszxqw/weed-fs/go/sequence" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/sequence" + "github.com/chrislusf/weed-fs/go/storage" "encoding/json" "fmt" "testing" diff --git a/go/topology/volume_layout.go b/go/topology/volume_layout.go index 35f45c08d..7bb0cf7e3 100644 --- a/go/topology/volume_layout.go +++ b/go/topology/volume_layout.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" "errors" "math/rand" "sync" diff --git a/go/util/config.go b/go/util/config.go index 1287ed881..050fd0e64 100644 --- a/go/util/config.go +++ b/go/util/config.go @@ -10,7 +10,7 @@ package util import ( "bytes" - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "encoding/json" "os" ) diff --git a/go/util/file_util.go b/go/util/file_util.go index 82f6bffe3..412d98458 100644 --- a/go/util/file_util.go +++ b/go/util/file_util.go @@ -2,7 +2,7 @@ package util import ( "bufio" - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "errors" "os" ) diff --git a/go/util/net_timeout.go b/go/util/net_timeout.go index ec9796e63..eb80822b5 100644 --- a/go/util/net_timeout.go +++ b/go/util/net_timeout.go @@ -1,7 +1,7 @@ package util import ( - "github.com/aszxqw/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/stats" "net" "time" ) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index e33d37816..fec8472e5 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -2,9 +2,9 @@ package main import ( "bufio" - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/operation" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "math" diff --git a/go/weed/compact.go b/go/weed/compact.go index 1006701aa..a99e6c93e 100644 --- a/go/weed/compact.go +++ b/go/weed/compact.go @@ -1,8 +1,8 @@ package main import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/download.go b/go/weed/download.go index 3b3df6d61..c30d17915 100644 --- a/go/weed/download.go +++ b/go/weed/download.go @@ -1,8 +1,8 @@ package main import ( - "github.com/aszxqw/weed-fs/go/operation" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "io/ioutil" diff --git a/go/weed/export.go b/go/weed/export.go index e24108220..81bc21f6e 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -3,8 +3,8 @@ package main import ( "archive/tar" "bytes" - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" "fmt" "os" "path" diff --git a/go/weed/filer.go b/go/weed/filer.go index 0301d1e5c..7dbecb4d0 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -1,9 +1,9 @@ package main import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/util" - "github.com/aszxqw/weed-fs/go/weed/weed_server" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" "net/http" "os" "strconv" diff --git a/go/weed/fix.go b/go/weed/fix.go index f07d9269f..ad573875a 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -1,8 +1,8 @@ package main import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" "os" "path" "strconv" diff --git a/go/weed/master.go b/go/weed/master.go index 046cbff74..6617c8ca6 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -1,9 +1,9 @@ package main import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/util" - "github.com/aszxqw/weed-fs/go/weed/weed_server" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" "github.com/gorilla/mux" "net/http" "os" diff --git a/go/weed/mount_std.go b/go/weed/mount_std.go index 2df47ad62..e5fc0986c 100644 --- a/go/weed/mount_std.go +++ b/go/weed/mount_std.go @@ -5,10 +5,10 @@ package main import ( "bazil.org/fuse" "bazil.org/fuse/fs" - "github.com/aszxqw/weed-fs/go/filer" - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/storage" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/filer" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" "fmt" "os" "runtime" diff --git a/go/weed/server.go b/go/weed/server.go index 624c1ca84..1d854d641 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -1,9 +1,9 @@ package main import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/util" - "github.com/aszxqw/weed-fs/go/weed/weed_server" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" "github.com/gorilla/mux" "net/http" "os" diff --git a/go/weed/shell.go b/go/weed/shell.go index 16a414deb..c8043e0dd 100644 --- a/go/weed/shell.go +++ b/go/weed/shell.go @@ -2,7 +2,7 @@ package main import ( "bufio" - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "fmt" "os" ) diff --git a/go/weed/upload.go b/go/weed/upload.go index dee013645..4eae4d274 100644 --- a/go/weed/upload.go +++ b/go/weed/upload.go @@ -1,7 +1,7 @@ package main import ( - "github.com/aszxqw/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/operation" "encoding/json" "fmt" "os" diff --git a/go/weed/version.go b/go/weed/version.go index 92eb4548d..63441509e 100644 --- a/go/weed/version.go +++ b/go/weed/version.go @@ -1,7 +1,7 @@ package main import ( - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" "fmt" "runtime" ) diff --git a/go/weed/volume.go b/go/weed/volume.go index ba6df265b..17d03f0c5 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -1,9 +1,9 @@ package main import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/util" - "github.com/aszxqw/weed-fs/go/weed/weed_server" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" "net/http" "os" "runtime" diff --git a/go/weed/volume_test.go b/go/weed/volume_test.go index c5231cf5f..764362a2b 100644 --- a/go/weed/volume_test.go +++ b/go/weed/volume_test.go @@ -1,7 +1,7 @@ package main import ( - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "net/http" "testing" "time" diff --git a/go/weed/weed.go b/go/weed/weed.go index 642de6b4a..c1f5a72de 100644 --- a/go/weed/weed.go +++ b/go/weed/weed.go @@ -1,7 +1,7 @@ package main import ( - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "flag" "fmt" "io" diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 4af150225..816107dc5 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -2,11 +2,11 @@ package weed_server import ( "bytes" - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/operation" - "github.com/aszxqw/weed-fs/go/stats" - "github.com/aszxqw/weed-fs/go/storage" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" "encoding/json" "fmt" "net" diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 0e889fe9a..5ff0ed986 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -1,8 +1,8 @@ package weed_server import ( - "github.com/aszxqw/weed-fs/go/filer" - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/filer" + "github.com/chrislusf/weed-fs/go/glog" "net/http" "strconv" ) diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index af667666e..e36e7c310 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -1,9 +1,9 @@ package weed_server import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/operation" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "github.com/syndtr/goleveldb/leveldb" diff --git a/go/weed/weed_server/filer_server_handlers_admin.go b/go/weed/weed_server/filer_server_handlers_admin.go index d693769f2..ff52dff24 100644 --- a/go/weed/weed_server/filer_server_handlers_admin.go +++ b/go/weed/weed_server/filer_server_handlers_admin.go @@ -1,7 +1,7 @@ package weed_server import ( - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "net/http" ) diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go index b3dbe2045..401f6cfdb 100644 --- a/go/weed/weed_server/master_server.go +++ b/go/weed/weed_server/master_server.go @@ -1,10 +1,10 @@ package weed_server import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/sequence" - "github.com/aszxqw/weed-fs/go/topology" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/sequence" + "github.com/chrislusf/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/util" "github.com/goraft/raft" "github.com/gorilla/mux" "net/http" diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 5a5ebc225..93e9e7d9a 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -1,9 +1,9 @@ package weed_server import ( - "github.com/aszxqw/weed-fs/go/operation" - "github.com/aszxqw/weed-fs/go/stats" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" "net/http" "strconv" "strings" diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 80cabf093..c9a8020c2 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -2,11 +2,11 @@ package weed_server import ( proto "code.google.com/p/goprotobuf/proto" - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/operation" - "github.com/aszxqw/weed-fs/go/storage" - "github.com/aszxqw/weed-fs/go/topology" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "io/ioutil" diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index 83c55c333..e41867076 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -2,8 +2,8 @@ package weed_server import ( "bytes" - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/topology" "encoding/json" "errors" "fmt" diff --git a/go/weed/weed_server/raft_server_handlers.go b/go/weed/weed_server/raft_server_handlers.go index 6eba0cb62..4d51c0767 100644 --- a/go/weed/weed_server/raft_server_handlers.go +++ b/go/weed/weed_server/raft_server_handlers.go @@ -1,8 +1,8 @@ package weed_server import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" "encoding/json" "github.com/goraft/raft" "io/ioutil" diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index b480e0742..2a9085f3b 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -1,8 +1,8 @@ package weed_server import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" "math/rand" "net/http" "strconv" diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index 0f0e8012d..ce14f6a87 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -1,12 +1,12 @@ package weed_server import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/images" - "github.com/aszxqw/weed-fs/go/operation" - "github.com/aszxqw/weed-fs/go/stats" - "github.com/aszxqw/weed-fs/go/storage" - "github.com/aszxqw/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/images" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/topology" "io" "mime" "mime/multipart" diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go index 181de6017..1118c8017 100644 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ b/go/weed/weed_server/volume_server_handlers_admin.go @@ -1,9 +1,9 @@ package weed_server import ( - "github.com/aszxqw/weed-fs/go/glog" - "github.com/aszxqw/weed-fs/go/stats" - "github.com/aszxqw/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/util" "net/http" "path/filepath" ) diff --git a/go/weed/weed_server/volume_server_handlers_vacuum.go b/go/weed/weed_server/volume_server_handlers_vacuum.go index 4cbb1ba11..b0600d799 100644 --- a/go/weed/weed_server/volume_server_handlers_vacuum.go +++ b/go/weed/weed_server/volume_server_handlers_vacuum.go @@ -1,7 +1,7 @@ package weed_server import ( - "github.com/aszxqw/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "net/http" ) From 620b1244b44aa0f650a7ff5dac53d4c646efa22d Mon Sep 17 00:00:00 2001 From: wyy Date: Thu, 25 Sep 2014 17:07:24 +0800 Subject: [PATCH 20/58] update readme.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 75fb07b36..d58e3792a 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ This repo and the google code repo will be kept synchronized. ## Usage ``` -go get github.com/aszxqw/weed-fs/go/weed +go get github.com/chrislusf/weed-fs/go/weed ``` ## Reference From d0229d99ed3b1b430f6de246f7a7b5f9d49d0212 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 25 Sep 2014 21:53:32 -0700 Subject: [PATCH 21/58] name change on documents --- docs/api.rst | 2 +- docs/clients.rst | 20 ++++++++++---------- docs/directories.rst | 6 +++--- docs/gettingstarted.rst | 2 +- docs/optimization.rst | 14 +++++++------- docs/replication.rst | 4 ++-- docs/usecases.rst | 2 +- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index cb66d5627..f26408f5a 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -180,7 +180,7 @@ Upload File curl -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6 {"size": 43234} -The size returned is the size stored on WeedFS, sometimes the file is automatically gzipped based on the mime type. +The size returned is the size stored on Seaweed-FS, sometimes the file is automatically gzipped based on the mime type. Upload File Directly *********************************** diff --git a/docs/clients.rst b/docs/clients.rst index 66f7bca5f..6c6f09729 100644 --- a/docs/clients.rst +++ b/docs/clients.rst @@ -8,17 +8,17 @@ Clients +---------------------------------------------------------------------------------+--------------+-----------+ | `WeedPHP `_ | Mic Johnson | PHP | +---------------------------------------------------------------------------------+--------------+-----------+ -| `Weed-FS Symfony bundle `_ | Mic Johnson | PHP | +| `Seaweed-FS Symfony bundle `_ | Mic Johnson | PHP | +---------------------------------------------------------------------------------+--------------+-----------+ -| `Weed-FS Node.js client `_ | Aaron Blakely| Javascript| +| `Seaweed-FS Node.js client `_ | Aaron Blakely| Javascript| +---------------------------------------------------------------------------------+--------------+-----------+ -| `Amazon S3 API for weed-fs `_ | Tamás Gulácsi| Go | +| `Amazon S3 API for Seaweed-FS `_ | Tamás Gulácsi| Go | +---------------------------------------------------------------------------------+--------------+-----------+ | `File store upload test `_ | Tamás Gulácsi| Go | +---------------------------------------------------------------------------------+--------------+-----------+ -| `Java Weed-FS client `_ | Xu Zhang | Java | +| `Java Seaweed-FS client `_ | Xu Zhang | Java | +---------------------------------------------------------------------------------+--------------+-----------+ -| `Java Weed-FS client 2 `_ | Zenria | Java | +| `Java Seaweed-FS client 2 `_ | Zenria | Java | +---------------------------------------------------------------------------------+--------------+-----------+ | `Python-weed `_ | Darkdarkfruit| Python | +---------------------------------------------------------------------------------+--------------+-----------+ @@ -26,7 +26,7 @@ Clients +---------------------------------------------------------------------------------+--------------+-----------+ | `Camlistore blobserver Storage `_ | Tamás Gulácsi| Go | +---------------------------------------------------------------------------------+--------------+-----------+ -| `Scala Weed-FS client `_ | Chiradip | Scala | +| `Scala Seaweed-FS client `_ | Chiradip | Scala | +---------------------------------------------------------------------------------+--------------+-----------+ | `Module for kohana `_ | Bububa | PHP | +---------------------------------------------------------------------------------+--------------+-----------+ @@ -35,10 +35,10 @@ Clients | `Django-weed `_ | ProstoKSI | Python | +---------------------------------------------------------------------------------+--------------+-----------+ -Projects using Weed-Fs +Projects using Seaweed-FS ################################### -* An `email River Plugin `_ for Elasticsearch uses weed-fs server to save attachments +* An `email River Plugin `_ for Elasticsearch uses Seaweed-FS server to save attachments -Websites using Weed-Fs +Websites using Seaweed-FS ################################### -* `Email to create Web Pages `_ uses weed-fs to save email attachments. \ No newline at end of file +* `Email to create Web Pages `_ uses Seaweed-FS to save email attachments. \ No newline at end of file diff --git a/docs/directories.rst b/docs/directories.rst index 57d5d246d..787c7a2ab 100644 --- a/docs/directories.rst +++ b/docs/directories.rst @@ -37,7 +37,7 @@ Design A common file system would use inode to store meta data for each folder and file. The folder tree structure are usually linked. And sub folders and files are usually organized as an on-disk b+tree or similar variations. This scales well in terms of storage, but not well for fast file retrieval due to multiple disk access just for the file meta data, before even trying to get the file content. -WeedFS wants to make as small number of disk access as possible, yet still be able to store a lot of file metadata. So we need to think very differently. +Seaweed-FS wants to make as small number of disk access as possible, yet still be able to store a lot of file metadata. So we need to think very differently. From a full file path to get to the file content, there are several steps: @@ -48,7 +48,7 @@ From a full file path to get to the file content, there are several steps: file_id => data_block -Because default WeedFS only provides file_id=>data_block mapping, the first 2 steps need to be implemented. +Because default Seaweed-FS only provides file_id=>data_block mapping, the first 2 steps need to be implemented. There are several data features I noticed: @@ -122,7 +122,7 @@ The LevelDB implementation may be switched underneath to external data storage, Also, a HA feature will be added, so that multiple "weed filer" instance can share the same set of view of files. -Later, FUSE or HCFS plugins will be created, to really integrate WeedFS to existing systems. +Later, FUSE or HCFS plugins will be created, to really integrate Seaweed-FS to existing systems. Helps Wanted ######################## diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index 0b598a2bf..19eb130a7 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -3,7 +3,7 @@ Getting started Installing Weed-Fs ################################### -Download a proper version from `WeedFS download page `_. +Download a proper version from `Seaweed-FS download page `_. Decompress the downloaded file. You will only find one executable file, either "weed" on most systems or "weed.exe" on windows. diff --git a/docs/optimization.rst b/docs/optimization.rst index 49f0656c8..9d96de060 100644 --- a/docs/optimization.rst +++ b/docs/optimization.rst @@ -1,14 +1,14 @@ Optimization ============== -Here are the strategies or best ways to optimize WeedFS. +Here are the strategies or best ways to optimize Seaweed-FS. Increase concurrent writes ################################ -By default, WeedFS grows the volumes automatically. For example, for no-replication volumes, there will be concurrently 7 writable volumes allocated. +By default, Seaweed-FS grows the volumes automatically. For example, for no-replication volumes, there will be concurrently 7 writable volumes allocated. -If you want to distribute writes to more volumes, you can do so by instructing WeedFS master via this URL. +If you want to distribute writes to more volumes, you can do so by instructing Seaweed-FS master via this URL. .. code-block:: bash @@ -31,14 +31,14 @@ More hard drives will give you better write/read throughput. Gzip content ################################ -WeedFS determines the file can be gzipped based on the file name extension. So if you submit a textual file, it's better to use an common file name extension, like ".txt", ".html", ".js", ".css", etc. If the name is unknown, like ".go", WeedFS will not gzip the content, but just save the content as is. +Seaweed-FS determines the file can be gzipped based on the file name extension. So if you submit a textual file, it's better to use an common file name extension, like ".txt", ".html", ".js", ".css", etc. If the name is unknown, like ".go", Seaweed-FS will not gzip the content, but just save the content as is. -You can also manually gzip content before submission. If you do so, make sure the submitted file has file name with ends with ".gz". For example, "my.css" can be gzipped to "my.css.gz" and sent to WeedFS. When retrieving the content, if the http client supports "gzip" encoding, the gzipped content would be sent back. Otherwise, the unzipped content would be sent back. +You can also manually gzip content before submission. If you do so, make sure the submitted file has file name with ends with ".gz". For example, "my.css" can be gzipped to "my.css.gz" and sent to Seaweed-FS. When retrieving the content, if the http client supports "gzip" encoding, the gzipped content would be sent back. Otherwise, the unzipped content would be sent back. Memory consumption ################################# -For volume servers, the memory consumption is tightly related to the number of files. For example, one 32G volume can easily have 1.5 million files if each file is only 20KB. To store the 1.5 million entries of meta data in memory, currently WeedFS consumes 36MB memory, about 24bytes per entry in memory. So if you allocate 64 volumes(2TB), you would need 2~3GB memory. However, if the average file size is larger, say 200KB, only 200~300MB memory is needed. +For volume servers, the memory consumption is tightly related to the number of files. For example, one 32G volume can easily have 1.5 million files if each file is only 20KB. To store the 1.5 million entries of meta data in memory, currently Seaweed-FS consumes 36MB memory, about 24bytes per entry in memory. So if you allocate 64 volumes(2TB), you would need 2~3GB memory. However, if the average file size is larger, say 200KB, only 200~300MB memory is needed. Theoretically the memory consumption can go even lower by compacting since the file ids are mostly monotonically increasing. I did not invest time on that yet since the memory consumption, 24bytes/entry(including uncompressed 8bytes file id, 4 bytes file size, plus additional map data structure cost) is already pretty low. But I welcome any one to compact these data in memory even more efficiently. @@ -106,7 +106,7 @@ In case you need to delete them later, you can go to the volume servers and dele Logging ############################## -When going to production, you will want to collect the logs. WeedFS uses glog. Here are some examples: +When going to production, you will want to collect the logs. Seaweed-FS uses glog. Here are some examples: .. code-block:: bash diff --git a/docs/replication.rst b/docs/replication.rst index 136f41b97..54e4b2503 100644 --- a/docs/replication.rst +++ b/docs/replication.rst @@ -1,6 +1,6 @@ Replication =================================== -Weed-FS can support replication. The replication is implemented not on file level, but on volume level. +Seaweed-FS can support replication. The replication is implemented not on file level, but on volume level. How to use ################################### @@ -58,7 +58,7 @@ Each replication type will physically create x+y+z+1 copies of volume data files Example topology configuration ################################### -The WeedFS master server tries to read the default topology configuration file are read from /etc/weedfs/weedfs.conf, if it exists. The topology setting to configure data center and racks file format is as this. +The Seaweed-FS master server tries to read the default topology configuration file are read from /etc/weedfs/weedfs.conf, if it exists. The topology setting to configure data center and racks file format is as this. .. code-block:: xml diff --git a/docs/usecases.rst b/docs/usecases.rst index c3cb8e3af..8a9a6d44a 100644 --- a/docs/usecases.rst +++ b/docs/usecases.rst @@ -41,7 +41,7 @@ The wrong way to send it: curl -H "Content-Type:image/png" -F file=@myImage.png http://127.0.0.1:8080/5,2730a7f18b44 -Securing WeedFS +Securing Seaweed-FS ############################# The simple way is to front all master and volume servers with firewall. From b22579310397e5e14678fddb458903643604a277 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 25 Sep 2014 22:25:51 -0700 Subject: [PATCH 22/58] Move introduction page to github. --- README.md | 259 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 249 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d58e3792a..fc89c05e5 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,10 @@ -weed-fs +Seaweed File System ======= [![Build Status](https://travis-ci.org/chrislusf/weed-fs.svg?branch=master)](https://travis-ci.org/chrislusf/weed-fs) [![GoDoc](https://godoc.org/github.com/chrislusf/weed-fs/go?status.svg)](https://godoc.org/github.com/chrislusf/weed-fs/go) [![RTD](https://readthedocs.org/projects/weed-fs/badge/?version=latest)](http://weed-fs.readthedocs.org/en/latest/) -## Introduction - -An official mirror of code.google.com/p/weed-fs. -Moving to github.com to make cooperations easier. -This repo and the google code repo will be kept synchronized. - ## Usage ``` @@ -19,9 +13,254 @@ go get github.com/chrislusf/weed-fs/go/weed ## Reference -For documents and bug reporting, Please visit - http://weed-fs.googlecode.com - For pre-compiled releases, https://bintray.com/chrislusf/Weed-FS/weed +## Introduction + +Seaweed-FS is a simple and highly scalable distributed file system. There are two objectives: + +1. to store billions of files! +2. to serve the files fast! + +Instead of supporting full POSIX file system semantics, Seaweed-FS choose to implement only a key~file mapping. Similar to the word "NoSQL", you can call it as "NoFS". + +Instead of managing all file metadata in a central master, Seaweed-FS choose to manages file volumes in the central master, and let volume servers manage files and the metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers' memories, allowing faster file access with just one disk read operation! + +Seaweed-FS models after [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). + +Seaweed-FS costs only 40 bytes disk storage for each file's metadata. It is so simple with O(1) disk read that you are welcome to challenge the performance with your actual use cases. + + +![](https://api.bintray.com/packages/chrislusf/Weed-FS/seaweed/images/download.png) +https://bintray.com/chrislusf/Weed-FS/seaweed Download latest compiled binaries for different platforms here. + +## Additional Features +* Can choose no replication or different replication level, rack and data center aware +* Automatic master servers failover. No single point of failure, SPOF. +* Automatic Gzip compression depending on file mime type +* Automatic compaction to reclaimed disk spaces after deletion or update +* Servers in the same cluster can have different disk spaces, different file systems, different OS +* Adding/Removing servers do not cause any data re-balancing +* Optional [filer server](https://code.google.com/p/weed-fs/wiki/DirectoriesAndFiles) provides "normal" directories and files via http +* For jpeg pictures, optionally fix the orientation. +* Support Etag, Accept-Range, Last-Modified, etc. + +## Example Usage +By default, the master node runs on port 9333, and the volume nodes runs on port 8080. +Here I will start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. Here I just use localhost as example. + +Seaweed-FS uses HTTP REST operations to write, read, delete. The return results are JSON or JSONP format. + +### Start Master Server + +``` +> ./weed master + +``` + +### Start Volume Servers ### + +``` +> weed volume -dir="/tmp/data1" -max=5 -mserver="localhost:9333" -port=8080 & +> weed volume -dir="/tmp/data2" -max=10 -mserver="localhost:9333" -port=8081 & + +``` + +### Write File ### +Here is a simple usage on how to save a file: + +``` +> curl http://localhost:9333/dir/assign +{"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"} +``` +First, send a HTTP request to get an fid and a volume server url. + +``` +> curl -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6 +{"size": 43234} +``` +Second, send a HTTP multipart POST request to the volume server url+'/'+fid, to really store the file content. + +For update, send another POST request with updated file content. + +For deletion, send a http DELETE request +``` +> curl -X DELETE http://127.0.0.1:8080/3,01637037d6 +``` +### Save File Id ### +Now you can save the fid, 3,01637037d6 in this case, to some database field. + +The number 3 here, is a volume id. After the comma, it's one file key, 01, and a file cookie, 637037d6. + +The volume id is an unsigned 32 bit integer. The file key is an unsigned 64bit integer. The file cookie is an unsigned 32bit integer, used to prevent URL guessing. + +The file key and file cookie are both coded in hex. You can store the tuple in your own format, or simply store the fid as string. + +If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most usage would not need 2^32 volumes. + +If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, 4-byte integer for file cookie. So 16 bytes are enough (more than enough). + +### Read File ### +Here is the example on how to render the URL. +``` +> curl http://localhost:9333/dir/lookup?volumeId=3 +{"locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]} +``` +First lookup the volume server's URLs by the file's volumeId. However, since usually there are not too many volume servers, and volumes does not move often, you can cache the results most of the time. Depends on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read. + +Now you can take the public url, render the url or directly read from the volume server via url: +``` + http://localhost:8080/3,01637037d6.jpg +``` +Notice we add an file extension ".jpg" here. It's optional and just one way for the client to specify the file content type. + +If you want a nicer URL, you can use one of these alternative URL formats: +``` + http://localhost:8080/3/01637037d6/my_preferred_name.jpg + http://localhost:8080/3/01637037d6.jpg + http://localhost:8080/3,01637037d6.jpg + http://localhost:8080/3/01637037d6 + http://localhost:8080/3,01637037d6 +``` + +### Rack-Aware and Data Center-Aware Replication ### +Seaweed-FS apply the replication strategy on a volume level. So when you are getting a file id, you can specify the replication strategy. For example: +``` +curl http://localhost:9333/dir/assign?replication=001 +``` + +Here is the meaning of the replication parameter +``` +000: no replication +001: replicate once on the same rack +010: replicate once on a different rack, but same data center +100: replicate once on a different data center +200: replicate twice on two different data center +110: replicate once on a different rack, and once on a different data center +``` + +More details about replication can be found here: +https://code.google.com/p/weed-fs/wiki/RackDataCenterAwareReplication + +You can also set the default replication strategy when starting the master server. + +### Allocate File Key on specific data center ### +Volume servers can start with a specific data center name. +``` + weed volume -dir=/tmp/1 -port=8080 -dataCenter=dc1 + weed volume -dir=/tmp/2 -port=8081 -dataCenter=dc2 +``` +Or the master server can determine the data center via volume server's IP address and settings in weed.conf file. + +Now when requesting a file key, an optional "dataCenter" parameter can limit the assigned volume to the specific data center. For example, this specify +``` + http://localhost:9333/dir/assign?dataCenter=dc1 +``` + +### Other Features ### + * [No Single Point of Failure](https://code.google.com/p/weed-fs/wiki/FailoverMasterServer) + * [Insert with your own keys](https://code.google.com/p/weed-fs/wiki/Optimization#Insert_with_your_own_keys) + * [ Chunking large files](https://code.google.com/p/weed-fs/wiki/Optimization#Upload_large_files) + * [Collection as a Simple Name Space](https://code.google.com/p/weed-fs/wiki/Optimization#Collection_as_a_Simple_Name_Space) + +## Architecture ## +Usually distributed file system split each file into chunks, and a central master keeps a mapping of a filename and a chunk index to chunk handles, and also which chunks each chunk server has. + +This has the draw back that the central master can not handle many small files efficiently, and since all read requests need to go through the chunk master, responses would be slow for many concurrent web users. + +Instead of managing chunks, Seaweed-FS choose to manage data volumes in the master server. Each data volume is size 32GB, and can hold a lot of files. And each storage node can has many data volumes. So the master node only needs to store the metadata about the volumes, which is fairly small amount of data and pretty static most of the time. + +The actual file metadata is stored in each volume on volume servers. Since each volume server only manage metadata of files on its own disk, and only 16 bytes for each file, all file access can read file metadata just from memory and only needs one disk operation to actually read file data. + +For comparison, consider that an xfs inode structure in Linux is 536 bytes. + +### Master Server and Volume Server ### +The architecture is fairly simple. The actual data is stored in volumes on storage nodes. One volume server can have multiple volumes, and can both support read and write access with basic authentication. + +All volumes are managed by a master server. The master server contains volume id to volume server mapping. This is fairly static information, and could be cached easily. + +On each write request, the master server also generates a file key, which is a growing 64bit unsigned integer. Since the write requests are not as busy as read requests, one master server should be able to handle the concurrency well. + + +### Write and Read files ### + +When a client sends a write request, the master server returns for the file. The client then contact the volume node and POST the file content via REST. + +When a client needs to read a file based on , it can ask the master server by the for the , or from cache. Then the client can HTTP GET the content via REST, or just render the URL on web pages and let browsers to fetch the content. + +Please see the example for details on write-read process. + +### Storage Size ### +In current implementation, each volume can be size of 8x2^32^=32G bytes. This is because of aligning contents to 8 bytes. We can be easily increased to 64G, or 128G, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment. + +There can be 2^32^ volumes. So total system size is 8 x 2^32^ x 2^32^ = 8 x 4G x 4G = 128GG bytes. (Sorry, I don't know the word for giga of giga bytes.) + +Each individual file size is limited to the volume size. + +### Saving memory ### +All file meta information on volume server is readable from memory without disk access. Each file just takes an 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own the space cost for the map. But usually the disk runs out before the memory does. + + +## Compared to Other File Systems## +Frankly, I don't use other distributed file systems too often. All seems more complicated than necessary. Please correct me if anything here is wrong. + +### Compared to Ceph ### +Ceph can be setup similar to Seaweed-FS as a key~blob store. It is much more complicated, with the need to support layers on top of it. Here is a more detailed comparison. https://code.google.com/p/weed-fs/issues/detail?id=44 + +Seaweed-FS is meant to be fast and simple, both during usage and during setup. If you do not understand how it works when you reach here, we failed! Jokes aside, you should not need any consulting service for it. + +Seaweed-FS has a centralized master to lookup free volumes, while Ceph uses hashing to locate its objects. Having a centralized master makes it easy to code and manage. HDFS/GFS has the single name node for years. Seaweed-FS now support multiple master nodes. + +Ceph hashing avoids SPOF, but makes it complicated when moving or adding servers. + +### Compared to HDFS ### +HDFS uses the chunk approach for each file, and is ideal for streaming large files. + +Seaweed-FS is ideal for serving relatively smaller files quickly and concurrently. + +Seaweed-FS can also store extra large files by splitting them into manageable data chunks, and store the file ids of the data chunks into a meta chunk. This is managed by "weed upload/download" tool, and the weed master or volume servers are agnostic about it. + +### Compared to MogileFS### +Seaweed-FS has 2 components: directory server, storage nodes. + +MogileFS has 3 components: tracers, database, storage nodes. + +One more layer means slower access, more operation complexity, more failure possibility. + +### Compared to GlusterFS ### +Seaweed-FS is not POSIX compliant, and has simple implementation. + +GlusterFS is POSIX compliant, much more complex. + +### Compared to Mongo's GridFS ### +Mongo's GridFS splits files into chunks and manage chunks in the central mongodb. For every read or write request, the database needs to query the metadata. It's OK if this is not a bottleneck yet, but for a lot of concurrent reads this unnecessary query could slow things down. + +Since files are chunked(default to 256KB), there will be multiple metadata readings and multiple chunk readings, linear to the file size. One 2.56MB file would require at least 20 disk read requests. + +On the contrary, Seaweed-FS uses large file volume of 32G size to store lots of files, and only manages file volumes in the master server. Each volume manages file metadata themselves. So all the file metadata is spread onto the volume nodes memories, and just one disk read is needed. + +## Dev plan ## + +More tools and documentation, on how to maintain and scale the system. For example, how to move volumes, automatically balancing data, how to grow volumes, how to check system status, etc. + +This is a super exciting project! And I need helpers! + +## Contributions ## + +To make contributions easier, I have mirrored a repo in github.com +``` + https://github.com/chrislusf/weed-fs.git +``` +## Disk Related topics ## + +### Hard Drive Performance ### +When testing read performance on Seaweed-FS, it basically becomes performance test your hard drive's random read speed. Hard Drive usually get 100MB/s~200MB/s. + +### Solid State Disk + +To modify or delete small files, SSD must delete a whole block at a time, and move content in existing blocks to a new block. SSD is fast when brand new, but will get fragmented over time and you have to garbage collect, compacting blocks. Seaweed-FS is friendly to SSD since it is append-only. Deletion and compaction are done on volume level in the background, not slowing reading and not causing fragmentation. + +## Not Planned + +POSIX support From 599d0ae21b8545a9dd667b42c7d804a2b6cbeb35 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 25 Sep 2014 22:32:12 -0700 Subject: [PATCH 23/58] Add benchmark to the introduction page. --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/README.md b/README.md index fc89c05e5..64c827718 100644 --- a/README.md +++ b/README.md @@ -264,3 +264,60 @@ To modify or delete small files, SSD must delete a whole block at a time, and mo ## Not Planned POSIX support + + +## Benchmark + +My Own Unscientific Single Machine Results on Mac Book with Solid State Disk, CPU: 1 Intel Core i7@2.2GHz. + +Write 1 million 1KB file: +``` +Concurrency Level: 64 +Time taken for tests: 182.456 seconds +Complete requests: 1048576 +Failed requests: 0 +Total transferred: 1073741824 bytes +Requests per second: 5747.01 [#/sec] +Transfer rate: 5747.01 [Kbytes/sec] + +Connection Times (ms) + min avg max std +Total: 0.3 10.9 430.9 5.7 + +Percentage of the requests served within a certain time (ms) + 50% 10.2 ms + 66% 12.0 ms + 75% 12.6 ms + 80% 12.9 ms + 90% 14.0 ms + 95% 14.9 ms + 98% 16.2 ms + 99% 17.3 ms + 100% 430.9 ms +``` + +Randomly read 1 million files: +``` +Concurrency Level: 64 +Time taken for tests: 80.732 seconds +Complete requests: 1048576 +Failed requests: 0 +Total transferred: 1073741824 bytes +Requests per second: 12988.37 [#/sec] +Transfer rate: 12988.37 [Kbytes/sec] + +Connection Times (ms) + min avg max std +Total: 0.0 4.7 254.3 6.3 + +Percentage of the requests served within a certain time (ms) + 50% 2.6 ms + 66% 2.9 ms + 75% 3.7 ms + 80% 4.7 ms + 90% 10.3 ms + 95% 16.6 ms + 98% 26.3 ms + 99% 34.8 ms + 100% 254.3 ms +``` From a09240391f59331b90ae65813a489d7bcb07a240 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 25 Sep 2014 22:34:06 -0700 Subject: [PATCH 24/58] Adjust wiki syntax error. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 64c827718..2575cf4b5 100644 --- a/README.md +++ b/README.md @@ -268,7 +268,7 @@ POSIX support ## Benchmark -My Own Unscientific Single Machine Results on Mac Book with Solid State Disk, CPU: 1 Intel Core i7@2.2GHz. +My Own Unscientific Single Machine Results on Mac Book with Solid State Disk, CPU: 1 Intel Core i7 2.2GHz. Write 1 million 1KB file: ``` From 88e7c98e6eed145a4101e4a18e899e8afee21278 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 8 Oct 2014 18:13:23 -0700 Subject: [PATCH 25/58] tweaking the docker file --- Dockerfile | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1243727a2..7332f660a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,16 @@ -FROM cydev/go -RUN go get code.google.com/p/weed-fs/go/weed +# install docker +# sudo docker build -t seaweed . +# docker run seaweed + +FROM golang + +# Copy the local package files to the container's workspace. +ADD . /go/src/github.com/chrislusf/weed-fs + +# Build the weed command inside the container. +RUN go get github.com/chrislusf/weed-fs/go/weed + EXPOSE 8080 EXPOSE 9333 VOLUME /data -ENTRYPOINT ["weed"] +ENTRYPOINT ["/go/bin/weed"] \ No newline at end of file From cbaaf1b539335b19278dad9baa2435f2100a29a5 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Tue, 14 Oct 2014 17:29:23 -0700 Subject: [PATCH 26/58] Update Dockerfile revert back to original version to keep documentation consistent. --- Dockerfile | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7332f660a..1243727a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,6 @@ -# install docker -# sudo docker build -t seaweed . -# docker run seaweed - -FROM golang - -# Copy the local package files to the container's workspace. -ADD . /go/src/github.com/chrislusf/weed-fs - -# Build the weed command inside the container. -RUN go get github.com/chrislusf/weed-fs/go/weed - +FROM cydev/go +RUN go get code.google.com/p/weed-fs/go/weed EXPOSE 8080 EXPOSE 9333 VOLUME /data -ENTRYPOINT ["/go/bin/weed"] \ No newline at end of file +ENTRYPOINT ["weed"] From 92653d2c413d2efb3a5322be6e8347146fc19542 Mon Sep 17 00:00:00 2001 From: bmcquee Date: Sat, 18 Oct 2014 20:17:28 -0700 Subject: [PATCH 27/58] fix link to download binaries --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2575cf4b5..ab07683dd 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ go get github.com/chrislusf/weed-fs/go/weed ## Reference For pre-compiled releases, - https://bintray.com/chrislusf/Weed-FS/weed + https://bintray.com/chrislusf/Weed-FS/seaweed ## Introduction From bff7b71389557235c8573a77b0c030765c85fd1f Mon Sep 17 00:00:00 2001 From: bmcquee Date: Sun, 19 Oct 2014 08:51:07 -0700 Subject: [PATCH 28/58] fix func name HasWriableVolume --- go/topology/topology.go | 2 +- go/weed/weed_server/master_server_handlers.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/topology/topology.go b/go/topology/topology.go index c90e8de0b..876b7c418 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -109,7 +109,7 @@ func (t *Topology) NextVolumeId() storage.VolumeId { return next } -func (t *Topology) HasWriableVolume(option *VolumeGrowOption) bool { +func (t *Topology) HasWriteableVolume(option *VolumeGrowOption) bool { vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 93e9e7d9a..7d54521d9 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -78,7 +78,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) return } - if !ms.Topo.HasWriableVolume(option) { + if !ms.Topo.HasWriteableVolume(option) { if ms.Topo.FreeSpace() <= 0 { w.WriteHeader(http.StatusNotFound) writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"}) @@ -86,7 +86,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } else { ms.vgLock.Lock() defer ms.vgLock.Unlock() - if !ms.Topo.HasWriableVolume(option) { + if !ms.Topo.HasWriteableVolume(option) { if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil { writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()}) return From 7ca10d8dcfe937ef91a6cfb261bf4f1909815071 Mon Sep 17 00:00:00 2001 From: bmcquee Date: Sun, 19 Oct 2014 20:03:00 -0700 Subject: [PATCH 29/58] add another logging line verbosity 2 --- go/weed/weed_server/raft_server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index e41867076..9a21249a9 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -44,6 +44,7 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin var err error transporter := raft.NewHTTPTransporter("/cluster", 0) transporter.Transport.MaxIdleConnsPerHost = 1024 + glog.V(1).Infof("Starting RaftServer with IP:%v:", httpAddr) s.raftServer, err = raft.NewServer(s.httpAddr, s.dataDir, transporter, nil, topo, "") if err != nil { From 626b896448473b3ce7f2ec40b644c2bfd0e3c714 Mon Sep 17 00:00:00 2001 From: bmcquee Date: Sun, 19 Oct 2014 23:01:15 -0700 Subject: [PATCH 30/58] change wriable to writeable --- go/weed/weed_server/master_server_handlers_admin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index c9a8020c2..17a09dac7 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -143,7 +143,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r * } } -func (ms *MasterServer) hasWriableVolume(option *topology.VolumeGrowOption) bool { +func (ms *MasterServer) hasWriteableVolume(option *topology.VolumeGrowOption) bool { vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } From 02ae8b98a579dda3dd22d97a48a2890aec84cb62 Mon Sep 17 00:00:00 2001 From: bmcquee Date: Sun, 19 Oct 2014 23:07:58 -0700 Subject: [PATCH 31/58] Revert "change wriable to writeable" This reverts commit 626b896448473b3ce7f2ec40b644c2bfd0e3c714. --- go/weed/weed_server/master_server_handlers_admin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 17a09dac7..c9a8020c2 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -143,7 +143,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r * } } -func (ms *MasterServer) hasWriteableVolume(option *topology.VolumeGrowOption) bool { +func (ms *MasterServer) hasWriableVolume(option *topology.VolumeGrowOption) bool { vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } From bd664def45925d81dfae9c7edfb244d2367170ca Mon Sep 17 00:00:00 2001 From: bmcquee Date: Sun, 19 Oct 2014 23:08:01 -0700 Subject: [PATCH 32/58] Revert "fix func name HasWriableVolume" This reverts commit bff7b71389557235c8573a77b0c030765c85fd1f. --- go/topology/topology.go | 2 +- go/weed/weed_server/master_server_handlers.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/topology/topology.go b/go/topology/topology.go index 876b7c418..c90e8de0b 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -109,7 +109,7 @@ func (t *Topology) NextVolumeId() storage.VolumeId { return next } -func (t *Topology) HasWriteableVolume(option *VolumeGrowOption) bool { +func (t *Topology) HasWriableVolume(option *VolumeGrowOption) bool { vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 7d54521d9..93e9e7d9a 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -78,7 +78,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) return } - if !ms.Topo.HasWriteableVolume(option) { + if !ms.Topo.HasWriableVolume(option) { if ms.Topo.FreeSpace() <= 0 { w.WriteHeader(http.StatusNotFound) writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"}) @@ -86,7 +86,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } else { ms.vgLock.Lock() defer ms.vgLock.Unlock() - if !ms.Topo.HasWriteableVolume(option) { + if !ms.Topo.HasWriableVolume(option) { if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil { writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()}) return From e9a8999f63f79406249debb8023be014e8e12eda Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 21 Oct 2014 01:27:06 -0700 Subject: [PATCH 33/58] print error the correct way. --- go/storage/volume.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/go/storage/volume.go b/go/storage/volume.go index de79e9107..60ee1a0c4 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -2,9 +2,9 @@ package storage import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "errors" "fmt" + "github.com/chrislusf/weed-fs/go/glog" "io" "os" "path" @@ -72,7 +72,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error { if e != nil { if !os.IsPermission(e) { - return fmt.Errorf("cannot load Volume Data %s.dat: %s", fileName, e.Error()) + return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, e) } } @@ -92,12 +92,12 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error { if v.readOnly { glog.V(1).Infoln("open to read file", fileName+".idx") if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); e != nil { - return fmt.Errorf("cannot read Volume Index %s.idx: %s", fileName, e.Error()) + return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, e) } } else { glog.V(1).Infoln("open to write file", fileName+".idx") if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); e != nil { - return fmt.Errorf("cannot write Volume Index %s.idx: %s", fileName, e.Error()) + return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, e) } } glog.V(0).Infoln("loading file", fileName+".idx", "readonly", v.readOnly) @@ -115,7 +115,7 @@ func (v *Volume) Size() int64 { if e == nil { return stat.Size() } - glog.V(0).Infof("Failed to read file size %s %s", v.dataFile.Name(), e.Error()) + glog.V(0).Infof("Failed to read file size %s %v", v.dataFile.Name(), e) return -1 } func (v *Volume) Close() { @@ -170,6 +170,7 @@ func (v *Volume) write(n *Needle) (size uint32, err error) { } var offset int64 if offset, err = v.dataFile.Seek(0, 2); err != nil { + glog.V(0).Infof("faile to seek the end of file: %v", err) return } @@ -177,21 +178,21 @@ func (v *Volume) write(n *Needle) (size uint32, err error) { if offset%NeedlePaddingSize != 0 { offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize) if offset, err = v.dataFile.Seek(offset, 0); err != nil { - glog.V(4).Infof("failed to align in datafile %s: %s", v.dataFile.Name(), err.Error()) + glog.V(0).Infof("failed to align in datafile %s: %v", v.dataFile.Name(), err) return } } if size, err = n.Append(v.dataFile, v.Version()); err != nil { if e := v.dataFile.Truncate(offset); e != nil { - err = fmt.Errorf("%s\ncannot truncate %s: %s", err, v.dataFile.Name(), e.Error()) + err = fmt.Errorf("%s\ncannot truncate %s: %v", err, v.dataFile.Name(), e) } return } nv, ok := v.nm.Get(n.Id) if !ok || int64(nv.Offset)*NeedlePaddingSize < offset { if _, err = v.nm.Put(n.Id, uint32(offset/NeedlePaddingSize), n.Size); err != nil { - glog.V(4).Infof("failed to save in needle map %d: %s", n.Id, err.Error()) + glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err) } } if v.lastModifiedTime < n.LastModified { @@ -292,13 +293,13 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId, offset := int64(SuperBlockSize) n, rest, e := ReadNeedleHeader(v.dataFile, version, offset) if e != nil { - err = fmt.Errorf("cannot read needle header: %s", e) + err = fmt.Errorf("cannot read needle header: %v", e) return } for n != nil { if readNeedleBody { if err = n.ReadNeedleBody(v.dataFile, version, offset+int64(NeedleHeaderSize), rest); err != nil { - err = fmt.Errorf("cannot read needle body: %s", err) + err = fmt.Errorf("cannot read needle body: %v", err) return } } @@ -310,7 +311,7 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId, if err == io.EOF { return nil } - return fmt.Errorf("cannot read needle header: %s", err) + return fmt.Errorf("cannot read needle header: %v", err) } } @@ -360,7 +361,7 @@ func (v *Volume) ensureConvertIdxToCdb(fileName string) (cdbCanRead bool) { defer indexFile.Close() glog.V(0).Infof("converting %s.idx to %s.cdb", fileName, fileName) if e = ConvertIndexToCdb(fileName+".cdb", indexFile); e != nil { - glog.V(0).Infof("error converting %s.idx to %s.cdb: %s", fileName, fileName, e.Error()) + glog.V(0).Infof("error converting %s.idx to %s.cdb: %v", fileName, fileName, e) return false } return true From 670b240a266f23fe9d8e5120c61d32065a1519b3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 21 Oct 2014 01:27:40 -0700 Subject: [PATCH 34/58] Fix help text error. --- go/weed/compact.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/weed/compact.go b/go/weed/compact.go index a99e6c93e..71c4ea90f 100644 --- a/go/weed/compact.go +++ b/go/weed/compact.go @@ -12,7 +12,7 @@ func init() { var cmdCompact = &Command{ UsageLine: "compact -dir=/tmp -volumeId=234", - Short: "run weed tool compact on volume file if corrupted", + Short: "run weed tool compact on volume file", Long: `Force an compaction to remove deleted files from volume files. The compacted .dat file is stored as .cpd file. The compacted .idx file is stored as .cpx file. From b5aa2ef6050bc052e73536481893011f55005099 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 21 Oct 2014 01:28:17 -0700 Subject: [PATCH 35/58] Add master bind ip address option. --- go/weed/master.go | 3 ++- go/weed/volume.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go/weed/master.go b/go/weed/master.go index 6617c8ca6..f96ff4c08 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -29,6 +29,7 @@ var cmdMaster = &Command{ var ( mport = cmdMaster.Flag.Int("port", 9333, "http listen port") masterIp = cmdMaster.Flag.String("ip", "", "master listening ip address, default to listen on all network interfaces") + masterBindIp = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") mPublicIp = cmdMaster.Flag.String("publicIp", "", "peer accessible |") metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data") masterPeers = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list") @@ -61,7 +62,7 @@ func runMaster(cmd *Command, args []string) bool { *volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold, masterWhiteList, ) - listeningAddress := *masterIp + ":" + strconv.Itoa(*mport) + listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport) glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress) diff --git a/go/weed/volume.go b/go/weed/volume.go index 17d03f0c5..ce05fcdf3 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -30,7 +30,7 @@ var ( maxVolumeCounts = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...") ip = cmdVolume.Flag.String("ip", "", "ip or server name") publicIp = cmdVolume.Flag.String("publicIp", "", "Publicly accessible ") - bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") + volumeBindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") masterNode = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location") vpulse = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting") vTimeout = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds") @@ -85,7 +85,7 @@ func runVolume(cmd *Command, args []string) bool { *fixJpgOrientation, ) - listeningAddress := *bindIp + ":" + strconv.Itoa(*vport) + listeningAddress := *volumeBindIp + ":" + strconv.Itoa(*vport) glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress) From b85eb293feda0c4b2c322a3d07d17df566df02a4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 23 Oct 2014 09:55:05 -0700 Subject: [PATCH 36/58] fix doc about submit api --- docs/api.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index f26408f5a..b67463960 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -187,10 +187,11 @@ Upload File Directly .. code-block:: bash - curl -F file=@/home/chris/myphoto.jpg http://localhost:8080/submit + curl -F file=@/home/chris/myphoto.jpg http://localhost:9333/submit {"fid":"3,01fbe0dc6f1f38","fileName":"myphoto.jpg","fileUrl":"localhost:8080/3,01fbe0dc6f1f38","size":68231} -This API is a little convenient. The volume server would contact the master to get an file id and store it to the right volume server(not necessarily itself). +This API is just for convenience. The master server would get an file id and store the file to the right volume server. +It is a convenient API and does not support different parameters when assigning file id. (or you can add the support and send a push request.) Delete File *********************************** From 30bcda7136f847b59ac6e6610a40829dc10ec8cd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 25 Oct 2014 18:10:32 -0700 Subject: [PATCH 37/58] fix typo --- go/topology/topology.go | 2 +- go/weed/weed_server/master_server_handlers.go | 4 ++-- go/weed/weed_server/master_server_handlers_admin.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/topology/topology.go b/go/topology/topology.go index c90e8de0b..a81829bb6 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -109,7 +109,7 @@ func (t *Topology) NextVolumeId() storage.VolumeId { return next } -func (t *Topology) HasWriableVolume(option *VolumeGrowOption) bool { +func (t *Topology) hasWritableVolume(option *VolumeGrowOption) bool { vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 93e9e7d9a..be2a7ca7e 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -78,7 +78,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) return } - if !ms.Topo.HasWriableVolume(option) { + if !ms.Topo.hasWritableVolume(option) { if ms.Topo.FreeSpace() <= 0 { w.WriteHeader(http.StatusNotFound) writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"}) @@ -86,7 +86,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } else { ms.vgLock.Lock() defer ms.vgLock.Unlock() - if !ms.Topo.HasWriableVolume(option) { + if !ms.Topo.hasWritableVolume(option) { if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil { writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()}) return diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index c9a8020c2..080405e54 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -143,7 +143,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r * } } -func (ms *MasterServer) hasWriableVolume(option *topology.VolumeGrowOption) bool { +func (ms *MasterServer) hasWritableVolume(option *topology.VolumeGrowOption) bool { vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } From f527fc1d5ee975fd29e49bf542d893a2d433a491 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 25 Oct 2014 23:45:31 -0700 Subject: [PATCH 38/58] adjust visibility --- go/topology/topology.go | 2 +- go/weed/weed_server/master_server_handlers.go | 4 ++-- go/weed/weed_server/master_server_handlers_admin.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/topology/topology.go b/go/topology/topology.go index a81829bb6..cfce0c9a8 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -109,7 +109,7 @@ func (t *Topology) NextVolumeId() storage.VolumeId { return next } -func (t *Topology) hasWritableVolume(option *VolumeGrowOption) bool { +func (t *Topology) HasWritableVolume(option *VolumeGrowOption) bool { vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index be2a7ca7e..d7f1f4ce3 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -78,7 +78,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) return } - if !ms.Topo.hasWritableVolume(option) { + if !ms.Topo.HasWritableVolume(option) { if ms.Topo.FreeSpace() <= 0 { w.WriteHeader(http.StatusNotFound) writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"}) @@ -86,7 +86,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } else { ms.vgLock.Lock() defer ms.vgLock.Unlock() - if !ms.Topo.hasWritableVolume(option) { + if !ms.Topo.HasWritableVolume(option) { if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil { writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()}) return diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 080405e54..1a2c6b8e0 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -143,7 +143,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r * } } -func (ms *MasterServer) hasWritableVolume(option *topology.VolumeGrowOption) bool { +func (ms *MasterServer) HasWritableVolume(option *topology.VolumeGrowOption) bool { vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } From a5d6e70299559c10126f6ef248f475152303a79a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 26 Oct 2014 11:25:02 -0700 Subject: [PATCH 39/58] fix commenting error. --- go/weed/export.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/weed/export.go b/go/weed/export.go index 81bc21f6e..9f33d852f 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -3,9 +3,9 @@ package main import ( "archive/tar" "bytes" + "fmt" "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/storage" - "fmt" "os" "path" "strconv" @@ -36,7 +36,7 @@ var cmdExport = &Command{ var ( exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "input data directory to store volume data files") exportCollection = cmdExport.Flag.String("collection", "", "the volume collection name") - exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.") + exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.") dest = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout") format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}") tarFh *tar.Writer From 179d36ba0e35e4bdad86988a49828836b649e3df Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 26 Oct 2014 11:34:55 -0700 Subject: [PATCH 40/58] formatting code by: goimports -w=true . --- go/filer/client_operations.go | 6 +++--- go/filer/directory.go | 2 -- go/filer/directory_in_map.go | 3 ++- go/filer/filer.go | 2 -- go/filer/filer_embedded.go | 3 ++- go/filer/files_in_leveldb.go | 1 + go/glog/convenient_api.go | 2 -- go/images/orientation.go | 3 ++- go/images/resizing.go | 3 ++- go/operation/assign_file_id.go | 5 +++-- go/operation/data_struts.go | 2 -- go/operation/delete_content.go | 3 ++- go/operation/list_masters.go | 3 ++- go/operation/lookup.go | 3 ++- go/operation/submit.go | 3 ++- go/operation/system_message_test.go | 3 ++- go/operation/upload_content.go | 3 ++- go/sequence/sequence.go | 2 -- go/stats/disk.go | 2 -- go/stats/disk_notsupported.go | 2 -- go/stats/memory_notsupported.go | 2 -- go/storage/cdb_map.go | 5 +++-- go/storage/cdb_map_test.go | 3 ++- go/storage/compact_map.go | 2 -- go/storage/compact_map_perf_test.go | 5 +++-- go/storage/compress.go | 3 ++- go/storage/crc.go | 3 ++- go/storage/file_id.go | 5 +++-- go/storage/needle.go | 7 ++++--- go/storage/needle_map.go | 5 +++-- go/storage/needle_read_write.go | 5 +++-- go/storage/store.go | 9 +++++---- go/storage/store_vacuum.go | 3 ++- go/storage/volume.go | 3 ++- go/storage/volume_super_block.go | 3 ++- go/storage/volume_vacuum.go | 3 ++- go/storage/volume_version.go | 2 -- go/tools/read_index.go | 3 ++- go/topology/allocate_volume.go | 5 +++-- go/topology/data_center.go | 2 -- go/topology/data_node.go | 3 ++- go/topology/node.go | 5 +++-- go/topology/store_replicate.go | 5 +++-- go/topology/topology.go | 7 ++++--- go/topology/topology_event_handling.go | 5 +++-- go/topology/topology_map.go | 2 -- go/topology/topology_vacuum.go | 7 ++++--- go/topology/volume_growth.go | 5 +++-- go/topology/volume_growth_test.go | 5 +++-- go/topology/volume_layout.go | 5 +++-- go/topology/volume_location_list.go | 2 -- go/util/config.go | 3 ++- go/util/constants.go | 2 -- go/util/file_util.go | 3 ++- go/util/net_timeout.go | 3 ++- go/weed/benchmark.go | 7 ++++--- go/weed/download.go | 5 +++-- go/weed/export.go | 5 +++-- go/weed/filer.go | 7 ++++--- go/weed/fix.go | 5 +++-- go/weed/master.go | 9 +++++---- go/weed/mount.go | 2 -- go/weed/mount_std.go | 7 ++++--- go/weed/server.go | 9 +++++---- go/weed/shell.go | 3 ++- go/weed/signal_handling_notsupported.go | 2 -- go/weed/upload.go | 3 ++- go/weed/version.go | 3 ++- go/weed/volume.go | 7 ++++--- go/weed/volume_test.go | 3 ++- go/weed/weed.go | 3 ++- go/weed/weed_server/common.go | 11 ++++++----- go/weed/weed_server/filer_server.go | 5 +++-- go/weed/weed_server/filer_server_handlers.go | 9 +++++---- go/weed/weed_server/filer_server_handlers_admin.go | 3 ++- go/weed/weed_server/master_server.go | 9 +++++---- go/weed/weed_server/master_server_handlers.go | 7 ++++--- go/weed/weed_server/master_server_handlers_admin.go | 13 +++++++------ go/weed/weed_server/raft_server.go | 9 +++++---- go/weed/weed_server/raft_server_handlers.go | 7 ++++--- go/weed/weed_server/volume_server.go | 5 +++-- go/weed/weed_server/volume_server_handlers.go | 13 +++++++------ go/weed/weed_server/volume_server_handlers_admin.go | 5 +++-- .../weed_server/volume_server_handlers_vacuum.go | 3 ++- 84 files changed, 205 insertions(+), 170 deletions(-) diff --git a/go/filer/client_operations.go b/go/filer/client_operations.go index 0b006289f..b38368735 100644 --- a/go/filer/client_operations.go +++ b/go/filer/client_operations.go @@ -1,12 +1,12 @@ package filer -import () - import ( - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "fmt" + + "github.com/chrislusf/weed-fs/go/util" + "net/url" ) diff --git a/go/filer/directory.go b/go/filer/directory.go index 956a2f504..66d1aeba5 100644 --- a/go/filer/directory.go +++ b/go/filer/directory.go @@ -1,7 +1,5 @@ package filer -import () - type DirectoryId int32 type DirectoryEntry struct { diff --git a/go/filer/directory_in_map.go b/go/filer/directory_in_map.go index 1d88a78be..ee601066c 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/directory_in_map.go @@ -2,7 +2,6 @@ package filer import ( "bufio" - "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "os" @@ -10,6 +9,8 @@ import ( "strconv" "strings" "sync" + + "github.com/chrislusf/weed-fs/go/util" ) var writeLock sync.Mutex //serialize changes to dir.log diff --git a/go/filer/filer.go b/go/filer/filer.go index de877fc1f..bf4a1cb19 100644 --- a/go/filer/filer.go +++ b/go/filer/filer.go @@ -1,7 +1,5 @@ package filer -import () - type FileId string //file id on weedfs type FileEntry struct { diff --git a/go/filer/filer_embedded.go b/go/filer/filer_embedded.go index 3d3dac941..0b9b4e668 100644 --- a/go/filer/filer_embedded.go +++ b/go/filer/filer_embedded.go @@ -1,11 +1,12 @@ package filer import ( - "github.com/chrislusf/weed-fs/go/operation" "errors" "fmt" "path/filepath" "strings" + + "github.com/chrislusf/weed-fs/go/operation" ) type FilerEmbedded struct { diff --git a/go/filer/files_in_leveldb.go b/go/filer/files_in_leveldb.go index 41fbc74bd..781bb0e5f 100644 --- a/go/filer/files_in_leveldb.go +++ b/go/filer/files_in_leveldb.go @@ -2,6 +2,7 @@ package filer import ( "bytes" + "github.com/chrislusf/weed-fs/go/glog" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" diff --git a/go/glog/convenient_api.go b/go/glog/convenient_api.go index 3c378083f..cb43d60e2 100644 --- a/go/glog/convenient_api.go +++ b/go/glog/convenient_api.go @@ -1,7 +1,5 @@ package glog -import () - /* Copying the original glog because it is missing several convenient methods. 1. remove nano time in log format diff --git a/go/images/orientation.go b/go/images/orientation.go index 41ed3f0af..4bff89311 100644 --- a/go/images/orientation.go +++ b/go/images/orientation.go @@ -2,11 +2,12 @@ package images import ( "bytes" - "github.com/rwcarlsen/goexif/exif" "image" "image/draw" "image/jpeg" "log" + + "github.com/rwcarlsen/goexif/exif" ) //many code is copied from http://camlistore.org/pkg/images/images.go diff --git a/go/images/resizing.go b/go/images/resizing.go index 08a1e15d2..e9de5f7d7 100644 --- a/go/images/resizing.go +++ b/go/images/resizing.go @@ -2,11 +2,12 @@ package images import ( "bytes" - "github.com/disintegration/imaging" "image" "image/gif" "image/jpeg" "image/png" + + "github.com/disintegration/imaging" ) func Resized(ext string, data []byte, width, height int) (resized []byte, w int, h int) { diff --git a/go/operation/assign_file_id.go b/go/operation/assign_file_id.go index 4e72ad939..672bfa99c 100644 --- a/go/operation/assign_file_id.go +++ b/go/operation/assign_file_id.go @@ -1,12 +1,13 @@ package operation import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" "strconv" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type AssignResult struct { diff --git a/go/operation/data_struts.go b/go/operation/data_struts.go index 09fab05d1..4980f9913 100644 --- a/go/operation/data_struts.go +++ b/go/operation/data_struts.go @@ -1,7 +1,5 @@ package operation -import () - type JoinResult struct { VolumeSizeLimit uint64 `json:"VolumeSizeLimit,omitempty"` Error string `json:"error,omitempty"` diff --git a/go/operation/delete_content.go b/go/operation/delete_content.go index 84391b634..416a852b3 100644 --- a/go/operation/delete_content.go +++ b/go/operation/delete_content.go @@ -1,12 +1,13 @@ package operation import ( - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" "strings" "sync" + + "github.com/chrislusf/weed-fs/go/util" ) type DeleteResult struct { diff --git a/go/operation/list_masters.go b/go/operation/list_masters.go index 7d46a9ebc..7ada94243 100644 --- a/go/operation/list_masters.go +++ b/go/operation/list_masters.go @@ -1,9 +1,10 @@ package operation import ( + "encoding/json" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/util" - "encoding/json" ) type ClusterStatusResult struct { diff --git a/go/operation/lookup.go b/go/operation/lookup.go index ebf153d27..e6b6658da 100644 --- a/go/operation/lookup.go +++ b/go/operation/lookup.go @@ -1,7 +1,6 @@ package operation import ( - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" _ "fmt" @@ -9,6 +8,8 @@ import ( "net/url" "strings" "time" + + "github.com/chrislusf/weed-fs/go/util" ) type Location struct { diff --git a/go/operation/submit.go b/go/operation/submit.go index 3e09c2edf..62db46617 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -2,13 +2,14 @@ package operation import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "io" "mime" "os" "path" "strconv" "strings" + + "github.com/chrislusf/weed-fs/go/glog" ) type FilePart struct { diff --git a/go/operation/system_message_test.go b/go/operation/system_message_test.go index 2731d0b2f..b5624c258 100644 --- a/go/operation/system_message_test.go +++ b/go/operation/system_message_test.go @@ -1,10 +1,11 @@ package operation import ( - proto "code.google.com/p/goprotobuf/proto" "encoding/json" "log" "testing" + + proto "code.google.com/p/goprotobuf/proto" ) func TestSerialDeserial(t *testing.T) { diff --git a/go/operation/upload_content.go b/go/operation/upload_content.go index 38737702d..480d76dca 100644 --- a/go/operation/upload_content.go +++ b/go/operation/upload_content.go @@ -2,7 +2,6 @@ package operation import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "encoding/json" "errors" "fmt" @@ -14,6 +13,8 @@ import ( "net/textproto" "path/filepath" "strings" + + "github.com/chrislusf/weed-fs/go/glog" ) type UploadResult struct { diff --git a/go/sequence/sequence.go b/go/sequence/sequence.go index 5a1bceaaf..1aa167b6b 100644 --- a/go/sequence/sequence.go +++ b/go/sequence/sequence.go @@ -1,7 +1,5 @@ package sequence -import () - type Sequencer interface { NextFileId(count int) (uint64, int) SetMax(uint64) diff --git a/go/stats/disk.go b/go/stats/disk.go index d5275e571..46d8c465e 100644 --- a/go/stats/disk.go +++ b/go/stats/disk.go @@ -1,7 +1,5 @@ package stats -import () - type DiskStatus struct { Dir string All uint64 diff --git a/go/stats/disk_notsupported.go b/go/stats/disk_notsupported.go index 37f9bc47d..e380d27ea 100644 --- a/go/stats/disk_notsupported.go +++ b/go/stats/disk_notsupported.go @@ -2,8 +2,6 @@ package stats -import () - func (disk *DiskStatus) fillInStatus() { return } diff --git a/go/stats/memory_notsupported.go b/go/stats/memory_notsupported.go index 64c3d7c2f..ba8229364 100644 --- a/go/stats/memory_notsupported.go +++ b/go/stats/memory_notsupported.go @@ -2,8 +2,6 @@ package stats -import () - func (mem *MemStatus) fillInStatus() { return } diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go index 1869a563e..fbb59e9c0 100644 --- a/go/storage/cdb_map.go +++ b/go/storage/cdb_map.go @@ -1,13 +1,14 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "fmt" - "github.com/tgulacsi/go-cdb" "os" "path/filepath" + + "github.com/chrislusf/weed-fs/go/util" + "github.com/tgulacsi/go-cdb" ) // CDB-backed read-only needle map diff --git a/go/storage/cdb_map_test.go b/go/storage/cdb_map_test.go index cff7dfa61..ed690f44f 100644 --- a/go/storage/cdb_map_test.go +++ b/go/storage/cdb_map_test.go @@ -1,11 +1,12 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" "math/rand" "os" "runtime" "testing" + + "github.com/chrislusf/weed-fs/go/glog" ) var testIndexFilename string = "../../test/sample.idx" diff --git a/go/storage/compact_map.go b/go/storage/compact_map.go index 9cfc3e8f7..6ac18b012 100644 --- a/go/storage/compact_map.go +++ b/go/storage/compact_map.go @@ -1,7 +1,5 @@ package storage -import () - type NeedleValue struct { Key Key Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index ef43de25b..f74684225 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -1,11 +1,12 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "log" "os" "testing" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) func TestMemoryUsage(t *testing.T) { diff --git a/go/storage/compress.go b/go/storage/compress.go index a353c9d3a..5efc9e1f1 100644 --- a/go/storage/compress.go +++ b/go/storage/compress.go @@ -2,11 +2,12 @@ package storage import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "compress/flate" "compress/gzip" "io/ioutil" "strings" + + "github.com/chrislusf/weed-fs/go/glog" ) /* diff --git a/go/storage/crc.go b/go/storage/crc.go index 7aa400959..af25b9e53 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -1,9 +1,10 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/util" "fmt" "hash/crc32" + + "github.com/chrislusf/weed-fs/go/util" ) var table = crc32.MakeTable(crc32.Castagnoli) diff --git a/go/storage/file_id.go b/go/storage/file_id.go index ec566826c..f6e36a98c 100644 --- a/go/storage/file_id.go +++ b/go/storage/file_id.go @@ -1,11 +1,12 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "encoding/hex" "errors" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type FileId struct { diff --git a/go/storage/needle.go b/go/storage/needle.go index daede321b..aa3206920 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -1,9 +1,6 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/images" - "github.com/chrislusf/weed-fs/go/util" "encoding/hex" "errors" "io/ioutil" @@ -13,6 +10,10 @@ import ( "strconv" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/images" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index dca2e6c5d..504ca1552 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -1,11 +1,12 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "os" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type NeedleMapper interface { diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index bf452ba37..663b5abbd 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -1,12 +1,13 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "errors" "fmt" "io" "os" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/store.go b/go/storage/store.go index e7a9dac94..80d8a30b8 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -1,10 +1,6 @@ package storage import ( - proto "code.google.com/p/goprotobuf/proto" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "fmt" @@ -12,6 +8,11 @@ import ( "math/rand" "strconv" "strings" + + proto "code.google.com/p/goprotobuf/proto" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/store_vacuum.go b/go/storage/store_vacuum.go index 3527e4f59..209e3b4b3 100644 --- a/go/storage/store_vacuum.go +++ b/go/storage/store_vacuum.go @@ -1,9 +1,10 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" "fmt" "strconv" + + "github.com/chrislusf/weed-fs/go/glog" ) func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) { diff --git a/go/storage/volume.go b/go/storage/volume.go index 60ee1a0c4..5b0a83605 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -4,12 +4,13 @@ import ( "bytes" "errors" "fmt" - "github.com/chrislusf/weed-fs/go/glog" "io" "os" "path" "sync" "time" + + "github.com/chrislusf/weed-fs/go/glog" ) type Volume struct { diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index a7e86b1c3..57e0deea9 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -1,9 +1,10 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" "fmt" "os" + + "github.com/chrislusf/weed-fs/go/glog" ) const ( diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go index b348434d2..7e026a61d 100644 --- a/go/storage/volume_vacuum.go +++ b/go/storage/volume_vacuum.go @@ -1,10 +1,11 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" "fmt" "os" "time" + + "github.com/chrislusf/weed-fs/go/glog" ) func (v *Volume) garbageLevel() float64 { diff --git a/go/storage/volume_version.go b/go/storage/volume_version.go index 9702ae904..2e9f58aa2 100644 --- a/go/storage/volume_version.go +++ b/go/storage/volume_version.go @@ -1,7 +1,5 @@ package storage -import () - type Version uint8 const ( diff --git a/go/tools/read_index.go b/go/tools/read_index.go index b99c5b6b8..1104dc348 100644 --- a/go/tools/read_index.go +++ b/go/tools/read_index.go @@ -1,11 +1,12 @@ package main import ( - "github.com/chrislusf/weed-fs/go/storage" "flag" "fmt" "log" "os" + + "github.com/chrislusf/weed-fs/go/storage" ) var ( diff --git a/go/topology/allocate_volume.go b/go/topology/allocate_volume.go index 6562e9ac5..a791b4c1c 100644 --- a/go/topology/allocate_volume.go +++ b/go/topology/allocate_volume.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" + + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) type AllocateVolumeResult struct { diff --git a/go/topology/data_center.go b/go/topology/data_center.go index ebd07803b..bcf2dfd31 100644 --- a/go/topology/data_center.go +++ b/go/topology/data_center.go @@ -1,7 +1,5 @@ package topology -import () - type DataCenter struct { NodeImpl } diff --git a/go/topology/data_node.go b/go/topology/data_node.go index c3b90470f..109bd037f 100644 --- a/go/topology/data_node.go +++ b/go/topology/data_node.go @@ -1,9 +1,10 @@ package topology import ( + "strconv" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/storage" - "strconv" ) type DataNode struct { diff --git a/go/topology/node.go b/go/topology/node.go index 54118802e..10955fa72 100644 --- a/go/topology/node.go +++ b/go/topology/node.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "errors" "math/rand" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) type NodeId string diff --git a/go/topology/store_replicate.go b/go/topology/store_replicate.go index 6ea019bd8..0c52f9d30 100644 --- a/go/topology/store_replicate.go +++ b/go/topology/store_replicate.go @@ -2,12 +2,13 @@ package topology import ( "bytes" + "net/http" + "strconv" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/operation" "github.com/chrislusf/weed-fs/go/storage" "github.com/chrislusf/weed-fs/go/util" - "net/http" - "strconv" ) func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.VolumeId, needle *storage.Needle, r *http.Request) (size uint32, errorStatus string) { diff --git a/go/topology/topology.go b/go/topology/topology.go index cfce0c9a8..eb64d336c 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -1,14 +1,15 @@ package topology import ( + "errors" + "io/ioutil" + "math/rand" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/operation" "github.com/chrislusf/weed-fs/go/sequence" "github.com/chrislusf/weed-fs/go/storage" - "errors" "github.com/goraft/raft" - "io/ioutil" - "math/rand" ) type Topology struct { diff --git a/go/topology/topology_event_handling.go b/go/topology/topology_event_handling.go index eb4491484..7e36568b6 100644 --- a/go/topology/topology_event_handling.go +++ b/go/topology/topology_event_handling.go @@ -1,10 +1,11 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "math/rand" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func (t *Topology) StartRefreshWritableVolumes(garbageThreshold string) { diff --git a/go/topology/topology_map.go b/go/topology/topology_map.go index d6400c988..af95c6536 100644 --- a/go/topology/topology_map.go +++ b/go/topology/topology_map.go @@ -1,7 +1,5 @@ package topology -import () - func (t *Topology) ToMap() interface{} { m := make(map[string]interface{}) m["Max"] = t.GetMaxVolumeCount() diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index 72846f20b..97a76026d 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -1,13 +1,14 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList, garbageThreshold string) bool { diff --git a/go/topology/volume_growth.go b/go/topology/volume_growth.go index 2859d3992..b1f241990 100644 --- a/go/topology/volume_growth.go +++ b/go/topology/volume_growth.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "fmt" "math/rand" "sync" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) /* diff --git a/go/topology/volume_growth_test.go b/go/topology/volume_growth_test.go index 5581c87ce..267b36042 100644 --- a/go/topology/volume_growth_test.go +++ b/go/topology/volume_growth_test.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/sequence" - "github.com/chrislusf/weed-fs/go/storage" "encoding/json" "fmt" "testing" + + "github.com/chrislusf/weed-fs/go/sequence" + "github.com/chrislusf/weed-fs/go/storage" ) var topologyLayout = ` diff --git a/go/topology/volume_layout.go b/go/topology/volume_layout.go index 7bb0cf7e3..de72bf895 100644 --- a/go/topology/volume_layout.go +++ b/go/topology/volume_layout.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "errors" "math/rand" "sync" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) // mapping from volume to its locations, inverted from server to volume diff --git a/go/topology/volume_location_list.go b/go/topology/volume_location_list.go index 176f469b9..011614013 100644 --- a/go/topology/volume_location_list.go +++ b/go/topology/volume_location_list.go @@ -1,7 +1,5 @@ package topology -import () - type VolumeLocationList struct { list []*DataNode } diff --git a/go/util/config.go b/go/util/config.go index 050fd0e64..4cf1d7c64 100644 --- a/go/util/config.go +++ b/go/util/config.go @@ -10,9 +10,10 @@ package util import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "encoding/json" "os" + + "github.com/chrislusf/weed-fs/go/glog" ) type Config struct { diff --git a/go/util/constants.go b/go/util/constants.go index db1ca38e5..c407533a8 100644 --- a/go/util/constants.go +++ b/go/util/constants.go @@ -1,7 +1,5 @@ package util -import () - const ( VERSION = "0.64" ) diff --git a/go/util/file_util.go b/go/util/file_util.go index 412d98458..30e24f001 100644 --- a/go/util/file_util.go +++ b/go/util/file_util.go @@ -2,9 +2,10 @@ package util import ( "bufio" - "github.com/chrislusf/weed-fs/go/glog" "errors" "os" + + "github.com/chrislusf/weed-fs/go/glog" ) func TestFolderWritable(folder string) (err error) { diff --git a/go/util/net_timeout.go b/go/util/net_timeout.go index eb80822b5..f274e4802 100644 --- a/go/util/net_timeout.go +++ b/go/util/net_timeout.go @@ -1,9 +1,10 @@ package util import ( - "github.com/chrislusf/weed-fs/go/stats" "net" "time" + + "github.com/chrislusf/weed-fs/go/stats" ) // Listener wraps a net.Listener, and gives a place to store the timeout diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index fec8472e5..8339913cd 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -2,9 +2,6 @@ package main import ( "bufio" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "math" @@ -16,6 +13,10 @@ import ( "strings" "sync" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) type BenchmarkOptions struct { diff --git a/go/weed/download.go b/go/weed/download.go index c30d17915..c782654f5 100644 --- a/go/weed/download.go +++ b/go/weed/download.go @@ -1,14 +1,15 @@ package main import ( - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "io/ioutil" "os" "path" "strings" + + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) var ( diff --git a/go/weed/export.go b/go/weed/export.go index 9f33d852f..c9cc0e3fe 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -4,14 +4,15 @@ import ( "archive/tar" "bytes" "fmt" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "os" "path" "strconv" "strings" "text/template" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/filer.go b/go/weed/filer.go index 7dbecb4d0..b3fc9c878 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -1,13 +1,14 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" "net/http" "os" "strconv" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" ) var ( diff --git a/go/weed/fix.go b/go/weed/fix.go index ad573875a..9c8026448 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -1,11 +1,12 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "os" "path" "strconv" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/master.go b/go/weed/master.go index f96ff4c08..f88964b6d 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -1,16 +1,17 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" - "github.com/gorilla/mux" "net/http" "os" "runtime" "strconv" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/gorilla/mux" ) func init() { diff --git a/go/weed/mount.go b/go/weed/mount.go index 0290e0f53..66e645387 100644 --- a/go/weed/mount.go +++ b/go/weed/mount.go @@ -1,7 +1,5 @@ package main -import () - type MountOptions struct { filer *string dir *string diff --git a/go/weed/mount_std.go b/go/weed/mount_std.go index e5fc0986c..808c6c563 100644 --- a/go/weed/mount_std.go +++ b/go/weed/mount_std.go @@ -3,15 +3,16 @@ package main import ( + "fmt" + "os" + "runtime" + "bazil.org/fuse" "bazil.org/fuse/fs" "github.com/chrislusf/weed-fs/go/filer" "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/storage" "github.com/chrislusf/weed-fs/go/util" - "fmt" - "os" - "runtime" ) func runMount(cmd *Command, args []string) bool { diff --git a/go/weed/server.go b/go/weed/server.go index 1d854d641..6042c22e7 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -1,10 +1,6 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" - "github.com/gorilla/mux" "net/http" "os" "runtime" @@ -13,6 +9,11 @@ import ( "strings" "sync" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/gorilla/mux" ) type ServerOptions struct { diff --git a/go/weed/shell.go b/go/weed/shell.go index c8043e0dd..f2c4990ea 100644 --- a/go/weed/shell.go +++ b/go/weed/shell.go @@ -2,9 +2,10 @@ package main import ( "bufio" - "github.com/chrislusf/weed-fs/go/glog" "fmt" "os" + + "github.com/chrislusf/weed-fs/go/glog" ) func init() { diff --git a/go/weed/signal_handling_notsupported.go b/go/weed/signal_handling_notsupported.go index ad4f37b0c..343cf7de2 100644 --- a/go/weed/signal_handling_notsupported.go +++ b/go/weed/signal_handling_notsupported.go @@ -2,7 +2,5 @@ package main -import () - func OnInterrupt(fn func()) { } diff --git a/go/weed/upload.go b/go/weed/upload.go index 4eae4d274..2d67c0bd9 100644 --- a/go/weed/upload.go +++ b/go/weed/upload.go @@ -1,11 +1,12 @@ package main import ( - "github.com/chrislusf/weed-fs/go/operation" "encoding/json" "fmt" "os" "path/filepath" + + "github.com/chrislusf/weed-fs/go/operation" ) var ( diff --git a/go/weed/version.go b/go/weed/version.go index 63441509e..8d3a6fed7 100644 --- a/go/weed/version.go +++ b/go/weed/version.go @@ -1,9 +1,10 @@ package main import ( - "github.com/chrislusf/weed-fs/go/util" "fmt" "runtime" + + "github.com/chrislusf/weed-fs/go/util" ) var cmdVersion = &Command{ diff --git a/go/weed/volume.go b/go/weed/volume.go index ce05fcdf3..212cb4b33 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -1,15 +1,16 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" "net/http" "os" "runtime" "strconv" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" ) func init() { diff --git a/go/weed/volume_test.go b/go/weed/volume_test.go index 764362a2b..ef00a8c7c 100644 --- a/go/weed/volume_test.go +++ b/go/weed/volume_test.go @@ -1,10 +1,11 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" "net/http" "testing" "time" + + "github.com/chrislusf/weed-fs/go/glog" ) func TestXYZ(t *testing.T) { diff --git a/go/weed/weed.go b/go/weed/weed.go index c1f5a72de..c304b7f35 100644 --- a/go/weed/weed.go +++ b/go/weed/weed.go @@ -1,7 +1,6 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" "flag" "fmt" "io" @@ -13,6 +12,8 @@ import ( "time" "unicode" "unicode/utf8" + + "github.com/chrislusf/weed-fs/go/glog" ) var IsDebug *bool diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 816107dc5..39b0830e3 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -2,11 +2,6 @@ package weed_server import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "fmt" "net" @@ -14,6 +9,12 @@ import ( "path/filepath" "strconv" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) var serverStats *stats.ServerStats diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 5ff0ed986..9d6f7c71b 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -1,10 +1,11 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/filer" - "github.com/chrislusf/weed-fs/go/glog" "net/http" "strconv" + + "github.com/chrislusf/weed-fs/go/filer" + "github.com/chrislusf/weed-fs/go/glog" ) type FilerServer struct { diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index e36e7c310..781d7c0fc 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -1,12 +1,8 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" - "github.com/syndtr/goleveldb/leveldb" "io" "io/ioutil" "math/rand" @@ -14,6 +10,11 @@ import ( "net/url" "strconv" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" + "github.com/syndtr/goleveldb/leveldb" ) func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/filer_server_handlers_admin.go b/go/weed/weed_server/filer_server_handlers_admin.go index ff52dff24..5ba12e0b8 100644 --- a/go/weed/weed_server/filer_server_handlers_admin.go +++ b/go/weed/weed_server/filer_server_handlers_admin.go @@ -1,8 +1,9 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" "net/http" + + "github.com/chrislusf/weed-fs/go/glog" ) /* diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go index 401f6cfdb..d000cb610 100644 --- a/go/weed/weed_server/master_server.go +++ b/go/weed/weed_server/master_server.go @@ -1,16 +1,17 @@ package weed_server import ( + "net/http" + "net/http/httputil" + "net/url" + "sync" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/sequence" "github.com/chrislusf/weed-fs/go/topology" "github.com/chrislusf/weed-fs/go/util" "github.com/goraft/raft" "github.com/gorilla/mux" - "net/http" - "net/http/httputil" - "net/url" - "sync" ) type MasterServer struct { diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index d7f1f4ce3..7a7a3b70d 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -1,12 +1,13 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" "net/http" "strconv" "strings" + + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" ) func (ms *MasterServer) lookupVolumeId(vids []string, collection string) (volumeLocations map[string]operation.LookupResult) { diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 1a2c6b8e0..d7124e567 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -1,18 +1,19 @@ package weed_server import ( - proto "code.google.com/p/goprotobuf/proto" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/topology" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "io/ioutil" "net/http" "strconv" "strings" + + proto "code.google.com/p/goprotobuf/proto" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/util" ) func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index e41867076..0b049ce27 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -2,19 +2,20 @@ package weed_server import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/topology" "encoding/json" "errors" "fmt" - "github.com/goraft/raft" - "github.com/gorilla/mux" "io/ioutil" "math/rand" "net/http" "net/url" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/topology" + "github.com/goraft/raft" + "github.com/gorilla/mux" ) type RaftServer struct { diff --git a/go/weed/weed_server/raft_server_handlers.go b/go/weed/weed_server/raft_server_handlers.go index 4d51c0767..b466d9afa 100644 --- a/go/weed/weed_server/raft_server_handlers.go +++ b/go/weed/weed_server/raft_server_handlers.go @@ -1,13 +1,14 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" "encoding/json" - "github.com/goraft/raft" "io/ioutil" "net/http" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/goraft/raft" ) // Handles incoming RAFT joins. diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index 2a9085f3b..0a65fd2f6 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -1,12 +1,13 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "math/rand" "net/http" "strconv" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) type VolumeServer struct { diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index ce14f6a87..83f614941 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -1,12 +1,6 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/images" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/topology" "io" "mime" "mime/multipart" @@ -14,6 +8,13 @@ import ( "strconv" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/images" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/topology" ) var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go index 1118c8017..caf4c3be8 100644 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ b/go/weed/weed_server/volume_server_handlers_admin.go @@ -1,11 +1,12 @@ package weed_server import ( + "net/http" + "path/filepath" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/stats" "github.com/chrislusf/weed-fs/go/util" - "net/http" - "path/filepath" ) func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/volume_server_handlers_vacuum.go b/go/weed/weed_server/volume_server_handlers_vacuum.go index b0600d799..f115e3b8b 100644 --- a/go/weed/weed_server/volume_server_handlers_vacuum.go +++ b/go/weed/weed_server/volume_server_handlers_vacuum.go @@ -1,8 +1,9 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" "net/http" + + "github.com/chrislusf/weed-fs/go/glog" ) func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) { From 7ce628bf09ed949f1686a581d782cab3664511fd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 27 Oct 2014 01:09:45 -0700 Subject: [PATCH 41/58] Clean raft configurations if "peers" option is set. --- go/weed/weed_server/raft_server.go | 46 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index 0b049ce27..12148db49 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -9,6 +9,8 @@ import ( "math/rand" "net/http" "net/url" + "os" + "path" "strings" "time" @@ -46,6 +48,13 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin transporter := raft.NewHTTPTransporter("/cluster", 0) transporter.Transport.MaxIdleConnsPerHost = 1024 + // Clear old cluster configurations if peers are set + if len(s.peers) > 0 { + os.RemoveAll(path.Join(s.dataDir, "conf")) + os.RemoveAll(path.Join(s.dataDir, "log")) + os.RemoveAll(path.Join(s.dataDir, "snapshot")) + } + s.raftServer, err = raft.NewServer(s.httpAddr, s.dataDir, transporter, nil, topo, "") if err != nil { glog.V(0).Infoln(err) @@ -53,35 +62,30 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin } transporter.Install(s.raftServer, s) s.raftServer.SetHeartbeatInterval(1 * time.Second) - s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 1150 * time.Millisecond) + s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 3450 * time.Millisecond) s.raftServer.Start() s.router.HandleFunc("/cluster/join", s.joinHandler).Methods("POST") s.router.HandleFunc("/cluster/status", s.statusHandler).Methods("GET") - // Join to leader if specified. if len(s.peers) > 0 { - if !s.raftServer.IsLogEmpty() { - glog.V(0).Infoln("Starting cluster with existing logs.") - } else { - glog.V(0).Infoln("Joining cluster:", strings.Join(s.peers, ",")) - time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) - firstJoinError := s.Join(s.peers) - if firstJoinError != nil { - glog.V(0).Infoln("No existing server found. Starting as leader in the new cluster.") - _, err := s.raftServer.Do(&raft.DefaultJoinCommand{ - Name: s.raftServer.Name(), - ConnectionString: "http://" + s.httpAddr, - }) - if err != nil { - glog.V(0).Infoln(err) - return nil - } + // Join to leader if specified. + glog.V(0).Infoln("Joining cluster:", strings.Join(s.peers, ",")) + time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) + firstJoinError := s.Join(s.peers) + if firstJoinError != nil { + glog.V(0).Infoln("No existing server found. Starting as leader in the new cluster.") + _, err := s.raftServer.Do(&raft.DefaultJoinCommand{ + Name: s.raftServer.Name(), + ConnectionString: "http://" + s.httpAddr, + }) + if err != nil { + glog.V(0).Infoln(err) + return nil } } - - // Initialize the server by joining itself. } else if s.raftServer.IsLogEmpty() { + // Initialize the server by joining itself. glog.V(0).Infoln("Initializing new cluster") _, err := s.raftServer.Do(&raft.DefaultJoinCommand{ @@ -95,7 +99,7 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin } } else { - glog.V(0).Infoln("Recovered from log") + glog.V(0).Infoln("Old conf,log,snapshot should have been removed.") } return s From b0e8f8ca2a63c7be9fe0c32b68e15dc47001a672 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 27 Oct 2014 01:10:33 -0700 Subject: [PATCH 42/58] Upgraded to 0.97 goclipse. --- .project | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.project b/.project index f5551728c..4dee05c73 100644 --- a/.project +++ b/.project @@ -5,11 +5,6 @@ - - com.googlecode.goclipse.goBuilder - - - goclipse.goNature From cd1b70caf7969a5789f285cdb0c23f9b690626ab Mon Sep 17 00:00:00 2001 From: Yanyi Wu Date: Sat, 1 Nov 2014 01:48:19 +0800 Subject: [PATCH 43/58] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1243727a2..06d81ebf0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM cydev/go -RUN go get code.google.com/p/weed-fs/go/weed +RUN go get github.com/chrislusf/weed-fs/go/weed EXPOSE 8080 EXPOSE 9333 VOLUME /data From 8af475300295adf9aadc6f57216112db1e6765cd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 28 Nov 2014 16:34:03 -0800 Subject: [PATCH 44/58] Write request id to first 8 bytes of a file, instead of whole file, for better write performance. --- go/util/constants.go | 2 +- go/weed/benchmark.go | 26 ++++++++++++++++---------- go/weed/server.go | 10 +++++----- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/go/util/constants.go b/go/util/constants.go index c407533a8..61f05898f 100644 --- a/go/util/constants.go +++ b/go/util/constants.go @@ -1,5 +1,5 @@ package util const ( - VERSION = "0.64" + VERSION = "0.66" ) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index 8339913cd..e5cf831c6 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -31,6 +31,7 @@ type BenchmarkOptions struct { sequentialRead *bool collection *string cpuprofile *string + maxCpu *int vid2server map[string]string //cache for vid locations } @@ -51,7 +52,8 @@ func init() { b.read = cmdBenchmark.Flag.Bool("read", true, "enable read") b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file") b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection") - b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "write cpu profile to file") + b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file") + b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") b.vid2server = make(map[string]string) } @@ -59,25 +61,25 @@ var cmdBenchmark = &Command{ UsageLine: "benchmark -server=localhost:9333 -c=10 -n=100000", Short: "benchmark on writing millions of files and read out", Long: `benchmark on an empty weed file system. - + Two tests during benchmark: 1) write lots of small files to the system 2) read the files out - + The file content is mostly zero, but no compression is done. - + You can choose to only benchmark read or write. During write, the list of uploaded file ids is stored in "-list" specified file. You can also use your own list of file ids to run read test. - + Write speed and read speed will be collected. The numbers are used to get a sense of the system. Usually your network or the hard drive is the real bottleneck. - + Another thing to watch is whether the volumes are evenly distributed to each volume server. Because the 7 more benchmark volumes are randomly distributed to servers with free slots, it's highly possible some servers have uneven amount of - benchmark volumes. To remedy this, you can use this to grow the benchmark volumes + benchmark volumes. To remedy this, you can use this to grow the benchmark volumes before starting the benchmark command: http://localhost:9333/vol/grow?collection=benchmark&count=5 @@ -100,6 +102,10 @@ func init() { func runbenchmark(cmd *Command, args []string) bool { fmt.Printf("This is Seaweed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) + if *b.maxCpu < 1 { + *b.maxCpu = runtime.NumCPU() + } + runtime.GOMAXPROCS(*b.maxCpu) if *b.cpuprofile != "" { f, err := os.Create(*b.cpuprofile) if err != nil { @@ -497,9 +503,9 @@ func (l *FakeReader) Read(p []byte) (n int, err error) { } else { n = len(p) } - for i := 0; i < n-8; i += 8 { - for s := uint(0); s < 8; s++ { - p[i] = byte(l.id >> (s * 8)) + if n >= 8 { + for i := 0; i < 8; i++ { + p[i] = byte(l.id >> uint(i*8)) } } l.size -= int64(n) diff --git a/go/weed/server.go b/go/weed/server.go index 6042c22e7..38ccdc137 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -32,17 +32,17 @@ func init() { var cmdServer = &Command{ UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name", Short: "start a server, including volume server, and automatically elect a master server", - Long: `start both a volume server to provide storage spaces + Long: `start both a volume server to provide storage spaces and a master server to provide volume=>location mapping service and sequence number of file ids - + This is provided as a convenient way to start both volume server and master server. The servers are exactly the same as starting them separately. So other volume servers can use this embedded master server also. - + Optionally, one filer server can be started. Logically, filer servers should not be in a cluster. They run with meta data on disk, not shared. So each filer server is different. - + `, } @@ -73,7 +73,7 @@ var ( ) func init() { - serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "write cpu profile to file") + serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file") filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server") filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection") filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") From 9d03cd20ba518e2aad6db6abe8f51ee9445eaacd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 28 Nov 2014 16:55:40 -0800 Subject: [PATCH 45/58] De-support go 1.2 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a6f8b8815..79cec6d8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: go go: - - 1.2 - 1.3 - release - tip From ca67ed69a1e80bd1166105fb8a08d8c5b4f9c83f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 28 Nov 2014 17:02:10 -0800 Subject: [PATCH 46/58] Change name to Seaweed. --- docs/benchmarks.rst | 19 +++++++++++++++++-- docs/directories.rst | 2 +- docs/failover.rst | 2 +- docs/gettingstarted.rst | 6 +++--- go/weed/fix.go | 2 +- 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst index e01e6e4ba..d0b71428f 100644 --- a/docs/benchmarks.rst +++ b/docs/benchmarks.rst @@ -1,7 +1,7 @@ Benchmarks ====================== -Do we really need the benchmark? People always use benchmark to compare systems. But benchmarks are misleading. The resources, e.g., CPU, disk, memory, network, all matter a lot. And with Weed File System, single node vs multiple nodes, benchmarking on one machine vs several multiple machines, all matter a lot. +Do we really need the benchmark? People always use benchmark to compare systems. But benchmarks are misleading. The resources, e.g., CPU, disk, memory, network, all matter a lot. And with Seaweed File System, single node vs multiple nodes, benchmarking on one machine vs several multiple machines, all matter a lot. Here is the steps on how to run benchmark if you really need some numbers. @@ -38,7 +38,22 @@ Many options are options are configurable. Please check the help content: Common Problems ############################### -The most common problem is "too many open files" error. This is because the test itself starts too many network connections on one single machine. In my local macbook, if I ran "random read" following writing right away, the error happens always. I have to run "weed benchmark -write=false" to run the reading test only. Also, changing the concurrency level to "-c=16" would also help. +The most common +I start weed servers in one console for simplicity. Better run servers on different consoles. + +For more realistic tests, please start them on different machines. + +.. code-block:: bash + + # prepare directories + mkdir 3 4 5 + # start 3 servers + ./weed server -dir=./3 -master.port=9333 -volume.port=8083 & + ./weed volume -dir=./4 -port=8084 & + ./weed volume -dir=./5 -port=8085 & + ./weed benchmark -server=localhost:9333 + + problem is "too many open files" error. This is because the test itself starts too many network connections on one single machine. In my local macbook, if I ran "random read" following writing right away, the error happens always. I have to run "weed benchmark -write=false" to run the reading test only. Also, changing the concurrency level to "-c=16" would also help. My own unscientific single machine results ################################################### diff --git a/docs/directories.rst b/docs/directories.rst index 787c7a2ab..aa14da4f7 100644 --- a/docs/directories.rst +++ b/docs/directories.rst @@ -1,7 +1,7 @@ Directories and files =========================== -When talking about file systems, many people would assume directories, list files under a directory, etc. These are expected if we want to hook up Weed File System with linux by FUSE, or with Hadoop, etc. +When talking about file systems, many people would assume directories, list files under a directory, etc. These are expected if we want to hook up Seaweed File System with linux by FUSE, or with Hadoop, etc. Sample usage ##################### diff --git a/docs/failover.rst b/docs/failover.rst index f244918d8..506a33e3b 100644 --- a/docs/failover.rst +++ b/docs/failover.rst @@ -7,7 +7,7 @@ Introduction Some user will ask for no single point of failure. Although google runs its file system with a single master for years, no SPOF seems becoming a criteria for architects to pick solutions. -Luckily, it's not too difficult to enable Weed File System with failover master servers. +Luckily, it's not too difficult to enable Seaweed File System with failover master servers. Cheat Sheet: Startup multiple servers ######################################## diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index 19eb130a7..ad28319fe 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -1,6 +1,6 @@ Getting started =================================== -Installing Weed-Fs +Installing Seaweed-FS ################################### Download a proper version from `Seaweed-FS download page `_. @@ -57,7 +57,7 @@ Actually, forget about previous commands. You can setup one master server and on # use "weed server -h" to find out more ./weed server -master.port=9333 -volume.port=8080 -dir="./data" -Testing Weed-Fs +Testing Seaweed-FS ################################### With the master and volume server up, now what? Let's pump in a lot of files into the system! @@ -77,7 +77,7 @@ Then, you can simply check "du -m -s /some/big/folder" to see the actual disk us Now you can use your tools to hit weed-fs as hard as you can. -Using Weed-Fs in docker +Using Seaweed-FS in docker #################################### You can use image "cydev/weed" or build your own with `dockerfile `_ in the root of repo. diff --git a/go/weed/fix.go b/go/weed/fix.go index 9c8026448..e66075ed2 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -17,7 +17,7 @@ func init() { var cmdFix = &Command{ UsageLine: "fix -dir=/tmp -volumeId=234", Short: "run weed tool fix on index file if corrupted", - Long: `Fix runs the WeedFS fix command to re-create the index .idx file. + Long: `Fix runs the SeeweedFS fix command to re-create the index .idx file. `, } From 6c5a3d3dbfc3e801c962bd650cfd4455483cc63c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 30 Nov 2014 21:55:53 -0800 Subject: [PATCH 47/58] Increase performance by reusing []byte, reducing GC. --- go/weed/benchmark.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index e5cf831c6..f88d0d6a9 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -33,10 +33,12 @@ type BenchmarkOptions struct { cpuprofile *string maxCpu *int vid2server map[string]string //cache for vid locations + } var ( - b BenchmarkOptions + b BenchmarkOptions + sharedBytes []byte ) func init() { @@ -55,6 +57,7 @@ func init() { b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file") b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") b.vid2server = make(map[string]string) + sharedBytes = make([]byte, 1024) } var cmdBenchmark = &Command{ @@ -512,6 +515,23 @@ func (l *FakeReader) Read(p []byte) (n int, err error) { return } +func (l *FakeReader) WriteTo(w io.Writer) (n int64, err error) { + size := int(l.size) + bufferSize := len(sharedBytes) + for size > 0 { + tempBuffer := sharedBytes + if size < bufferSize { + tempBuffer = sharedBytes[0:size] + } + count, e := w.Write(tempBuffer) + if e != nil { + return int64(size), e + } + size -= count + } + return l.size, nil +} + func Readln(r *bufio.Reader) ([]byte, error) { var ( isPrefix bool = true @@ -524,3 +544,4 @@ func Readln(r *bufio.Reader) ([]byte, error) { } return ln, err } + From 74b76a2c41694a74920230afe5fa2559eeb885b3 Mon Sep 17 00:00:00 2001 From: Shaoshan Liu Date: Wed, 3 Dec 2014 21:04:02 -0800 Subject: [PATCH 48/58] add instructions for users who are not familiar with golang --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index ab07683dd..b22d07569 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,31 @@ Seaweed File System go get github.com/chrislusf/weed-fs/go/weed ``` +## Installation guide for users who are not familiar with golang + +step 1: install go on your machine and setup the environment by following the instructions from the following link: + +https://golang.org/doc/install + +make sure you set up your $GOPATH + + +step 2: also you may need to install Mercurial by following the instructions below + +http://mercurial.selenic.com/downloads + + +step 3: download, compile, and install the project by executing the following command + +go get github.com/chrislusf/weed-fs/go/weed + +once this is down, you should see the executable "weed" under $GOPATH/bin + +step 4: after you modify your code locally, you could start a local build by calling "go build" under $GOPATH/src/github.com/chrislusf/weed-fs/go/weed + + + + ## Reference For pre-compiled releases, From ed7b00bf0207749cb555d7d77bc35ddf5849ec37 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Wed, 3 Dec 2014 21:37:00 -0800 Subject: [PATCH 49/58] Update README.md --- README.md | 67 ++++++++++++++++++++++--------------------------------- 1 file changed, 27 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index b22d07569..95d2a5ade 100644 --- a/README.md +++ b/README.md @@ -5,41 +5,6 @@ Seaweed File System [![GoDoc](https://godoc.org/github.com/chrislusf/weed-fs/go?status.svg)](https://godoc.org/github.com/chrislusf/weed-fs/go) [![RTD](https://readthedocs.org/projects/weed-fs/badge/?version=latest)](http://weed-fs.readthedocs.org/en/latest/) -## Usage - -``` -go get github.com/chrislusf/weed-fs/go/weed -``` - -## Installation guide for users who are not familiar with golang - -step 1: install go on your machine and setup the environment by following the instructions from the following link: - -https://golang.org/doc/install - -make sure you set up your $GOPATH - - -step 2: also you may need to install Mercurial by following the instructions below - -http://mercurial.selenic.com/downloads - - -step 3: download, compile, and install the project by executing the following command - -go get github.com/chrislusf/weed-fs/go/weed - -once this is down, you should see the executable "weed" under $GOPATH/bin - -step 4: after you modify your code locally, you could start a local build by calling "go build" under $GOPATH/src/github.com/chrislusf/weed-fs/go/weed - - - - -## Reference - -For pre-compiled releases, - https://bintray.com/chrislusf/Weed-FS/seaweed ## Introduction @@ -271,12 +236,34 @@ More tools and documentation, on how to maintain and scale the system. For examp This is a super exciting project! And I need helpers! -## Contributions ## -To make contributions easier, I have mirrored a repo in github.com -``` - https://github.com/chrislusf/weed-fs.git -``` +## Installation guide for users who are not familiar with golang + +step 1: install go on your machine and setup the environment by following the instructions from the following link: + +https://golang.org/doc/install + +make sure you set up your $GOPATH + + +step 2: also you may need to install Mercurial by following the instructions below + +http://mercurial.selenic.com/downloads + + +step 3: download, compile, and install the project by executing the following command + +go get github.com/chrislusf/weed-fs/go/weed + +once this is done, you should see the executable "weed" under $GOPATH/bin + +step 4: after you modify your code locally, you could start a local build by calling "go install" under $GOPATH/src/github.com/chrislusf/weed-fs/go/weed + +## Reference + +For pre-compiled releases, + https://bintray.com/chrislusf/Weed-FS/seaweed + ## Disk Related topics ## ### Hard Drive Performance ### From 89fd1e4b6e586412f0692ac479b566a323296595 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Thu, 4 Dec 2014 18:30:44 -0800 Subject: [PATCH 50/58] Add more thread safe counters. Tighten thread synchronization. --- go/weed/benchmark.go | 79 +++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index f88d0d6a9..47b400fcd 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -132,12 +132,12 @@ func runbenchmark(cmd *Command, args []string) bool { func bench_write() { fileIdLineChan := make(chan string) finishChan := make(chan bool) - writeStats = newStats() + writeStats = newStats(*b.concurrency) idChan := make(chan int) - wait.Add(*b.concurrency) go writeFileIds(*b.idListFile, fileIdLineChan, finishChan) for i := 0; i < *b.concurrency; i++ { - go writeFiles(idChan, fileIdLineChan, writeStats) + wait.Add(1) + go writeFiles(idChan, fileIdLineChan, &writeStats.localStats[i]) } writeStats.start = time.Now() writeStats.total = *b.numberOfFiles @@ -150,7 +150,6 @@ func bench_write() { writeStats.end = time.Now() wait.Add(1) finishChan <- true - finishChan <- true close(finishChan) wait.Wait() writeStats.printStats() @@ -159,14 +158,14 @@ func bench_write() { func bench_read() { fileIdLineChan := make(chan string) finishChan := make(chan bool) - readStats = newStats() - wait.Add(*b.concurrency) + readStats = newStats(*b.concurrency) go readFileIds(*b.idListFile, fileIdLineChan) readStats.start = time.Now() readStats.total = *b.numberOfFiles go readStats.checkProgress("Randomly Reading Benchmark", finishChan) for i := 0; i < *b.concurrency; i++ { - go readFiles(fileIdLineChan, readStats) + wait.Add(1) + go readFiles(fileIdLineChan, &readStats.localStats[i]) } wait.Wait() finishChan <- true @@ -180,12 +179,13 @@ type delayedFile struct { fp *operation.FilePart } -func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) { +func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { + defer wait.Done() delayedDeleteChan := make(chan *delayedFile, 100) var waitForDeletions sync.WaitGroup for i := 0; i < 7; i++ { + waitForDeletions.Add(1) go func() { - waitForDeletions.Add(1) for df := range delayedDeleteChan { if df == nil { break @@ -228,6 +228,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) { s.transferred += fileSize } else { s.failed++ + fmt.Printf("Failed to write with error:%v\n", err) } writeStats.addSample(time.Now().Sub(start)) <-serverLimitChan[fp.Server] @@ -244,10 +245,10 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) { } close(delayedDeleteChan) waitForDeletions.Wait() - wait.Done() } -func readFiles(fileIdLineChan chan string, s *stats) { +func readFiles(fileIdLineChan chan string, s *stat) { + defer wait.Done() serverLimitChan := make(map[string]chan bool) masterLimitChan := make(chan bool, 1) for { @@ -288,7 +289,7 @@ func readFiles(fileIdLineChan chan string, s *stats) { readStats.addSample(time.Now().Sub(start)) } else { s.failed++ - println("!!!! Failed to read from ", url, " !!!!!") + fmt.Printf("Failed to read %s error:%v\n", url, err) } <-serverLimitChan[server] } else { @@ -299,7 +300,6 @@ func readFiles(fileIdLineChan chan string, s *stats) { break } } - wait.Done() } func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan bool) { @@ -363,20 +363,28 @@ const ( // An efficient statics collecting and rendering type stats struct { - data []int - overflow []int + data []int + overflow []int + localStats []stat + start time.Time + end time.Time + total int +} +type stat struct { completed int failed int total int transferred int64 - start time.Time - end time.Time } var percentages = []int{50, 66, 75, 80, 90, 95, 98, 99, 100} -func newStats() *stats { - return &stats{data: make([]int, benchResolution), overflow: make([]int, 0)} +func newStats(n int) *stats { + return &stats{ + data: make([]int, benchResolution), + overflow: make([]int, 0), + localStats: make([]stat, n), + } } func (s *stats) addSample(d time.Duration) { @@ -399,26 +407,38 @@ func (s *stats) checkProgress(testName string, finishChan chan bool) { case <-finishChan: return case t := <-ticker: - completed, transferred, taken := s.completed-lastCompleted, s.transferred-lastTransferred, t.Sub(lastTime) + completed, transferred, taken, total := 0, int64(0), t.Sub(lastTime), s.total + for _, localStat := range s.localStats { + completed += localStat.completed + transferred += localStat.transferred + total += localStat.total + } fmt.Printf("Completed %d of %d requests, %3.1f%% %3.1f/s %3.1fMB/s\n", - s.completed, s.total, float64(s.completed)*100/float64(s.total), - float64(completed)*float64(int64(time.Second))/float64(int64(taken)), - float64(transferred)*float64(int64(time.Second))/float64(int64(taken))/float64(1024*1024), + completed, total, float64(completed)*100/float64(total), + float64(completed-lastCompleted)*float64(int64(time.Second))/float64(int64(taken)), + float64(transferred-lastTransferred)*float64(int64(time.Second))/float64(int64(taken))/float64(1024*1024), ) - lastCompleted, lastTransferred, lastTime = s.completed, s.transferred, t + lastCompleted, lastTransferred, lastTime = completed, transferred, t } } } func (s *stats) printStats() { + completed, failed, transferred, total := 0, 0, int64(0), s.total + for _, localStat := range s.localStats { + completed += localStat.completed + failed += localStat.failed + transferred += localStat.transferred + total += localStat.total + } timeTaken := float64(int64(s.end.Sub(s.start))) / 1000000000 fmt.Printf("\nConcurrency Level: %d\n", *b.concurrency) fmt.Printf("Time taken for tests: %.3f seconds\n", timeTaken) - fmt.Printf("Complete requests: %d\n", s.completed) - fmt.Printf("Failed requests: %d\n", s.failed) - fmt.Printf("Total transferred: %d bytes\n", s.transferred) - fmt.Printf("Requests per second: %.2f [#/sec]\n", float64(s.completed)/timeTaken) - fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", float64(s.transferred)/1024/timeTaken) + fmt.Printf("Complete requests: %d\n", completed) + fmt.Printf("Failed requests: %d\n", failed) + fmt.Printf("Total transferred: %d bytes\n", transferred) + fmt.Printf("Requests per second: %.2f [#/sec]\n", float64(completed)/timeTaken) + fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", float64(transferred)/1024/timeTaken) n, sum := 0, 0 min, max := 10000000, 0 for i := 0; i < len(s.data); i++ { @@ -544,4 +564,3 @@ func Readln(r *bufio.Reader) ([]byte, error) { } return ln, err } - From 7a6394378cfd3a09bb273211e9bad1ec19e01e84 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Thu, 4 Dec 2014 21:22:09 -0800 Subject: [PATCH 51/58] Remove a volume server concurrent connection limit. --- go/weed/benchmark.go | 150 ++++++++++++++++++------------------------- 1 file changed, 62 insertions(+), 88 deletions(-) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index 47b400fcd..f4f0b1874 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -93,16 +93,11 @@ var cmdBenchmark = &Command{ } var ( - wait sync.WaitGroup - writeStats *stats - readStats *stats - serverLimitChan map[string]chan bool + wait sync.WaitGroup + writeStats *stats + readStats *stats ) -func init() { - serverLimitChan = make(map[string]chan bool) -} - func runbenchmark(cmd *Command, args []string) bool { fmt.Printf("This is Seaweed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) if *b.maxCpu < 1 { @@ -148,10 +143,11 @@ func bench_write() { close(idChan) wait.Wait() writeStats.end = time.Now() - wait.Add(1) + wait.Add(2) + finishChan <- true finishChan <- true - close(finishChan) wait.Wait() + close(finishChan) writeStats.printStats() } @@ -168,7 +164,9 @@ func bench_read() { go readFiles(fileIdLineChan, &readStats.localStats[i]) } wait.Wait() + wait.Add(1) finishChan <- true + wait.Wait() close(finishChan) readStats.end = time.Now() readStats.printStats() @@ -186,61 +184,46 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { for i := 0; i < 7; i++ { waitForDeletions.Add(1) go func() { + defer waitForDeletions.Done() for df := range delayedDeleteChan { - if df == nil { - break - } if df.enterTime.After(time.Now()) { time.Sleep(df.enterTime.Sub(time.Now())) } - fp := df.fp - serverLimitChan[fp.Server] <- true - if e := util.Delete("http://" + fp.Server + "/" + fp.Fid); e == nil { + if e := util.Delete("http://" + df.fp.Server + "/" + df.fp.Fid); e == nil { s.completed++ } else { s.failed++ } - <-serverLimitChan[fp.Server] } - waitForDeletions.Done() }() } - for { - if id, ok := <-idChan; ok { - start := time.Now() - fileSize := int64(*b.fileSize + rand.Intn(64)) - fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize} - if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil { - fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection - if _, ok := serverLimitChan[fp.Server]; !ok { - serverLimitChan[fp.Server] = make(chan bool, 7) - } - serverLimitChan[fp.Server] <- true - if _, err := fp.Upload(0, *b.server); err == nil { - if rand.Intn(100) < *b.deletePercentage { - s.total++ - delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp} - } else { - fileIdLineChan <- fp.Fid - } - s.completed++ - s.transferred += fileSize + for id := range idChan { + start := time.Now() + fileSize := int64(*b.fileSize + rand.Intn(64)) + fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize} + if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil { + fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection + if _, err := fp.Upload(0, *b.server); err == nil { + if rand.Intn(100) < *b.deletePercentage { + s.total++ + delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp} } else { - s.failed++ - fmt.Printf("Failed to write with error:%v\n", err) - } - writeStats.addSample(time.Now().Sub(start)) - <-serverLimitChan[fp.Server] - if *cmdBenchmark.IsDebug { - fmt.Printf("writing %d file %s\n", id, fp.Fid) + fileIdLineChan <- fp.Fid } + s.completed++ + s.transferred += fileSize } else { s.failed++ - println("writing file error:", err.Error()) + fmt.Printf("Failed to write with error:%v\n", err) + } + writeStats.addSample(time.Now().Sub(start)) + if *cmdBenchmark.IsDebug { + fmt.Printf("writing %d file %s\n", id, fp.Fid) } } else { - break + s.failed++ + println("writing file error:", err.Error()) } } close(delayedDeleteChan) @@ -249,55 +232,45 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { func readFiles(fileIdLineChan chan string, s *stat) { defer wait.Done() - serverLimitChan := make(map[string]chan bool) masterLimitChan := make(chan bool, 1) - for { - if fid, ok := <-fileIdLineChan; ok { - if len(fid) == 0 { - continue - } - if fid[0] == '#' { - continue - } - if *cmdBenchmark.IsDebug { - fmt.Printf("reading file %s\n", fid) - } - parts := strings.SplitN(fid, ",", 2) - vid := parts[0] - start := time.Now() - if server, ok := b.vid2server[vid]; !ok { - masterLimitChan <- true - if _, now_ok := b.vid2server[vid]; !now_ok { - if ret, err := operation.Lookup(*b.server, vid); err == nil { - if len(ret.Locations) > 0 { - server = ret.Locations[0].PublicUrl - b.vid2server[vid] = server - } + for fid := range fileIdLineChan { + if len(fid) == 0 { + continue + } + if fid[0] == '#' { + continue + } + if *cmdBenchmark.IsDebug { + fmt.Printf("reading file %s\n", fid) + } + parts := strings.SplitN(fid, ",", 2) + vid := parts[0] + start := time.Now() + if server, ok := b.vid2server[vid]; !ok { + masterLimitChan <- true + if _, now_ok := b.vid2server[vid]; !now_ok { + if ret, err := operation.Lookup(*b.server, vid); err == nil { + if len(ret.Locations) > 0 { + server = ret.Locations[0].PublicUrl + b.vid2server[vid] = server } } - <-masterLimitChan } - if server, ok := b.vid2server[vid]; ok { - if _, ok := serverLimitChan[server]; !ok { - serverLimitChan[server] = make(chan bool, 7) - } - serverLimitChan[server] <- true - url := "http://" + server + "/" + fid - if bytesRead, err := util.Get(url); err == nil { - s.completed++ - s.transferred += int64(len(bytesRead)) - readStats.addSample(time.Now().Sub(start)) - } else { - s.failed++ - fmt.Printf("Failed to read %s error:%v\n", url, err) - } - <-serverLimitChan[server] + <-masterLimitChan + } + if server, ok := b.vid2server[vid]; ok { + url := "http://" + server + "/" + fid + if bytesRead, err := util.Get(url); err == nil { + s.completed++ + s.transferred += int64(len(bytesRead)) + readStats.addSample(time.Now().Sub(start)) } else { s.failed++ - println("!!!! volume id ", vid, " location not found!!!!!") + fmt.Printf("Failed to read %s error:%v\n", url, err) } } else { - break + s.failed++ + println("!!!! volume id ", vid, " location not found!!!!!") } } } @@ -405,6 +378,7 @@ func (s *stats) checkProgress(testName string, finishChan chan bool) { for { select { case <-finishChan: + wait.Done() return case t := <-ticker: completed, transferred, taken, total := 0, int64(0), t.Sub(lastTime), s.total From 482e3fb97325ae073f19b8293bb25d210beed721 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 5 Dec 2014 12:00:13 -0800 Subject: [PATCH 52/58] Improve Benchmark tool's performance. --- go/util/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/util/constants.go b/go/util/constants.go index 61f05898f..d677ba44f 100644 --- a/go/util/constants.go +++ b/go/util/constants.go @@ -1,5 +1,5 @@ package util const ( - VERSION = "0.66" + VERSION = "0.67" ) From ba972694c730429889c696bd9853a38843f64f65 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Mon, 8 Dec 2014 20:27:26 -0800 Subject: [PATCH 53/58] Add filer option to redirect instead of proxying to volume server on file GET requests. --- go/weed/filer.go | 8 +++++-- go/weed/server.go | 6 +++++- go/weed/weed_server/filer_server.go | 22 +++++++++++++------- go/weed/weed_server/filer_server_handlers.go | 12 +++++++++-- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/go/weed/filer.go b/go/weed/filer.go index b3fc9c878..5ae3a3496 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -21,6 +21,7 @@ type FilerOptions struct { collection *string defaultReplicaPlacement *string dir *string + redirectOnRead *bool } func init() { @@ -29,7 +30,8 @@ func init() { f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection") f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port") f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data") - f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "Default replication type if not specified.") + f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified") + f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") } var cmdFiler = &Command{ @@ -60,7 +62,9 @@ func runFiler(cmd *Command, args []string) bool { } r := http.NewServeMux() - _, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection) + _, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection, + *f.defaultReplicaPlacement, *f.redirectOnRead, + ) if nfs_err != nil { glog.Fatalf(nfs_err.Error()) } diff --git a/go/weed/server.go b/go/weed/server.go index 38ccdc137..22222d699 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -79,6 +79,8 @@ func init() { filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified") filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") + filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") + } func runServer(cmd *Command, args []string) bool { @@ -150,7 +152,9 @@ func runServer(cmd *Command, args []string) bool { if *isStartingFiler { go func() { r := http.NewServeMux() - _, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection) + _, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, + *filerOptions.defaultReplicaPlacement, *filerOptions.redirectOnRead, + ) if nfs_err != nil { glog.Fatalf(nfs_err.Error()) } diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 9d6f7c71b..0bda58d06 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -9,17 +9,23 @@ import ( ) type FilerServer struct { - port string - master string - collection string - filer filer.Filer + port string + master string + collection string + defaultReplication string + redirectOnRead bool + filer filer.Filer } -func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string) (fs *FilerServer, err error) { +func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string, + replication string, redirectOnRead bool, +) (fs *FilerServer, err error) { fs = &FilerServer{ - master: master, - collection: collection, - port: ":" + strconv.Itoa(port), + master: master, + collection: collection, + defaultReplication: replication, + redirectOnRead: redirectOnRead, + port: ":" + strconv.Itoa(port), } if fs.filer, err = filer.NewFilerEmbedded(master, dir); err != nil { diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 781d7c0fc..9d8481ac9 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -81,7 +81,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, return } urlLocation := lookup.Locations[rand.Intn(len(lookup.Locations))].PublicUrl - u, _ := url.Parse("http://" + urlLocation + "/" + fileId) + urlString := "http://" + urlLocation + "/" + fileId + if fs.redirectOnRead { + + } + u, _ := url.Parse(urlString) request := &http.Request{ Method: r.Method, URL: u, @@ -110,7 +114,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() - assignResult, ae := operation.Assign(fs.master, 1, query.Get("replication"), fs.collection, query.Get("ttl")) + replication := query.Get("replication") + if replication == "" { + replication = fs.defaultReplication + } + assignResult, ae := operation.Assign(fs.master, 1, replication, fs.collection, query.Get("ttl")) if ae != nil { glog.V(0).Infoln("failing to assign a file id", ae.Error()) writeJsonError(w, r, ae) From 52180f386b044af7a6daba3e33ff04b5e9da25ba Mon Sep 17 00:00:00 2001 From: chrislusf Date: Mon, 8 Dec 2014 20:29:25 -0800 Subject: [PATCH 54/58] Add read-write lock to guard topology changes on new collections and ttls. --- go/topology/collection.go | 21 ++++++++++--------- go/topology/topology.go | 29 +++++++++++++------------- go/topology/topology_map.go | 7 ++++--- go/topology/topology_vacuum.go | 14 +++++++------ go/util/concurrent_read_map.go | 37 ++++++++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 34 deletions(-) create mode 100644 go/util/concurrent_read_map.go diff --git a/go/topology/collection.go b/go/topology/collection.go index 506f43fbf..4b47ae88a 100644 --- a/go/topology/collection.go +++ b/go/topology/collection.go @@ -2,17 +2,18 @@ package topology import ( "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) type Collection struct { Name string volumeSizeLimit uint64 - storageType2VolumeLayout map[string]*VolumeLayout + storageType2VolumeLayout *util.ConcurrentReadMap } func NewCollection(name string, volumeSizeLimit uint64) *Collection { c := &Collection{Name: name, volumeSizeLimit: volumeSizeLimit} - c.storageType2VolumeLayout = make(map[string]*VolumeLayout) + c.storageType2VolumeLayout = util.NewConcurrentReadMap() return c } @@ -21,16 +22,16 @@ func (c *Collection) GetOrCreateVolumeLayout(rp *storage.ReplicaPlacement, ttl * if ttl != nil { keyString += ttl.String() } - if c.storageType2VolumeLayout[keyString] == nil { - c.storageType2VolumeLayout[keyString] = NewVolumeLayout(rp, ttl, c.volumeSizeLimit) - } - return c.storageType2VolumeLayout[keyString] + vl := c.storageType2VolumeLayout.Get(keyString, func() interface{} { + return NewVolumeLayout(rp, ttl, c.volumeSizeLimit) + }) + return vl.(*VolumeLayout) } func (c *Collection) Lookup(vid storage.VolumeId) []*DataNode { - for _, vl := range c.storageType2VolumeLayout { + for _, vl := range c.storageType2VolumeLayout.Items { if vl != nil { - if list := vl.Lookup(vid); list != nil { + if list := vl.(*VolumeLayout).Lookup(vid); list != nil { return list } } @@ -39,9 +40,9 @@ func (c *Collection) Lookup(vid storage.VolumeId) []*DataNode { } func (c *Collection) ListVolumeServers() (nodes []*DataNode) { - for _, vl := range c.storageType2VolumeLayout { + for _, vl := range c.storageType2VolumeLayout.Items { if vl != nil { - if list := vl.ListVolumeServers(); list != nil { + if list := vl.(*VolumeLayout).ListVolumeServers(); list != nil { nodes = append(nodes, list...) } } diff --git a/go/topology/topology.go b/go/topology/topology.go index eb64d336c..c2073ed2f 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -9,13 +9,14 @@ import ( "github.com/chrislusf/weed-fs/go/operation" "github.com/chrislusf/weed-fs/go/sequence" "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" "github.com/goraft/raft" ) type Topology struct { NodeImpl - collectionMap map[string]*Collection + collectionMap *util.ConcurrentReadMap pulse int64 @@ -38,7 +39,7 @@ func NewTopology(id string, confFile string, seq sequence.Sequencer, volumeSizeL t.nodeType = "Topology" t.NodeImpl.value = t t.children = make(map[NodeId]Node) - t.collectionMap = make(map[string]*Collection) + t.collectionMap = util.NewConcurrentReadMap() t.pulse = int64(pulse) t.volumeSizeLimit = volumeSizeLimit @@ -90,14 +91,14 @@ func (t *Topology) loadConfiguration(configurationFile string) error { func (t *Topology) Lookup(collection string, vid storage.VolumeId) []*DataNode { //maybe an issue if lots of collections? if collection == "" { - for _, c := range t.collectionMap { - if list := c.Lookup(vid); list != nil { + for _, c := range t.collectionMap.Items { + if list := c.(*Collection).Lookup(vid); list != nil { return list } } } else { - if c, ok := t.collectionMap[collection]; ok { - return c.Lookup(vid) + if c, ok := t.collectionMap.Items[collection]; ok { + return c.(*Collection).Lookup(vid) } } return nil @@ -125,20 +126,18 @@ func (t *Topology) PickForWrite(count int, option *VolumeGrowOption) (string, in } func (t *Topology) GetVolumeLayout(collectionName string, rp *storage.ReplicaPlacement, ttl *storage.TTL) *VolumeLayout { - _, ok := t.collectionMap[collectionName] - if !ok { - t.collectionMap[collectionName] = NewCollection(collectionName, t.volumeSizeLimit) - } - return t.collectionMap[collectionName].GetOrCreateVolumeLayout(rp, ttl) + return t.collectionMap.Get(collectionName, func() interface{} { + return NewCollection(collectionName, t.volumeSizeLimit) + }).(*Collection).GetOrCreateVolumeLayout(rp, ttl) } -func (t *Topology) GetCollection(collectionName string) (collection *Collection, ok bool) { - collection, ok = t.collectionMap[collectionName] - return +func (t *Topology) GetCollection(collectionName string) (*Collection, bool) { + c, hasCollection := t.collectionMap.Items[collectionName] + return c.(*Collection), hasCollection } func (t *Topology) DeleteCollection(collectionName string) { - delete(t.collectionMap, collectionName) + delete(t.collectionMap.Items, collectionName) } func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { diff --git a/go/topology/topology_map.go b/go/topology/topology_map.go index af95c6536..6a1423ca8 100644 --- a/go/topology/topology_map.go +++ b/go/topology/topology_map.go @@ -11,10 +11,11 @@ func (t *Topology) ToMap() interface{} { } m["DataCenters"] = dcs var layouts []interface{} - for _, c := range t.collectionMap { - for _, layout := range c.storageType2VolumeLayout { + for _, col := range t.collectionMap.Items { + c := col.(*Collection) + for _, layout := range c.storageType2VolumeLayout.Items { if layout != nil { - tmp := layout.ToMap() + tmp := layout.(*VolumeLayout).ToMap() tmp["collection"] = c.Name layouts = append(layouts, tmp) } diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index 97a76026d..d6fa2213e 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -80,13 +80,15 @@ func batchVacuumVolumeCommit(vl *VolumeLayout, vid storage.VolumeId, locationlis return isCommitSuccess } func (t *Topology) Vacuum(garbageThreshold string) int { - for _, c := range t.collectionMap { - for _, vl := range c.storageType2VolumeLayout { + for _, col := range t.collectionMap.Items { + c := col.(*Collection) + for _, vl := range c.storageType2VolumeLayout.Items { if vl != nil { - for vid, locationlist := range vl.vid2location { - if batchVacuumVolumeCheck(vl, vid, locationlist, garbageThreshold) { - if batchVacuumVolumeCompact(vl, vid, locationlist) { - batchVacuumVolumeCommit(vl, vid, locationlist) + volumeLayout := vl.(*VolumeLayout) + for vid, locationlist := range volumeLayout.vid2location { + if batchVacuumVolumeCheck(volumeLayout, vid, locationlist, garbageThreshold) { + if batchVacuumVolumeCompact(volumeLayout, vid, locationlist) { + batchVacuumVolumeCommit(volumeLayout, vid, locationlist) } } } diff --git a/go/util/concurrent_read_map.go b/go/util/concurrent_read_map.go new file mode 100644 index 000000000..d16fdbcaf --- /dev/null +++ b/go/util/concurrent_read_map.go @@ -0,0 +1,37 @@ +package util + +import "sync" + +// A mostly for read map, which can thread-safely +// initialize the map entries. +type ConcurrentReadMap struct { + rmutex sync.RWMutex + mutex sync.Mutex + Items map[string]interface{} +} + +func NewConcurrentReadMap() *ConcurrentReadMap { + return &ConcurrentReadMap{Items: make(map[string]interface{})} +} + +func (m *ConcurrentReadMap) initMapEntry(key string, newEntry func() interface{}) (value interface{}) { + m.mutex.Lock() + defer m.mutex.Unlock() + if value, ok := m.Items[key]; ok { + return value + } + value = newEntry() + m.Items[key] = value + return value +} + +func (m *ConcurrentReadMap) Get(key string, newEntry func() interface{}) interface{} { + m.rmutex.RLock() + if value, ok := m.Items[key]; ok { + m.rmutex.RUnlock() + return value + } else { + m.rmutex.RUnlock() + return m.initMapEntry(key, newEntry) + } +} From e431d4121e8da8d7fc243b29b780c2cd535a4210 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Mon, 8 Dec 2014 20:34:27 -0800 Subject: [PATCH 55/58] Add optional http redirect for filer GET requests. --- go/weed/weed_server/filer_server_handlers.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 9d8481ac9..6f22912a7 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -83,7 +83,8 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, urlLocation := lookup.Locations[rand.Intn(len(lookup.Locations))].PublicUrl urlString := "http://" + urlLocation + "/" + fileId if fs.redirectOnRead { - + http.Redirect(w, r, urlString, http.StatusFound) + return } u, _ := url.Parse(urlString) request := &http.Request{ From c0dfdf439264964cf702b8a872c59f7a79fbd05b Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Sun, 14 Dec 2014 00:20:21 -0800 Subject: [PATCH 56/58] switch to my forked repo --- go/filer/client_operations.go | 2 +- go/filer/directory_in_map.go | 2 +- go/filer/filer_embedded.go | 2 +- go/filer/files_in_leveldb.go | 2 +- go/operation/assign_file_id.go | 4 ++-- go/operation/delete_content.go | 2 +- go/operation/list_masters.go | 4 ++-- go/operation/lookup.go | 2 +- go/operation/submit.go | 2 +- go/operation/upload_content.go | 2 +- go/storage/cdb_map.go | 2 +- go/storage/cdb_map_test.go | 2 +- go/storage/compact_map_perf_test.go | 4 ++-- go/storage/compress.go | 2 +- go/storage/crc.go | 2 +- go/storage/file_id.go | 4 ++-- go/storage/needle.go | 6 +++--- go/storage/needle_map.go | 4 ++-- go/storage/needle_read_write.go | 4 ++-- go/storage/store.go | 6 +++--- go/storage/store_vacuum.go | 2 +- go/storage/volume.go | 2 +- go/storage/volume_info.go | 2 +- go/storage/volume_super_block.go | 2 +- go/storage/volume_vacuum.go | 2 +- go/tools/read_index.go | 2 +- go/topology/allocate_volume.go | 4 ++-- go/topology/cluster_commands.go | 4 ++-- go/topology/collection.go | 4 ++-- go/topology/data_node.go | 4 ++-- go/topology/node.go | 4 ++-- go/topology/store_replicate.go | 8 ++++---- go/topology/topology.go | 10 +++++----- go/topology/topology_event_handling.go | 4 ++-- go/topology/topology_vacuum.go | 6 +++--- go/topology/volume_growth.go | 4 ++-- go/topology/volume_growth_test.go | 4 ++-- go/topology/volume_layout.go | 4 ++-- go/util/config.go | 2 +- go/util/file_util.go | 2 +- go/util/net_timeout.go | 2 +- go/weed/benchmark.go | 6 +++--- go/weed/compact.go | 4 ++-- go/weed/download.go | 4 ++-- go/weed/export.go | 4 ++-- go/weed/filer.go | 7 ++++--- go/weed/fix.go | 4 ++-- go/weed/master.go | 6 +++--- go/weed/mount_std.go | 8 ++++---- go/weed/server.go | 6 +++--- go/weed/shell.go | 2 +- go/weed/upload.go | 2 +- go/weed/version.go | 2 +- go/weed/volume.go | 6 +++--- go/weed/volume_test.go | 2 +- go/weed/weed.go | 2 +- go/weed/weed_server/common.go | 10 +++++----- go/weed/weed_server/filer_server.go | 4 ++-- go/weed/weed_server/filer_server_handlers.go | 6 +++--- go/weed/weed_server/filer_server_handlers_admin.go | 2 +- go/weed/weed_server/master_server.go | 8 ++++---- go/weed/weed_server/master_server_handlers.go | 6 +++--- go/weed/weed_server/master_server_handlers_admin.go | 10 +++++----- go/weed/weed_server/raft_server.go | 4 ++-- go/weed/weed_server/raft_server_handlers.go | 4 ++-- go/weed/weed_server/volume_server.go | 4 ++-- go/weed/weed_server/volume_server_handlers.go | 12 ++++++------ go/weed/weed_server/volume_server_handlers_admin.go | 6 +++--- go/weed/weed_server/volume_server_handlers_vacuum.go | 2 +- 69 files changed, 141 insertions(+), 140 deletions(-) diff --git a/go/filer/client_operations.go b/go/filer/client_operations.go index b38368735..50f95bab1 100644 --- a/go/filer/client_operations.go +++ b/go/filer/client_operations.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" "net/url" ) diff --git a/go/filer/directory_in_map.go b/go/filer/directory_in_map.go index ee601066c..d3fe7080f 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/directory_in_map.go @@ -10,7 +10,7 @@ import ( "strings" "sync" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" ) var writeLock sync.Mutex //serialize changes to dir.log diff --git a/go/filer/filer_embedded.go b/go/filer/filer_embedded.go index 0b9b4e668..58684a4d4 100644 --- a/go/filer/filer_embedded.go +++ b/go/filer/filer_embedded.go @@ -6,7 +6,7 @@ import ( "path/filepath" "strings" - "github.com/chrislusf/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/operation" ) type FilerEmbedded struct { diff --git a/go/filer/files_in_leveldb.go b/go/filer/files_in_leveldb.go index 781bb0e5f..ca2c4e796 100644 --- a/go/filer/files_in_leveldb.go +++ b/go/filer/files_in_leveldb.go @@ -3,7 +3,7 @@ package filer import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" ) diff --git a/go/operation/assign_file_id.go b/go/operation/assign_file_id.go index 672bfa99c..4fe571d59 100644 --- a/go/operation/assign_file_id.go +++ b/go/operation/assign_file_id.go @@ -6,8 +6,8 @@ import ( "net/url" "strconv" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" ) type AssignResult struct { diff --git a/go/operation/delete_content.go b/go/operation/delete_content.go index 416a852b3..06787dabe 100644 --- a/go/operation/delete_content.go +++ b/go/operation/delete_content.go @@ -7,7 +7,7 @@ import ( "strings" "sync" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" ) type DeleteResult struct { diff --git a/go/operation/list_masters.go b/go/operation/list_masters.go index 7ada94243..542a3cb38 100644 --- a/go/operation/list_masters.go +++ b/go/operation/list_masters.go @@ -3,8 +3,8 @@ package operation import ( "encoding/json" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" ) type ClusterStatusResult struct { diff --git a/go/operation/lookup.go b/go/operation/lookup.go index e6b6658da..2fd238e7a 100644 --- a/go/operation/lookup.go +++ b/go/operation/lookup.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" ) type Location struct { diff --git a/go/operation/submit.go b/go/operation/submit.go index 62db46617..c4f64f5a2 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) type FilePart struct { diff --git a/go/operation/upload_content.go b/go/operation/upload_content.go index 480d76dca..297d78a04 100644 --- a/go/operation/upload_content.go +++ b/go/operation/upload_content.go @@ -14,7 +14,7 @@ import ( "path/filepath" "strings" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) type UploadResult struct { diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go index fbb59e9c0..824672cae 100644 --- a/go/storage/cdb_map.go +++ b/go/storage/cdb_map.go @@ -7,7 +7,7 @@ import ( "os" "path/filepath" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" "github.com/tgulacsi/go-cdb" ) diff --git a/go/storage/cdb_map_test.go b/go/storage/cdb_map_test.go index ed690f44f..3b13d5b76 100644 --- a/go/storage/cdb_map_test.go +++ b/go/storage/cdb_map_test.go @@ -6,7 +6,7 @@ import ( "runtime" "testing" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) var testIndexFilename string = "../../test/sample.idx" diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index f74684225..bebed48ba 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -5,8 +5,8 @@ import ( "os" "testing" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" ) func TestMemoryUsage(t *testing.T) { diff --git a/go/storage/compress.go b/go/storage/compress.go index 5efc9e1f1..88a9c9c15 100644 --- a/go/storage/compress.go +++ b/go/storage/compress.go @@ -7,7 +7,7 @@ import ( "io/ioutil" "strings" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) /* diff --git a/go/storage/crc.go b/go/storage/crc.go index af25b9e53..01452dd4a 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -4,7 +4,7 @@ import ( "fmt" "hash/crc32" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" ) var table = crc32.MakeTable(crc32.Castagnoli) diff --git a/go/storage/file_id.go b/go/storage/file_id.go index f6e36a98c..55ccdd110 100644 --- a/go/storage/file_id.go +++ b/go/storage/file_id.go @@ -5,8 +5,8 @@ import ( "errors" "strings" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" ) type FileId struct { diff --git a/go/storage/needle.go b/go/storage/needle.go index aa3206920..13ee34bd6 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -11,9 +11,9 @@ import ( "strings" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/images" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/images" + "github.com/mcqueenorama/weed-fs/go/util" ) const ( diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index 504ca1552..c7150d4d9 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -5,8 +5,8 @@ import ( "io" "os" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" ) type NeedleMapper interface { diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index 663b5abbd..d4529f92d 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -6,8 +6,8 @@ import ( "io" "os" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" ) const ( diff --git a/go/storage/store.go b/go/storage/store.go index 80d8a30b8..97dc16f70 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -10,9 +10,9 @@ import ( "strings" proto "code.google.com/p/goprotobuf/proto" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/util" ) const ( diff --git a/go/storage/store_vacuum.go b/go/storage/store_vacuum.go index 209e3b4b3..0db083055 100644 --- a/go/storage/store_vacuum.go +++ b/go/storage/store_vacuum.go @@ -4,7 +4,7 @@ import ( "fmt" "strconv" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) { diff --git a/go/storage/volume.go b/go/storage/volume.go index 5b0a83605..540da3140 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -10,7 +10,7 @@ import ( "sync" "time" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) type Volume struct { diff --git a/go/storage/volume_info.go b/go/storage/volume_info.go index 6410c1784..503967c97 100644 --- a/go/storage/volume_info.go +++ b/go/storage/volume_info.go @@ -1,7 +1,7 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/operation" ) type VolumeInfo struct { diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index 57e0deea9..32b799f26 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -4,7 +4,7 @@ import ( "fmt" "os" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) const ( diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go index 7e026a61d..28028241a 100644 --- a/go/storage/volume_vacuum.go +++ b/go/storage/volume_vacuum.go @@ -5,7 +5,7 @@ import ( "os" "time" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) func (v *Volume) garbageLevel() float64 { diff --git a/go/tools/read_index.go b/go/tools/read_index.go index 1104dc348..ed18167b2 100644 --- a/go/tools/read_index.go +++ b/go/tools/read_index.go @@ -6,7 +6,7 @@ import ( "log" "os" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/storage" ) var ( diff --git a/go/topology/allocate_volume.go b/go/topology/allocate_volume.go index a791b4c1c..38843dd54 100644 --- a/go/topology/allocate_volume.go +++ b/go/topology/allocate_volume.go @@ -5,8 +5,8 @@ import ( "errors" "net/url" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" ) type AllocateVolumeResult struct { diff --git a/go/topology/cluster_commands.go b/go/topology/cluster_commands.go index cafc52c76..c88de9e0b 100644 --- a/go/topology/cluster_commands.go +++ b/go/topology/cluster_commands.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" "github.com/goraft/raft" ) diff --git a/go/topology/collection.go b/go/topology/collection.go index 4b47ae88a..cded90650 100644 --- a/go/topology/collection.go +++ b/go/topology/collection.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" ) type Collection struct { diff --git a/go/topology/data_node.go b/go/topology/data_node.go index 109bd037f..c12396e87 100644 --- a/go/topology/data_node.go +++ b/go/topology/data_node.go @@ -3,8 +3,8 @@ package topology import ( "strconv" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) type DataNode struct { diff --git a/go/topology/node.go b/go/topology/node.go index 10955fa72..9c8e7700b 100644 --- a/go/topology/node.go +++ b/go/topology/node.go @@ -5,8 +5,8 @@ import ( "math/rand" "strings" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) type NodeId string diff --git a/go/topology/store_replicate.go b/go/topology/store_replicate.go index 0c52f9d30..b949b33c3 100644 --- a/go/topology/store_replicate.go +++ b/go/topology/store_replicate.go @@ -5,10 +5,10 @@ import ( "net/http" "strconv" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" ) func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.VolumeId, needle *storage.Needle, r *http.Request) (size uint32, errorStatus string) { diff --git a/go/topology/topology.go b/go/topology/topology.go index c2073ed2f..6c9cbe3da 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -5,11 +5,11 @@ import ( "io/ioutil" "math/rand" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/sequence" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/sequence" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" "github.com/goraft/raft" ) diff --git a/go/topology/topology_event_handling.go b/go/topology/topology_event_handling.go index 7e36568b6..49e67c8f5 100644 --- a/go/topology/topology_event_handling.go +++ b/go/topology/topology_event_handling.go @@ -4,8 +4,8 @@ import ( "math/rand" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) func (t *Topology) StartRefreshWritableVolumes(garbageThreshold string) { diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index d6fa2213e..9e532876f 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -6,9 +6,9 @@ import ( "net/url" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" ) func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList, garbageThreshold string) bool { diff --git a/go/topology/volume_growth.go b/go/topology/volume_growth.go index b1f241990..02466d539 100644 --- a/go/topology/volume_growth.go +++ b/go/topology/volume_growth.go @@ -5,8 +5,8 @@ import ( "math/rand" "sync" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) /* diff --git a/go/topology/volume_growth_test.go b/go/topology/volume_growth_test.go index 267b36042..333ab5a3a 100644 --- a/go/topology/volume_growth_test.go +++ b/go/topology/volume_growth_test.go @@ -5,8 +5,8 @@ import ( "fmt" "testing" - "github.com/chrislusf/weed-fs/go/sequence" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/sequence" + "github.com/mcqueenorama/weed-fs/go/storage" ) var topologyLayout = ` diff --git a/go/topology/volume_layout.go b/go/topology/volume_layout.go index de72bf895..8c6f80954 100644 --- a/go/topology/volume_layout.go +++ b/go/topology/volume_layout.go @@ -5,8 +5,8 @@ import ( "math/rand" "sync" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) // mapping from volume to its locations, inverted from server to volume diff --git a/go/util/config.go b/go/util/config.go index 4cf1d7c64..071d731c9 100644 --- a/go/util/config.go +++ b/go/util/config.go @@ -13,7 +13,7 @@ import ( "encoding/json" "os" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) type Config struct { diff --git a/go/util/file_util.go b/go/util/file_util.go index 30e24f001..176aaac96 100644 --- a/go/util/file_util.go +++ b/go/util/file_util.go @@ -5,7 +5,7 @@ import ( "errors" "os" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) func TestFolderWritable(folder string) (err error) { diff --git a/go/util/net_timeout.go b/go/util/net_timeout.go index f274e4802..4e54b6798 100644 --- a/go/util/net_timeout.go +++ b/go/util/net_timeout.go @@ -4,7 +4,7 @@ import ( "net" "time" - "github.com/chrislusf/weed-fs/go/stats" + "github.com/mcqueenorama/weed-fs/go/stats" ) // Listener wraps a net.Listener, and gives a place to store the timeout diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index f4f0b1874..2d507ebb6 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -14,9 +14,9 @@ import ( "sync" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/util" ) type BenchmarkOptions struct { diff --git a/go/weed/compact.go b/go/weed/compact.go index 71c4ea90f..1ecb85cf9 100644 --- a/go/weed/compact.go +++ b/go/weed/compact.go @@ -1,8 +1,8 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) func init() { diff --git a/go/weed/download.go b/go/weed/download.go index c782654f5..82c0fd45e 100644 --- a/go/weed/download.go +++ b/go/weed/download.go @@ -8,8 +8,8 @@ import ( "path" "strings" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/util" ) var ( diff --git a/go/weed/export.go b/go/weed/export.go index c9cc0e3fe..472e63aea 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -11,8 +11,8 @@ import ( "text/template" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) func init() { diff --git a/go/weed/filer.go b/go/weed/filer.go index 5ae3a3496..173ccddc7 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -6,9 +6,9 @@ import ( "strconv" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/weed/weed_server" ) var ( @@ -57,6 +57,7 @@ var cmdFiler = &Command{ } func runFiler(cmd *Command, args []string) bool { + if err := util.TestFolderWritable(*f.dir); err != nil { glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err) } diff --git a/go/weed/fix.go b/go/weed/fix.go index e66075ed2..0f5f0b7f3 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -5,8 +5,8 @@ import ( "path" "strconv" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) func init() { diff --git a/go/weed/master.go b/go/weed/master.go index f88964b6d..885e6e207 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -8,9 +8,9 @@ import ( "strings" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/weed/weed_server" "github.com/gorilla/mux" ) diff --git a/go/weed/mount_std.go b/go/weed/mount_std.go index 808c6c563..e930240b1 100644 --- a/go/weed/mount_std.go +++ b/go/weed/mount_std.go @@ -9,10 +9,10 @@ import ( "bazil.org/fuse" "bazil.org/fuse/fs" - "github.com/chrislusf/weed-fs/go/filer" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/filer" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" ) func runMount(cmd *Command, args []string) bool { diff --git a/go/weed/server.go b/go/weed/server.go index 22222d699..13e9797ba 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -10,9 +10,9 @@ import ( "sync" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/weed/weed_server" "github.com/gorilla/mux" ) diff --git a/go/weed/shell.go b/go/weed/shell.go index f2c4990ea..04cfb4730 100644 --- a/go/weed/shell.go +++ b/go/weed/shell.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) func init() { diff --git a/go/weed/upload.go b/go/weed/upload.go index 2d67c0bd9..7623f7d9b 100644 --- a/go/weed/upload.go +++ b/go/weed/upload.go @@ -6,7 +6,7 @@ import ( "os" "path/filepath" - "github.com/chrislusf/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/operation" ) var ( diff --git a/go/weed/version.go b/go/weed/version.go index 8d3a6fed7..13f7ea23a 100644 --- a/go/weed/version.go +++ b/go/weed/version.go @@ -4,7 +4,7 @@ import ( "fmt" "runtime" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" ) var cmdVersion = &Command{ diff --git a/go/weed/volume.go b/go/weed/volume.go index 212cb4b33..71a296702 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -8,9 +8,9 @@ import ( "strings" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/weed/weed_server" ) func init() { diff --git a/go/weed/volume_test.go b/go/weed/volume_test.go index ef00a8c7c..b936ba4f2 100644 --- a/go/weed/volume_test.go +++ b/go/weed/volume_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) func TestXYZ(t *testing.T) { diff --git a/go/weed/weed.go b/go/weed/weed.go index c304b7f35..0620cdf17 100644 --- a/go/weed/weed.go +++ b/go/weed/weed.go @@ -13,7 +13,7 @@ import ( "unicode" "unicode/utf8" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) var IsDebug *bool diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 39b0830e3..55f5e69a4 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -10,11 +10,11 @@ import ( "strconv" "strings" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/stats" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" ) var serverStats *stats.ServerStats diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 0bda58d06..2630a3398 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -4,8 +4,8 @@ import ( "net/http" "strconv" - "github.com/chrislusf/weed-fs/go/filer" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/filer" + "github.com/mcqueenorama/weed-fs/go/glog" ) type FilerServer struct { diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 6f22912a7..e5a7a860e 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -11,9 +11,9 @@ import ( "strconv" "strings" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/util" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/go/weed/weed_server/filer_server_handlers_admin.go b/go/weed/weed_server/filer_server_handlers_admin.go index 5ba12e0b8..f63701a6b 100644 --- a/go/weed/weed_server/filer_server_handlers_admin.go +++ b/go/weed/weed_server/filer_server_handlers_admin.go @@ -3,7 +3,7 @@ package weed_server import ( "net/http" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) /* diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go index d000cb610..d96354eb9 100644 --- a/go/weed/weed_server/master_server.go +++ b/go/weed/weed_server/master_server.go @@ -6,10 +6,10 @@ import ( "net/url" "sync" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/sequence" - "github.com/chrislusf/weed-fs/go/topology" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/sequence" + "github.com/mcqueenorama/weed-fs/go/topology" + "github.com/mcqueenorama/weed-fs/go/util" "github.com/goraft/raft" "github.com/gorilla/mux" ) diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 7a7a3b70d..25839669f 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -5,9 +5,9 @@ import ( "strconv" "strings" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/stats" + "github.com/mcqueenorama/weed-fs/go/storage" ) func (ms *MasterServer) lookupVolumeId(vids []string, collection string) (volumeLocations map[string]operation.LookupResult) { diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index d7124e567..6ad83784b 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -9,11 +9,11 @@ import ( "strings" proto "code.google.com/p/goprotobuf/proto" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/topology" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/topology" + "github.com/mcqueenorama/weed-fs/go/util" ) func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index b9aaef2b0..fb85daecf 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -14,8 +14,8 @@ import ( "strings" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/topology" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/topology" "github.com/goraft/raft" "github.com/gorilla/mux" ) diff --git a/go/weed/weed_server/raft_server_handlers.go b/go/weed/weed_server/raft_server_handlers.go index b466d9afa..340a34e7b 100644 --- a/go/weed/weed_server/raft_server_handlers.go +++ b/go/weed/weed_server/raft_server_handlers.go @@ -6,8 +6,8 @@ import ( "net/http" "strings" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" "github.com/goraft/raft" ) diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index 0a65fd2f6..3e40a790a 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -6,8 +6,8 @@ import ( "strconv" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) type VolumeServer struct { diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index 83f614941..e14bcea9e 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -9,12 +9,12 @@ import ( "strings" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/images" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/topology" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/images" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/stats" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/topology" ) var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go index caf4c3be8..0d3f6ac5b 100644 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ b/go/weed/weed_server/volume_server_handlers_admin.go @@ -4,9 +4,9 @@ import ( "net/http" "path/filepath" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/stats" + "github.com/mcqueenorama/weed-fs/go/util" ) func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/volume_server_handlers_vacuum.go b/go/weed/weed_server/volume_server_handlers_vacuum.go index f115e3b8b..da7172ae9 100644 --- a/go/weed/weed_server/volume_server_handlers_vacuum.go +++ b/go/weed/weed_server/volume_server_handlers_vacuum.go @@ -3,7 +3,7 @@ package weed_server import ( "net/http" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) { From 57ec736941a554919cfed2bd2ac2bb694095a7d2 Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Sun, 14 Dec 2014 00:33:16 -0800 Subject: [PATCH 57/58] make it turn on the filer when filer.redirectOnRead is set --- go/weed/server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go/weed/server.go b/go/weed/server.go index 13e9797ba..6a5a0a935 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -101,6 +101,10 @@ func runServer(cmd *Command, args []string) bool { } } + if *filerOptions.redirectOnRead { + *isStartingFiler = true + } + *filerOptions.master = *serverPublicIp + ":" + strconv.Itoa(*masterPort) if *filerOptions.defaultReplicaPlacement == "" { From d56c748fa889a119139c3a40528a18e8ce4123ad Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Sun, 14 Dec 2014 00:35:26 -0800 Subject: [PATCH 58/58] switch it back to chris's repo --- go/filer/client_operations.go | 2 +- go/filer/directory_in_map.go | 2 +- go/filer/filer_embedded.go | 2 +- go/filer/files_in_leveldb.go | 2 +- go/operation/assign_file_id.go | 4 ++-- go/operation/delete_content.go | 2 +- go/operation/list_masters.go | 4 ++-- go/operation/lookup.go | 2 +- go/operation/submit.go | 2 +- go/operation/upload_content.go | 2 +- go/storage/cdb_map.go | 2 +- go/storage/cdb_map_test.go | 2 +- go/storage/compact_map_perf_test.go | 4 ++-- go/storage/compress.go | 2 +- go/storage/crc.go | 2 +- go/storage/file_id.go | 4 ++-- go/storage/needle.go | 6 +++--- go/storage/needle_map.go | 4 ++-- go/storage/needle_read_write.go | 4 ++-- go/storage/store.go | 6 +++--- go/storage/store_vacuum.go | 2 +- go/storage/volume.go | 2 +- go/storage/volume_info.go | 2 +- go/storage/volume_super_block.go | 2 +- go/storage/volume_vacuum.go | 2 +- go/tools/read_index.go | 2 +- go/topology/allocate_volume.go | 4 ++-- go/topology/cluster_commands.go | 4 ++-- go/topology/collection.go | 4 ++-- go/topology/data_node.go | 4 ++-- go/topology/node.go | 4 ++-- go/topology/store_replicate.go | 8 ++++---- go/topology/topology.go | 10 +++++----- go/topology/topology_event_handling.go | 4 ++-- go/topology/topology_vacuum.go | 6 +++--- go/topology/volume_growth.go | 4 ++-- go/topology/volume_growth_test.go | 4 ++-- go/topology/volume_layout.go | 4 ++-- go/util/config.go | 2 +- go/util/file_util.go | 2 +- go/util/net_timeout.go | 2 +- go/weed/benchmark.go | 6 +++--- go/weed/compact.go | 4 ++-- go/weed/download.go | 4 ++-- go/weed/export.go | 4 ++-- go/weed/filer.go | 6 +++--- go/weed/fix.go | 4 ++-- go/weed/master.go | 6 +++--- go/weed/mount_std.go | 8 ++++---- go/weed/server.go | 6 +++--- go/weed/shell.go | 2 +- go/weed/upload.go | 2 +- go/weed/version.go | 2 +- go/weed/volume.go | 6 +++--- go/weed/volume_test.go | 2 +- go/weed/weed.go | 2 +- go/weed/weed_server/common.go | 10 +++++----- go/weed/weed_server/filer_server.go | 4 ++-- go/weed/weed_server/filer_server_handlers.go | 6 +++--- go/weed/weed_server/filer_server_handlers_admin.go | 2 +- go/weed/weed_server/master_server.go | 8 ++++---- go/weed/weed_server/master_server_handlers.go | 6 +++--- go/weed/weed_server/master_server_handlers_admin.go | 10 +++++----- go/weed/weed_server/raft_server.go | 4 ++-- go/weed/weed_server/raft_server_handlers.go | 4 ++-- go/weed/weed_server/volume_server.go | 4 ++-- go/weed/weed_server/volume_server_handlers.go | 12 ++++++------ go/weed/weed_server/volume_server_handlers_admin.go | 6 +++--- go/weed/weed_server/volume_server_handlers_vacuum.go | 2 +- 69 files changed, 140 insertions(+), 140 deletions(-) diff --git a/go/filer/client_operations.go b/go/filer/client_operations.go index 50f95bab1..b38368735 100644 --- a/go/filer/client_operations.go +++ b/go/filer/client_operations.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" "net/url" ) diff --git a/go/filer/directory_in_map.go b/go/filer/directory_in_map.go index d3fe7080f..ee601066c 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/directory_in_map.go @@ -10,7 +10,7 @@ import ( "strings" "sync" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" ) var writeLock sync.Mutex //serialize changes to dir.log diff --git a/go/filer/filer_embedded.go b/go/filer/filer_embedded.go index 58684a4d4..0b9b4e668 100644 --- a/go/filer/filer_embedded.go +++ b/go/filer/filer_embedded.go @@ -6,7 +6,7 @@ import ( "path/filepath" "strings" - "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/operation" ) type FilerEmbedded struct { diff --git a/go/filer/files_in_leveldb.go b/go/filer/files_in_leveldb.go index ca2c4e796..781bb0e5f 100644 --- a/go/filer/files_in_leveldb.go +++ b/go/filer/files_in_leveldb.go @@ -3,7 +3,7 @@ package filer import ( "bytes" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" ) diff --git a/go/operation/assign_file_id.go b/go/operation/assign_file_id.go index 4fe571d59..672bfa99c 100644 --- a/go/operation/assign_file_id.go +++ b/go/operation/assign_file_id.go @@ -6,8 +6,8 @@ import ( "net/url" "strconv" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type AssignResult struct { diff --git a/go/operation/delete_content.go b/go/operation/delete_content.go index 06787dabe..416a852b3 100644 --- a/go/operation/delete_content.go +++ b/go/operation/delete_content.go @@ -7,7 +7,7 @@ import ( "strings" "sync" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" ) type DeleteResult struct { diff --git a/go/operation/list_masters.go b/go/operation/list_masters.go index 542a3cb38..7ada94243 100644 --- a/go/operation/list_masters.go +++ b/go/operation/list_masters.go @@ -3,8 +3,8 @@ package operation import ( "encoding/json" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type ClusterStatusResult struct { diff --git a/go/operation/lookup.go b/go/operation/lookup.go index 2fd238e7a..e6b6658da 100644 --- a/go/operation/lookup.go +++ b/go/operation/lookup.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" ) type Location struct { diff --git a/go/operation/submit.go b/go/operation/submit.go index c4f64f5a2..62db46617 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) type FilePart struct { diff --git a/go/operation/upload_content.go b/go/operation/upload_content.go index 297d78a04..480d76dca 100644 --- a/go/operation/upload_content.go +++ b/go/operation/upload_content.go @@ -14,7 +14,7 @@ import ( "path/filepath" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) type UploadResult struct { diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go index 824672cae..fbb59e9c0 100644 --- a/go/storage/cdb_map.go +++ b/go/storage/cdb_map.go @@ -7,7 +7,7 @@ import ( "os" "path/filepath" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" "github.com/tgulacsi/go-cdb" ) diff --git a/go/storage/cdb_map_test.go b/go/storage/cdb_map_test.go index 3b13d5b76..ed690f44f 100644 --- a/go/storage/cdb_map_test.go +++ b/go/storage/cdb_map_test.go @@ -6,7 +6,7 @@ import ( "runtime" "testing" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) var testIndexFilename string = "../../test/sample.idx" diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index bebed48ba..f74684225 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -5,8 +5,8 @@ import ( "os" "testing" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) func TestMemoryUsage(t *testing.T) { diff --git a/go/storage/compress.go b/go/storage/compress.go index 88a9c9c15..5efc9e1f1 100644 --- a/go/storage/compress.go +++ b/go/storage/compress.go @@ -7,7 +7,7 @@ import ( "io/ioutil" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) /* diff --git a/go/storage/crc.go b/go/storage/crc.go index 01452dd4a..af25b9e53 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -4,7 +4,7 @@ import ( "fmt" "hash/crc32" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" ) var table = crc32.MakeTable(crc32.Castagnoli) diff --git a/go/storage/file_id.go b/go/storage/file_id.go index 55ccdd110..f6e36a98c 100644 --- a/go/storage/file_id.go +++ b/go/storage/file_id.go @@ -5,8 +5,8 @@ import ( "errors" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type FileId struct { diff --git a/go/storage/needle.go b/go/storage/needle.go index 13ee34bd6..aa3206920 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -11,9 +11,9 @@ import ( "strings" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/images" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/images" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index c7150d4d9..504ca1552 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -5,8 +5,8 @@ import ( "io" "os" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type NeedleMapper interface { diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index d4529f92d..663b5abbd 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -6,8 +6,8 @@ import ( "io" "os" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/store.go b/go/storage/store.go index 97dc16f70..80d8a30b8 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -10,9 +10,9 @@ import ( "strings" proto "code.google.com/p/goprotobuf/proto" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/store_vacuum.go b/go/storage/store_vacuum.go index 0db083055..209e3b4b3 100644 --- a/go/storage/store_vacuum.go +++ b/go/storage/store_vacuum.go @@ -4,7 +4,7 @@ import ( "fmt" "strconv" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) { diff --git a/go/storage/volume.go b/go/storage/volume.go index 540da3140..5b0a83605 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -10,7 +10,7 @@ import ( "sync" "time" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) type Volume struct { diff --git a/go/storage/volume_info.go b/go/storage/volume_info.go index 503967c97..6410c1784 100644 --- a/go/storage/volume_info.go +++ b/go/storage/volume_info.go @@ -1,7 +1,7 @@ package storage import ( - "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/operation" ) type VolumeInfo struct { diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index 32b799f26..57e0deea9 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -4,7 +4,7 @@ import ( "fmt" "os" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) const ( diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go index 28028241a..7e026a61d 100644 --- a/go/storage/volume_vacuum.go +++ b/go/storage/volume_vacuum.go @@ -5,7 +5,7 @@ import ( "os" "time" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) func (v *Volume) garbageLevel() float64 { diff --git a/go/tools/read_index.go b/go/tools/read_index.go index ed18167b2..1104dc348 100644 --- a/go/tools/read_index.go +++ b/go/tools/read_index.go @@ -6,7 +6,7 @@ import ( "log" "os" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/storage" ) var ( diff --git a/go/topology/allocate_volume.go b/go/topology/allocate_volume.go index 38843dd54..a791b4c1c 100644 --- a/go/topology/allocate_volume.go +++ b/go/topology/allocate_volume.go @@ -5,8 +5,8 @@ import ( "errors" "net/url" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) type AllocateVolumeResult struct { diff --git a/go/topology/cluster_commands.go b/go/topology/cluster_commands.go index c88de9e0b..cafc52c76 100644 --- a/go/topology/cluster_commands.go +++ b/go/topology/cluster_commands.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" "github.com/goraft/raft" ) diff --git a/go/topology/collection.go b/go/topology/collection.go index cded90650..4b47ae88a 100644 --- a/go/topology/collection.go +++ b/go/topology/collection.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) type Collection struct { diff --git a/go/topology/data_node.go b/go/topology/data_node.go index c12396e87..109bd037f 100644 --- a/go/topology/data_node.go +++ b/go/topology/data_node.go @@ -3,8 +3,8 @@ package topology import ( "strconv" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) type DataNode struct { diff --git a/go/topology/node.go b/go/topology/node.go index 9c8e7700b..10955fa72 100644 --- a/go/topology/node.go +++ b/go/topology/node.go @@ -5,8 +5,8 @@ import ( "math/rand" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) type NodeId string diff --git a/go/topology/store_replicate.go b/go/topology/store_replicate.go index b949b33c3..0c52f9d30 100644 --- a/go/topology/store_replicate.go +++ b/go/topology/store_replicate.go @@ -5,10 +5,10 @@ import ( "net/http" "strconv" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.VolumeId, needle *storage.Needle, r *http.Request) (size uint32, errorStatus string) { diff --git a/go/topology/topology.go b/go/topology/topology.go index 6c9cbe3da..c2073ed2f 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -5,11 +5,11 @@ import ( "io/ioutil" "math/rand" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/sequence" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/sequence" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" "github.com/goraft/raft" ) diff --git a/go/topology/topology_event_handling.go b/go/topology/topology_event_handling.go index 49e67c8f5..7e36568b6 100644 --- a/go/topology/topology_event_handling.go +++ b/go/topology/topology_event_handling.go @@ -4,8 +4,8 @@ import ( "math/rand" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func (t *Topology) StartRefreshWritableVolumes(garbageThreshold string) { diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index 9e532876f..d6fa2213e 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -6,9 +6,9 @@ import ( "net/url" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList, garbageThreshold string) bool { diff --git a/go/topology/volume_growth.go b/go/topology/volume_growth.go index 02466d539..b1f241990 100644 --- a/go/topology/volume_growth.go +++ b/go/topology/volume_growth.go @@ -5,8 +5,8 @@ import ( "math/rand" "sync" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) /* diff --git a/go/topology/volume_growth_test.go b/go/topology/volume_growth_test.go index 333ab5a3a..267b36042 100644 --- a/go/topology/volume_growth_test.go +++ b/go/topology/volume_growth_test.go @@ -5,8 +5,8 @@ import ( "fmt" "testing" - "github.com/mcqueenorama/weed-fs/go/sequence" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/sequence" + "github.com/chrislusf/weed-fs/go/storage" ) var topologyLayout = ` diff --git a/go/topology/volume_layout.go b/go/topology/volume_layout.go index 8c6f80954..de72bf895 100644 --- a/go/topology/volume_layout.go +++ b/go/topology/volume_layout.go @@ -5,8 +5,8 @@ import ( "math/rand" "sync" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) // mapping from volume to its locations, inverted from server to volume diff --git a/go/util/config.go b/go/util/config.go index 071d731c9..4cf1d7c64 100644 --- a/go/util/config.go +++ b/go/util/config.go @@ -13,7 +13,7 @@ import ( "encoding/json" "os" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) type Config struct { diff --git a/go/util/file_util.go b/go/util/file_util.go index 176aaac96..30e24f001 100644 --- a/go/util/file_util.go +++ b/go/util/file_util.go @@ -5,7 +5,7 @@ import ( "errors" "os" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) func TestFolderWritable(folder string) (err error) { diff --git a/go/util/net_timeout.go b/go/util/net_timeout.go index 4e54b6798..f274e4802 100644 --- a/go/util/net_timeout.go +++ b/go/util/net_timeout.go @@ -4,7 +4,7 @@ import ( "net" "time" - "github.com/mcqueenorama/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/stats" ) // Listener wraps a net.Listener, and gives a place to store the timeout diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index 2d507ebb6..f4f0b1874 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -14,9 +14,9 @@ import ( "sync" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) type BenchmarkOptions struct { diff --git a/go/weed/compact.go b/go/weed/compact.go index 1ecb85cf9..71c4ea90f 100644 --- a/go/weed/compact.go +++ b/go/weed/compact.go @@ -1,8 +1,8 @@ package main import ( - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/download.go b/go/weed/download.go index 82c0fd45e..c782654f5 100644 --- a/go/weed/download.go +++ b/go/weed/download.go @@ -8,8 +8,8 @@ import ( "path" "strings" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) var ( diff --git a/go/weed/export.go b/go/weed/export.go index 472e63aea..c9cc0e3fe 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -11,8 +11,8 @@ import ( "text/template" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/filer.go b/go/weed/filer.go index 173ccddc7..11154f183 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -6,9 +6,9 @@ import ( "strconv" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" - "github.com/mcqueenorama/weed-fs/go/weed/weed_server" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" ) var ( diff --git a/go/weed/fix.go b/go/weed/fix.go index 0f5f0b7f3..e66075ed2 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -5,8 +5,8 @@ import ( "path" "strconv" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/master.go b/go/weed/master.go index 885e6e207..f88964b6d 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -8,9 +8,9 @@ import ( "strings" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" - "github.com/mcqueenorama/weed-fs/go/weed/weed_server" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" "github.com/gorilla/mux" ) diff --git a/go/weed/mount_std.go b/go/weed/mount_std.go index e930240b1..808c6c563 100644 --- a/go/weed/mount_std.go +++ b/go/weed/mount_std.go @@ -9,10 +9,10 @@ import ( "bazil.org/fuse" "bazil.org/fuse/fs" - "github.com/mcqueenorama/weed-fs/go/filer" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/filer" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) func runMount(cmd *Command, args []string) bool { diff --git a/go/weed/server.go b/go/weed/server.go index 6a5a0a935..67bbdb370 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -10,9 +10,9 @@ import ( "sync" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" - "github.com/mcqueenorama/weed-fs/go/weed/weed_server" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" "github.com/gorilla/mux" ) diff --git a/go/weed/shell.go b/go/weed/shell.go index 04cfb4730..f2c4990ea 100644 --- a/go/weed/shell.go +++ b/go/weed/shell.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) func init() { diff --git a/go/weed/upload.go b/go/weed/upload.go index 7623f7d9b..2d67c0bd9 100644 --- a/go/weed/upload.go +++ b/go/weed/upload.go @@ -6,7 +6,7 @@ import ( "os" "path/filepath" - "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/operation" ) var ( diff --git a/go/weed/version.go b/go/weed/version.go index 13f7ea23a..8d3a6fed7 100644 --- a/go/weed/version.go +++ b/go/weed/version.go @@ -4,7 +4,7 @@ import ( "fmt" "runtime" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" ) var cmdVersion = &Command{ diff --git a/go/weed/volume.go b/go/weed/volume.go index 71a296702..212cb4b33 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -8,9 +8,9 @@ import ( "strings" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" - "github.com/mcqueenorama/weed-fs/go/weed/weed_server" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" ) func init() { diff --git a/go/weed/volume_test.go b/go/weed/volume_test.go index b936ba4f2..ef00a8c7c 100644 --- a/go/weed/volume_test.go +++ b/go/weed/volume_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) func TestXYZ(t *testing.T) { diff --git a/go/weed/weed.go b/go/weed/weed.go index 0620cdf17..c304b7f35 100644 --- a/go/weed/weed.go +++ b/go/weed/weed.go @@ -13,7 +13,7 @@ import ( "unicode" "unicode/utf8" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) var IsDebug *bool diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 55f5e69a4..39b0830e3 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -10,11 +10,11 @@ import ( "strconv" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/stats" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) var serverStats *stats.ServerStats diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 2630a3398..0bda58d06 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -4,8 +4,8 @@ import ( "net/http" "strconv" - "github.com/mcqueenorama/weed-fs/go/filer" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/filer" + "github.com/chrislusf/weed-fs/go/glog" ) type FilerServer struct { diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index e5a7a860e..6f22912a7 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -11,9 +11,9 @@ import ( "strconv" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/go/weed/weed_server/filer_server_handlers_admin.go b/go/weed/weed_server/filer_server_handlers_admin.go index f63701a6b..5ba12e0b8 100644 --- a/go/weed/weed_server/filer_server_handlers_admin.go +++ b/go/weed/weed_server/filer_server_handlers_admin.go @@ -3,7 +3,7 @@ package weed_server import ( "net/http" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) /* diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go index d96354eb9..d000cb610 100644 --- a/go/weed/weed_server/master_server.go +++ b/go/weed/weed_server/master_server.go @@ -6,10 +6,10 @@ import ( "net/url" "sync" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/sequence" - "github.com/mcqueenorama/weed-fs/go/topology" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/sequence" + "github.com/chrislusf/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/util" "github.com/goraft/raft" "github.com/gorilla/mux" ) diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 25839669f..7a7a3b70d 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -5,9 +5,9 @@ import ( "strconv" "strings" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/stats" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" ) func (ms *MasterServer) lookupVolumeId(vids []string, collection string) (volumeLocations map[string]operation.LookupResult) { diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 6ad83784b..d7124e567 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -9,11 +9,11 @@ import ( "strings" proto "code.google.com/p/goprotobuf/proto" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/topology" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/util" ) func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index fb85daecf..b9aaef2b0 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -14,8 +14,8 @@ import ( "strings" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/topology" "github.com/goraft/raft" "github.com/gorilla/mux" ) diff --git a/go/weed/weed_server/raft_server_handlers.go b/go/weed/weed_server/raft_server_handlers.go index 340a34e7b..b466d9afa 100644 --- a/go/weed/weed_server/raft_server_handlers.go +++ b/go/weed/weed_server/raft_server_handlers.go @@ -6,8 +6,8 @@ import ( "net/http" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" "github.com/goraft/raft" ) diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index 3e40a790a..0a65fd2f6 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -6,8 +6,8 @@ import ( "strconv" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) type VolumeServer struct { diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index e14bcea9e..83f614941 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -9,12 +9,12 @@ import ( "strings" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/images" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/stats" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/images" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/topology" ) var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go index 0d3f6ac5b..caf4c3be8 100644 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ b/go/weed/weed_server/volume_server_handlers_admin.go @@ -4,9 +4,9 @@ import ( "net/http" "path/filepath" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/stats" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/util" ) func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/volume_server_handlers_vacuum.go b/go/weed/weed_server/volume_server_handlers_vacuum.go index da7172ae9..f115e3b8b 100644 --- a/go/weed/weed_server/volume_server_handlers_vacuum.go +++ b/go/weed/weed_server/volume_server_handlers_vacuum.go @@ -3,7 +3,7 @@ package weed_server import ( "net/http" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) {