diff --git a/go/glog/convenient_api.go b/go/glog/convenient_api.go index ac2c2e7f8..db7ad7963 100644 --- a/go/glog/convenient_api.go +++ b/go/glog/convenient_api.go @@ -12,5 +12,5 @@ func ToStderr() { logging.toStderr = true } func ToStderrAndLog() { - logging.alsoToStderr = true + logging.alsoToStderr = true } diff --git a/go/operation/delete_content.go b/go/operation/delete_content.go index 8f8ea58f3..f14259ee6 100644 --- a/go/operation/delete_content.go +++ b/go/operation/delete_content.go @@ -6,19 +6,19 @@ import ( "net/http" ) -func DeleteFile(server string, fileId string) (error) { +func DeleteFile(server string, fileId string) error { fid, parseErr := storage.ParseFileId(fileId) if parseErr != nil { return parseErr } - lookup, lookupError := Lookup(server,fid.VolumeId) + lookup, lookupError := Lookup(server, fid.VolumeId) if lookupError != nil { - return lookupError + return lookupError } if len(lookup.Locations) == 0 { - return nil + return nil } - return Delete("http://"+lookup.Locations[0].PublicUrl+"/"+fileId) + return Delete("http://" + lookup.Locations[0].PublicUrl + "/" + fileId) } func Delete(url string) error { req, err := http.NewRequest("DELETE", url, nil) diff --git a/go/operation/submit.go b/go/operation/submit.go index e0001e60b..d24a89b2b 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -95,8 +95,8 @@ func (fi FilePart) Upload(server string, fid string) (int, error) { if fi.ModTime != 0 { fileUrl += "?ts=" + strconv.Itoa(int(fi.ModTime)) } - if closer, ok := fi.Reader.(io.Closer); ok{ - defer closer.Close() + if closer, ok := fi.Reader.(io.Closer); ok { + defer closer.Close() } ret, e := Upload(fileUrl, fi.FileName, fi.Reader, fi.IsGzipped, fi.MimeType) if e != nil { diff --git a/go/replication/store_replicate.go b/go/replication/store_replicate.go index de2f3fcd8..bc630c5d1 100644 --- a/go/replication/store_replicate.go +++ b/go/replication/store_replicate.go @@ -2,9 +2,9 @@ package replication import ( "bytes" + "code.google.com/p/weed-fs/go/glog" "code.google.com/p/weed-fs/go/operation" "code.google.com/p/weed-fs/go/storage" - "code.google.com/p/weed-fs/go/glog" "net/http" "strconv" ) @@ -25,7 +25,7 @@ func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.Volum if needToReplicate { //send to other replica locations if r.FormValue("type") != "replicate" { if !distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool { - _, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified,10), string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime)) + _, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified, 10), string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime)) return err == nil }) { ret = 0 diff --git a/go/replication/volume_growth.go b/go/replication/volume_growth.go index 18295498e..2adf72dee 100644 --- a/go/replication/volume_growth.go +++ b/go/replication/volume_growth.go @@ -1,12 +1,12 @@ package replication import ( + "code.google.com/p/weed-fs/go/glog" "code.google.com/p/weed-fs/go/operation" "code.google.com/p/weed-fs/go/storage" "code.google.com/p/weed-fs/go/topology" "errors" "fmt" - "code.google.com/p/weed-fs/go/glog" "math/rand" "sync" ) @@ -39,13 +39,13 @@ func (vg *VolumeGrowth) AutomaticGrowByType(repType storage.ReplicationType, dat factor = 1 count, err = vg.GrowByCountAndType(vg.copy1factor, repType, dataCenter, topo) case storage.Copy001: - factor = 2 + factor = 2 count, err = vg.GrowByCountAndType(vg.copy2factor, repType, dataCenter, topo) case storage.Copy010: - factor = 2 + factor = 2 count, err = vg.GrowByCountAndType(vg.copy2factor, repType, dataCenter, topo) case storage.Copy100: - factor = 2 + factor = 2 count, err = vg.GrowByCountAndType(vg.copy2factor, repType, dataCenter, topo) case storage.Copy110: factor = 3 diff --git a/go/sequence/sequence.go b/go/sequence/sequence.go index 658996153..6def37d49 100644 --- a/go/sequence/sequence.go +++ b/go/sequence/sequence.go @@ -1,8 +1,8 @@ package sequence import ( - "encoding/gob" "code.google.com/p/weed-fs/go/glog" + "encoding/gob" "os" "path" "sync" diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go index 451d8727a..a5480ac11 100644 --- a/go/storage/cdb_map.go +++ b/go/storage/cdb_map.go @@ -76,6 +76,9 @@ func (m cdbMap) FileCount() int { func (m *cdbMap) DeletedCount() int { return m.DeletionCounter } +func (m *cdbMap) NextFileKey(count int) (uint64) { + return 0 +} func getMetric(c *cdb.Cdb, m *mapMetric) error { data, err := c.Data([]byte{'M'}) diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index d97e7a66b..460085dee 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -1,8 +1,8 @@ package storage import ( - "code.google.com/p/weed-fs/go/util" "code.google.com/p/weed-fs/go/glog" + "code.google.com/p/weed-fs/go/util" "os" "testing" ) diff --git a/go/storage/compress.go b/go/storage/compress.go index eaf3c9ad8..b557b3411 100644 --- a/go/storage/compress.go +++ b/go/storage/compress.go @@ -2,9 +2,9 @@ package storage import ( "bytes" + "code.google.com/p/weed-fs/go/glog" "compress/flate" "compress/gzip" - "code.google.com/p/weed-fs/go/glog" "io/ioutil" "strings" ) diff --git a/go/storage/needle.go b/go/storage/needle.go index 54799df4e..e23f4e67b 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -1,10 +1,10 @@ package storage import ( + "code.google.com/p/weed-fs/go/glog" "code.google.com/p/weed-fs/go/util" "encoding/hex" "io/ioutil" - "code.google.com/p/weed-fs/go/glog" "mime" "net/http" "path" @@ -86,19 +86,19 @@ func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string fileName = fileName[:len(fileName)-3] } modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64) - return + return } func NewNeedle(r *http.Request) (n *Needle, e error) { - fname, mimeType, isGzipped := "", "", false + fname, mimeType, isGzipped := "", "", false n = new(Needle) fname, n.Data, mimeType, isGzipped, n.LastModified, e = ParseUpload(r) if e != nil { - return + return + } + if len(fname) < 256 { + n.Name = []byte(fname) + n.SetHasName() } - if len(fname) < 256 { - n.Name = []byte(fname) - n.SetHasName() - } if len(mimeType) < 256 { n.Mime = []byte(mimeType) n.SetHasMime() @@ -108,7 +108,7 @@ func NewNeedle(r *http.Request) (n *Needle, e error) { } if n.LastModified == 0 { n.LastModified = uint64(time.Now().Unix()) - n.SetHasLastModifiedDate() + n.SetHasLastModifiedDate() } n.Checksum = NewCRC(n.Data) @@ -127,9 +127,6 @@ func NewNeedle(r *http.Request) (n *Needle, e error) { func (n *Needle) ParsePath(fid string) { length := len(fid) if length <= 8 { - if length > 0 { - glog.V(0).Infoln("Invalid fid", fid, "length", length) - } return } delta := "" diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index a242cd673..29b71ae52 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -2,6 +2,7 @@ package storage import ( "bufio" + "code.google.com/p/weed-fs/go/glog" "code.google.com/p/weed-fs/go/util" "fmt" "io" @@ -18,6 +19,7 @@ type NeedleMapper interface { FileCount() int DeletedCount() int Visit(visit func(NeedleValue) error) (err error) + NextFileKey(count int) uint64 } type mapMetric struct { @@ -25,6 +27,7 @@ type mapMetric struct { FileCounter int `json:"FileCounter"` DeletionByteCounter uint64 `json:"DeletionByteCounter"` FileByteCounter uint64 `json:"FileByteCounter"` + MaximumFileKey uint64 `json:"MaxFileKey"` } type NeedleMap struct { @@ -53,23 +56,27 @@ const ( func LoadNeedleMap(file *os.File) (*NeedleMap, error) { nm := NewNeedleMap(file) e := walkIndexFile(file, func(key uint64, offset, size uint32) error { + if key > nm.MaximumFileKey { + nm.MaximumFileKey = key + } nm.FileCounter++ nm.FileByteCounter = nm.FileByteCounter + uint64(size) if offset > 0 { oldSize := nm.m.Set(Key(key), offset, size) - //glog.V(0).Infoln("reading key", key, "offset", offset, "size", size, "oldSize", oldSize) + glog.V(4).Infoln("reading key", key, "offset", offset, "size", size, "oldSize", oldSize) if oldSize > 0 { nm.DeletionCounter++ nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize) } } else { oldSize := nm.m.Delete(Key(key)) - //glog.V(0).Infoln("removing key", key, "offset", offset, "size", size, "oldSize", oldSize) + glog.V(4).Infoln("removing key", key, "offset", offset, "size", size, "oldSize", oldSize) nm.DeletionCounter++ nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize) } return nil }) + glog.V(1).Infoln("max file key:", nm.MaximumFileKey) return nm, e } @@ -163,3 +170,14 @@ func (nm NeedleMap) DeletedCount() int { func (nm *NeedleMap) Visit(visit func(NeedleValue) error) (err error) { return nm.m.Visit(visit) } +func (nm NeedleMap) MaxFileKey() uint64 { + return nm.MaximumFileKey +} +func (nm NeedleMap) NextFileKey(count int) (ret uint64) { + if count <= 0 { + return 0 + } + ret = nm.MaximumFileKey + nm.MaximumFileKey += uint64(count) + return +} diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index 0b833c0a9..d525a10d5 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -1,11 +1,11 @@ package storage import ( + "code.google.com/p/weed-fs/go/glog" "code.google.com/p/weed-fs/go/util" "errors" "fmt" "io" - "code.google.com/p/weed-fs/go/glog" "os" ) diff --git a/go/storage/store.go b/go/storage/store.go index 143d2b706..bea3bd34f 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -1,11 +1,11 @@ package storage import ( + "code.google.com/p/weed-fs/go/glog" "code.google.com/p/weed-fs/go/util" "encoding/json" "fmt" "io/ioutil" - "code.google.com/p/weed-fs/go/glog" "net/url" "strconv" "strings" @@ -97,10 +97,10 @@ func (s *Store) addVolume(vid VolumeId, replicationType ReplicationType) error { if location := s.findFreeLocation(); location != nil { glog.V(0).Infoln("In dir", location.directory, "adds volume =", vid, ", replicationType =", replicationType) if volume, err := NewVolume(location.directory, vid, replicationType); err == nil { - location.volumes[vid] = volume - return nil + location.volumes[vid] = volume + return nil } else { - return err + return err } } return fmt.Errorf("No more free space left") diff --git a/go/storage/volume.go b/go/storage/volume.go index 6097aeec5..094134356 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -64,7 +64,7 @@ func (v *Volume) load(alsoLoadIndex bool) error { v.dataFile, e = os.Open(fileName + ".dat") v.readOnly = true } else { - return fmt.Errorf("Unknown state about Volume Data file %s.dat", fileName) + return fmt.Errorf("Unknown state about Volume Data file %s.dat", fileName) } if e != nil { if !os.IsPermission(e) { diff --git a/go/topology/node.go b/go/topology/node.go index c8ff55f07..cfd6f6489 100644 --- a/go/topology/node.go +++ b/go/topology/node.go @@ -1,8 +1,8 @@ package topology import ( - "code.google.com/p/weed-fs/go/storage" "code.google.com/p/weed-fs/go/glog" + "code.google.com/p/weed-fs/go/storage" ) type NodeId string diff --git a/go/topology/node_list.go b/go/topology/node_list.go index db7685d82..bed151b54 100644 --- a/go/topology/node_list.go +++ b/go/topology/node_list.go @@ -1,8 +1,8 @@ package topology import ( - "code.google.com/p/weed-fs/go/storage" "code.google.com/p/weed-fs/go/glog" + "code.google.com/p/weed-fs/go/storage" "math/rand" ) diff --git a/go/topology/node_list_test.go b/go/topology/node_list_test.go index 0037cbaa9..c7b165ea6 100644 --- a/go/topology/node_list_test.go +++ b/go/topology/node_list_test.go @@ -25,21 +25,21 @@ func TestXYZ(t *testing.T) { t.Error("need to randomly pick 1 node") } - picked, ret = nl.RandomlyPickN(1, 0, "dc1") - if !ret || len(picked) != 1 { - t.Error("need to randomly pick 1 node") - } - if picked[0].Id() != "dc1" { - t.Error("need to randomly pick 1 dc1 node") - } - - picked, ret = nl.RandomlyPickN(2, 0, "dc1") - if !ret || len(picked) != 2 { - t.Error("need to randomly pick 1 node") - } - if picked[0].Id() != "dc1" { - t.Error("need to randomly pick 2 with one dc1 node") - } + picked, ret = nl.RandomlyPickN(1, 0, "dc1") + if !ret || len(picked) != 1 { + t.Error("need to randomly pick 1 node") + } + if picked[0].Id() != "dc1" { + t.Error("need to randomly pick 1 dc1 node") + } + + picked, ret = nl.RandomlyPickN(2, 0, "dc1") + if !ret || len(picked) != 2 { + t.Error("need to randomly pick 1 node") + } + if picked[0].Id() != "dc1" { + t.Error("need to randomly pick 2 with one dc1 node") + } picked, ret = nl.RandomlyPickN(4, 0, "") if !ret || len(picked) != 4 { diff --git a/go/topology/topology.go b/go/topology/topology.go index f70c53ae6..d0e9fb42b 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -1,11 +1,11 @@ package topology import ( + "code.google.com/p/weed-fs/go/glog" "code.google.com/p/weed-fs/go/sequence" "code.google.com/p/weed-fs/go/storage" "errors" "io/ioutil" - "code.google.com/p/weed-fs/go/glog" "math/rand" ) diff --git a/go/topology/topology_compact.go b/go/topology/topology_compact.go index 45ddda174..6394c59c6 100644 --- a/go/topology/topology_compact.go +++ b/go/topology/topology_compact.go @@ -1,11 +1,11 @@ package topology import ( + "code.google.com/p/weed-fs/go/glog" "code.google.com/p/weed-fs/go/storage" "code.google.com/p/weed-fs/go/util" "encoding/json" "errors" - "code.google.com/p/weed-fs/go/glog" "net/url" "time" ) diff --git a/go/topology/topology_event_handling.go b/go/topology/topology_event_handling.go index c780e2beb..f3b09c649 100644 --- a/go/topology/topology_event_handling.go +++ b/go/topology/topology_event_handling.go @@ -1,8 +1,8 @@ package topology import ( - "code.google.com/p/weed-fs/go/storage" "code.google.com/p/weed-fs/go/glog" + "code.google.com/p/weed-fs/go/storage" "math/rand" "time" ) diff --git a/go/util/config.go b/go/util/config.go index 43c5027e1..9a1ac680b 100644 --- a/go/util/config.go +++ b/go/util/config.go @@ -10,8 +10,8 @@ package util import ( "bytes" - "encoding/json" "code.google.com/p/weed-fs/go/glog" + "encoding/json" "os" ) diff --git a/go/util/post.go b/go/util/post.go index af5948f77..cbc6dcfd5 100644 --- a/go/util/post.go +++ b/go/util/post.go @@ -1,8 +1,8 @@ package util import ( - "io/ioutil" "code.google.com/p/weed-fs/go/glog" + "io/ioutil" "net/http" "net/url" ) diff --git a/go/weed/export.go b/go/weed/export.go index 23c0baf60..965a17699 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -3,9 +3,9 @@ package main import ( "archive/tar" "bytes" + "code.google.com/p/weed-fs/go/glog" "code.google.com/p/weed-fs/go/storage" "fmt" - "code.google.com/p/weed-fs/go/glog" "os" "path" "strconv" diff --git a/go/weed/fix.go b/go/weed/fix.go index 81b87eb18..e1cc63943 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -1,8 +1,8 @@ package main import ( - "code.google.com/p/weed-fs/go/storage" "code.google.com/p/weed-fs/go/glog" + "code.google.com/p/weed-fs/go/storage" "os" "path" "strconv" diff --git a/go/weed/master.go b/go/weed/master.go index 2d7adef1f..7aa48e59c 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -182,7 +182,7 @@ func volumeStatusHandler(w http.ResponseWriter, r *http.Request) { } func redirectHandler(w http.ResponseWriter, r *http.Request) { - vid, _, _, _ := parseURLPath(r.URL.Path) + vid, _, _, _, _ := parseURLPath(r.URL.Path) volumeId, err := storage.NewVolumeId(vid) if err != nil { debug("parsing error:", err, r.URL.Path) diff --git a/go/weed/shell.go b/go/weed/shell.go index 14c84a5d1..885a0fa71 100644 --- a/go/weed/shell.go +++ b/go/weed/shell.go @@ -2,8 +2,8 @@ package main import ( "bufio" - "fmt" "code.google.com/p/weed-fs/go/glog" + "fmt" "os" ) diff --git a/go/weed/volume.go b/go/weed/volume.go index 3c4db7ff1..cf58af799 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -118,7 +118,7 @@ func storeHandler(w http.ResponseWriter, r *http.Request) { } func GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) { n := new(storage.Needle) - vid, fid, filename, ext := parseURLPath(r.URL.Path) + vid, fid, filename, ext, _ := parseURLPath(r.URL.Path) volumeId, err := storage.NewVolumeId(vid) if err != nil { debug("parsing error:", err, r.URL.Path) @@ -207,7 +207,7 @@ func PostHandler(w http.ResponseWriter, r *http.Request) { writeJsonError(w, r, e) return } - vid, _, _, _ := parseURLPath(r.URL.Path) + vid, _, _, _, _ := parseURLPath(r.URL.Path) volumeId, ve := storage.NewVolumeId(vid) if ve != nil { debug("NewVolumeId error:", ve) @@ -231,7 +231,7 @@ func PostHandler(w http.ResponseWriter, r *http.Request) { } func DeleteHandler(w http.ResponseWriter, r *http.Request) { n := new(storage.Needle) - vid, fid, _, _ := parseURLPath(r.URL.Path) + vid, fid, _, _, _ := parseURLPath(r.URL.Path) volumeId, _ := storage.NewVolumeId(vid) n.ParsePath(fid) @@ -266,7 +266,7 @@ func DeleteHandler(w http.ResponseWriter, r *http.Request) { writeJsonQuiet(w, r, m) } -func parseURLPath(path string) (vid, fid, filename, ext string) { +func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) { switch strings.Count(path, "/") { case 3: parts := strings.Split(path, "/") @@ -284,9 +284,7 @@ func parseURLPath(path string) (vid, fid, filename, ext string) { sepIndex := strings.LastIndex(path, "/") commaIndex := strings.LastIndex(path[sepIndex:], ",") if commaIndex <= 0 { - if "favicon.ico" != path[sepIndex+1:] { - glog.V(0).Infoln("unknown file id", path[sepIndex+1:]) - } + vid, isVolumeIdOnly = path[sepIndex+1:], true return } dotIndex := strings.LastIndex(path[sepIndex:], ".") diff --git a/go/weed/weed.go b/go/weed/weed.go index dc74b7650..e9bc4e37d 100644 --- a/go/weed/weed.go +++ b/go/weed/weed.go @@ -7,7 +7,7 @@ import ( "fmt" "io" "math/rand" - "net" + "net" "net/http" "os" "strings" @@ -225,20 +225,20 @@ func debug(params ...interface{}) { } } func secure(whiteList []string, f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - if len(whiteList) == 0 { - f(w, r) - return - } - host, _, err := net.SplitHostPort(r.RemoteAddr) - if err == nil { - for _, ip := range whiteList { - if ip == host { - f(w, r) - return - } - } - } - writeJsonQuiet(w, r, map[string]interface{}{"error": "No write permisson from " + host}) - } + return func(w http.ResponseWriter, r *http.Request) { + if len(whiteList) == 0 { + f(w, r) + return + } + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err == nil { + for _, ip := range whiteList { + if ip == host { + f(w, r) + return + } + } + } + writeJsonQuiet(w, r, map[string]interface{}{"error": "No write permisson from " + host}) + } }