issue 43 "go fmt" chagnes from "Ryan S. Brown" <sb@ryansb.com>

some basic changes to parse upload url

go/glog/convenient_api.go

@@ -12,5 +12,5 @@ func ToStderr() {
 	logging.toStderr = true
 }
 func ToStderrAndLog() {
-  logging.alsoToStderr = true
+	logging.alsoToStderr = true
 }

go/operation/delete_content.go

@@ -6,19 +6,19 @@ import (
 	"net/http"
 )
 
-func DeleteFile(server string, fileId string) (error) {
+func DeleteFile(server string, fileId string) error {
 	fid, parseErr := storage.ParseFileId(fileId)
 	if parseErr != nil {
 		return parseErr
 	}
-	lookup, lookupError := Lookup(server,fid.VolumeId)
+	lookup, lookupError := Lookup(server, fid.VolumeId)
 	if lookupError != nil {
-	  return lookupError
+		return lookupError
 	}
 	if len(lookup.Locations) == 0 {
-	  return nil
+		return nil
 	}
-	return Delete("http://"+lookup.Locations[0].PublicUrl+"/"+fileId)
+	return Delete("http://" + lookup.Locations[0].PublicUrl + "/" + fileId)
 }
 func Delete(url string) error {
 	req, err := http.NewRequest("DELETE", url, nil)

go/operation/submit.go

@@ -95,8 +95,8 @@ func (fi FilePart) Upload(server string, fid string) (int, error) {
 	if fi.ModTime != 0 {
 		fileUrl += "?ts=" + strconv.Itoa(int(fi.ModTime))
 	}
-	if closer, ok := fi.Reader.(io.Closer); ok{
-	  defer closer.Close()
+	if closer, ok := fi.Reader.(io.Closer); ok {
+		defer closer.Close()
 	}
 	ret, e := Upload(fileUrl, fi.FileName, fi.Reader, fi.IsGzipped, fi.MimeType)
 	if e != nil {

go/replication/store_replicate.go

@@ -2,9 +2,9 @@ package replication
 
 import (
 	"bytes"
+	"code.google.com/p/weed-fs/go/glog"
 	"code.google.com/p/weed-fs/go/operation"
 	"code.google.com/p/weed-fs/go/storage"
-	"code.google.com/p/weed-fs/go/glog"
 	"net/http"
 	"strconv"
 )
@@ -25,7 +25,7 @@ func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.Volum
 	if needToReplicate { //send to other replica locations
 		if r.FormValue("type") != "replicate" {
 			if !distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool {
-				_, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified,10), string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime))
+				_, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified, 10), string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime))
 				return err == nil
 			}) {
 				ret = 0

go/replication/volume_growth.go

@@ -1,12 +1,12 @@
 package replication
 
 import (
+	"code.google.com/p/weed-fs/go/glog"
 	"code.google.com/p/weed-fs/go/operation"
 	"code.google.com/p/weed-fs/go/storage"
 	"code.google.com/p/weed-fs/go/topology"
 	"errors"
 	"fmt"
-	"code.google.com/p/weed-fs/go/glog"
 	"math/rand"
 	"sync"
 )
@@ -39,13 +39,13 @@ func (vg *VolumeGrowth) AutomaticGrowByType(repType storage.ReplicationType, dat
 		factor = 1
 		count, err = vg.GrowByCountAndType(vg.copy1factor, repType, dataCenter, topo)
 	case storage.Copy001:
-    factor = 2
+		factor = 2
 		count, err = vg.GrowByCountAndType(vg.copy2factor, repType, dataCenter, topo)
 	case storage.Copy010:
-    factor = 2
+		factor = 2
 		count, err = vg.GrowByCountAndType(vg.copy2factor, repType, dataCenter, topo)
 	case storage.Copy100:
-    factor = 2
+		factor = 2
 		count, err = vg.GrowByCountAndType(vg.copy2factor, repType, dataCenter, topo)
 	case storage.Copy110:
 		factor = 3

go/sequence/sequence.go

@@ -1,8 +1,8 @@
 package sequence
 
 import (
-	"encoding/gob"
 	"code.google.com/p/weed-fs/go/glog"
+	"encoding/gob"
 	"os"
 	"path"
 	"sync"

go/storage/cdb_map.go

@@ -76,6 +76,9 @@ func (m cdbMap) FileCount() int {
 func (m *cdbMap) DeletedCount() int {
 	return m.DeletionCounter
 }
+func (m *cdbMap) NextFileKey(count int) (uint64) {
+  return 0
+}
 
 func getMetric(c *cdb.Cdb, m *mapMetric) error {
 	data, err := c.Data([]byte{'M'})

go/storage/compact_map_perf_test.go

@@ -1,8 +1,8 @@
 package storage
 
 import (
-	"code.google.com/p/weed-fs/go/util"
 	"code.google.com/p/weed-fs/go/glog"
+	"code.google.com/p/weed-fs/go/util"
 	"os"
 	"testing"
 )

go/storage/compress.go

@@ -2,9 +2,9 @@ package storage
 
 import (
 	"bytes"
+	"code.google.com/p/weed-fs/go/glog"
 	"compress/flate"
 	"compress/gzip"
-	"code.google.com/p/weed-fs/go/glog"
 	"io/ioutil"
 	"strings"
 )

go/storage/needle.go

@@ -1,10 +1,10 @@
 package storage
 
 import (
+	"code.google.com/p/weed-fs/go/glog"
 	"code.google.com/p/weed-fs/go/util"
 	"encoding/hex"
 	"io/ioutil"
-	"code.google.com/p/weed-fs/go/glog"
 	"mime"
 	"net/http"
 	"path"
@@ -86,19 +86,19 @@ func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string
 		fileName = fileName[:len(fileName)-3]
 	}
 	modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
-  return
+	return
 }
 func NewNeedle(r *http.Request) (n *Needle, e error) {
-  fname, mimeType, isGzipped := "", "", false
+	fname, mimeType, isGzipped := "", "", false
 	n = new(Needle)
 	fname, n.Data, mimeType, isGzipped, n.LastModified, e = ParseUpload(r)
 	if e != nil {
-	  return
+		return
+	}
+	if len(fname) < 256 {
+		n.Name = []byte(fname)
+		n.SetHasName()
 	}
-  if len(fname) < 256 {
-    n.Name = []byte(fname)
-    n.SetHasName()
-  }
 	if len(mimeType) < 256 {
 		n.Mime = []byte(mimeType)
 		n.SetHasMime()
@@ -108,7 +108,7 @@ func NewNeedle(r *http.Request) (n *Needle, e error) {
 	}
 	if n.LastModified == 0 {
 		n.LastModified = uint64(time.Now().Unix())
-    n.SetHasLastModifiedDate()
+		n.SetHasLastModifiedDate()
 	}
 
 	n.Checksum = NewCRC(n.Data)
@@ -127,9 +127,6 @@ func NewNeedle(r *http.Request) (n *Needle, e error) {
 func (n *Needle) ParsePath(fid string) {
 	length := len(fid)
 	if length <= 8 {
-		if length > 0 {
-			glog.V(0).Infoln("Invalid fid", fid, "length", length)
-		}
 		return
 	}
 	delta := ""

go/storage/needle_map.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"bufio"
+	"code.google.com/p/weed-fs/go/glog"
 	"code.google.com/p/weed-fs/go/util"
 	"fmt"
 	"io"
@@ -18,6 +19,7 @@ type NeedleMapper interface {
 	FileCount() int
 	DeletedCount() int
 	Visit(visit func(NeedleValue) error) (err error)
+	NextFileKey(count int) uint64
 }
 
 type mapMetric struct {
@@ -25,6 +27,7 @@ type mapMetric struct {
 	FileCounter         int    `json:"FileCounter"`
 	DeletionByteCounter uint64 `json:"DeletionByteCounter"`
 	FileByteCounter     uint64 `json:"FileByteCounter"`
+	MaximumFileKey      uint64 `json:"MaxFileKey"`
 }
 
 type NeedleMap struct {
@@ -53,23 +56,27 @@ const (
 func LoadNeedleMap(file *os.File) (*NeedleMap, error) {
 	nm := NewNeedleMap(file)
 	e := walkIndexFile(file, func(key uint64, offset, size uint32) error {
+		if key > nm.MaximumFileKey {
+			nm.MaximumFileKey = key
+		}
 		nm.FileCounter++
 		nm.FileByteCounter = nm.FileByteCounter + uint64(size)
 		if offset > 0 {
 			oldSize := nm.m.Set(Key(key), offset, size)
-			//glog.V(0).Infoln("reading key", key, "offset", offset, "size", size, "oldSize", oldSize)
+			glog.V(4).Infoln("reading key", key, "offset", offset, "size", size, "oldSize", oldSize)
 			if oldSize > 0 {
 				nm.DeletionCounter++
 				nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
 			}
 		} else {
 			oldSize := nm.m.Delete(Key(key))
-			//glog.V(0).Infoln("removing key", key, "offset", offset, "size", size, "oldSize", oldSize)
+			glog.V(4).Infoln("removing key", key, "offset", offset, "size", size, "oldSize", oldSize)
 			nm.DeletionCounter++
 			nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
 		}
 		return nil
 	})
+	glog.V(1).Infoln("max file key:", nm.MaximumFileKey)
 	return nm, e
 }
 
@@ -163,3 +170,14 @@ func (nm NeedleMap) DeletedCount() int {
 func (nm *NeedleMap) Visit(visit func(NeedleValue) error) (err error) {
 	return nm.m.Visit(visit)
 }
+func (nm NeedleMap) MaxFileKey() uint64 {
+	return nm.MaximumFileKey
+}
+func (nm NeedleMap) NextFileKey(count int) (ret uint64) {
+	if count <= 0 {
+		return 0
+	}
+	ret = nm.MaximumFileKey
+	nm.MaximumFileKey += uint64(count)
+	return
+}

go/storage/needle_read_write.go

@@ -1,11 +1,11 @@
 package storage
 
 import (
+	"code.google.com/p/weed-fs/go/glog"
 	"code.google.com/p/weed-fs/go/util"
 	"errors"
 	"fmt"
 	"io"
-	"code.google.com/p/weed-fs/go/glog"
 	"os"
 )
 

go/storage/store.go

@@ -1,11 +1,11 @@
 package storage
 
 import (
+	"code.google.com/p/weed-fs/go/glog"
 	"code.google.com/p/weed-fs/go/util"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"code.google.com/p/weed-fs/go/glog"
 	"net/url"
 	"strconv"
 	"strings"
@@ -97,10 +97,10 @@ func (s *Store) addVolume(vid VolumeId, replicationType ReplicationType) error {
 	if location := s.findFreeLocation(); location != nil {
 		glog.V(0).Infoln("In dir", location.directory, "adds volume =", vid, ", replicationType =", replicationType)
 		if volume, err := NewVolume(location.directory, vid, replicationType); err == nil {
-      location.volumes[vid] = volume
-      return nil
+			location.volumes[vid] = volume
+			return nil
 		} else {
-      return err
+			return err
 		}
 	}
 	return fmt.Errorf("No more free space left")

go/storage/volume.go

@@ -64,7 +64,7 @@ func (v *Volume) load(alsoLoadIndex bool) error {
 		v.dataFile, e = os.Open(fileName + ".dat")
 		v.readOnly = true
 	} else {
-    return fmt.Errorf("Unknown state about Volume Data file %s.dat", fileName)
+		return fmt.Errorf("Unknown state about Volume Data file %s.dat", fileName)
 	}
 	if e != nil {
 		if !os.IsPermission(e) {

go/topology/node.go

@@ -1,8 +1,8 @@
 package topology
 
 import (
-	"code.google.com/p/weed-fs/go/storage"
 	"code.google.com/p/weed-fs/go/glog"
+	"code.google.com/p/weed-fs/go/storage"
 )
 
 type NodeId string

go/topology/node_list.go

@@ -1,8 +1,8 @@
 package topology
 
 import (
-	"code.google.com/p/weed-fs/go/storage"
 	"code.google.com/p/weed-fs/go/glog"
+	"code.google.com/p/weed-fs/go/storage"
 	"math/rand"
 )
 

go/topology/node_list_test.go

@@ -25,21 +25,21 @@ func TestXYZ(t *testing.T) {
 		t.Error("need to randomly pick 1 node")
 	}
 
-  picked, ret = nl.RandomlyPickN(1, 0, "dc1")
-  if !ret || len(picked) != 1 {
-    t.Error("need to randomly pick 1 node")
-  }
-  if picked[0].Id() != "dc1" {
-    t.Error("need to randomly pick 1 dc1 node")
-  }
-
-  picked, ret = nl.RandomlyPickN(2, 0, "dc1")
-  if !ret || len(picked) != 2 {
-    t.Error("need to randomly pick 1 node")
-  }
-  if picked[0].Id() != "dc1" {
-    t.Error("need to randomly pick 2 with one dc1 node")
-  }
+	picked, ret = nl.RandomlyPickN(1, 0, "dc1")
+	if !ret || len(picked) != 1 {
+		t.Error("need to randomly pick 1 node")
+	}
+	if picked[0].Id() != "dc1" {
+		t.Error("need to randomly pick 1 dc1 node")
+	}
+
+	picked, ret = nl.RandomlyPickN(2, 0, "dc1")
+	if !ret || len(picked) != 2 {
+		t.Error("need to randomly pick 1 node")
+	}
+	if picked[0].Id() != "dc1" {
+		t.Error("need to randomly pick 2 with one dc1 node")
+	}
 
 	picked, ret = nl.RandomlyPickN(4, 0, "")
 	if !ret || len(picked) != 4 {

go/topology/topology.go

@@ -1,11 +1,11 @@
 package topology
 
 import (
+	"code.google.com/p/weed-fs/go/glog"
 	"code.google.com/p/weed-fs/go/sequence"
 	"code.google.com/p/weed-fs/go/storage"
 	"errors"
 	"io/ioutil"
-	"code.google.com/p/weed-fs/go/glog"
 	"math/rand"
 )
 

go/topology/topology_compact.go

@@ -1,11 +1,11 @@
 package topology
 
 import (
+	"code.google.com/p/weed-fs/go/glog"
 	"code.google.com/p/weed-fs/go/storage"
 	"code.google.com/p/weed-fs/go/util"
 	"encoding/json"
 	"errors"
-	"code.google.com/p/weed-fs/go/glog"
 	"net/url"
 	"time"
 )

go/topology/topology_event_handling.go

@@ -1,8 +1,8 @@
 package topology
 
 import (
-	"code.google.com/p/weed-fs/go/storage"
 	"code.google.com/p/weed-fs/go/glog"
+	"code.google.com/p/weed-fs/go/storage"
 	"math/rand"
 	"time"
 )

go/util/config.go

@@ -10,8 +10,8 @@ package util
 
 import (
 	"bytes"
-	"encoding/json"
 	"code.google.com/p/weed-fs/go/glog"
+	"encoding/json"
 	"os"
 )
 

go/util/post.go

@@ -1,8 +1,8 @@
 package util
 
 import (
-	"io/ioutil"
 	"code.google.com/p/weed-fs/go/glog"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 )

go/weed/export.go

@@ -3,9 +3,9 @@ package main
 import (
 	"archive/tar"
 	"bytes"
+	"code.google.com/p/weed-fs/go/glog"
 	"code.google.com/p/weed-fs/go/storage"
 	"fmt"
-	"code.google.com/p/weed-fs/go/glog"
 	"os"
 	"path"
 	"strconv"

go/weed/fix.go

@@ -1,8 +1,8 @@
 package main
 
 import (
-	"code.google.com/p/weed-fs/go/storage"
 	"code.google.com/p/weed-fs/go/glog"
+	"code.google.com/p/weed-fs/go/storage"
 	"os"
 	"path"
 	"strconv"

go/weed/master.go

@@ -182,7 +182,7 @@ func volumeStatusHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func redirectHandler(w http.ResponseWriter, r *http.Request) {
-	vid, _, _, _ := parseURLPath(r.URL.Path)
+	vid, _, _, _, _ := parseURLPath(r.URL.Path)
 	volumeId, err := storage.NewVolumeId(vid)
 	if err != nil {
 		debug("parsing error:", err, r.URL.Path)

go/weed/shell.go

@@ -2,8 +2,8 @@ package main
 
 import (
 	"bufio"
-	"fmt"
 	"code.google.com/p/weed-fs/go/glog"
+	"fmt"
 	"os"
 )
 

go/weed/volume.go

@@ -118,7 +118,7 @@ func storeHandler(w http.ResponseWriter, r *http.Request) {
 }
 func GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) {
 	n := new(storage.Needle)
-	vid, fid, filename, ext := parseURLPath(r.URL.Path)
+	vid, fid, filename, ext, _ := parseURLPath(r.URL.Path)
 	volumeId, err := storage.NewVolumeId(vid)
 	if err != nil {
 		debug("parsing error:", err, r.URL.Path)
@@ -207,7 +207,7 @@ func PostHandler(w http.ResponseWriter, r *http.Request) {
 		writeJsonError(w, r, e)
 		return
 	}
-	vid, _, _, _ := parseURLPath(r.URL.Path)
+	vid, _, _, _, _ := parseURLPath(r.URL.Path)
 	volumeId, ve := storage.NewVolumeId(vid)
 	if ve != nil {
 		debug("NewVolumeId error:", ve)
@@ -231,7 +231,7 @@ func PostHandler(w http.ResponseWriter, r *http.Request) {
 }
 func DeleteHandler(w http.ResponseWriter, r *http.Request) {
 	n := new(storage.Needle)
-	vid, fid, _, _ := parseURLPath(r.URL.Path)
+	vid, fid, _, _, _ := parseURLPath(r.URL.Path)
 	volumeId, _ := storage.NewVolumeId(vid)
 	n.ParsePath(fid)
 
@@ -266,7 +266,7 @@ func DeleteHandler(w http.ResponseWriter, r *http.Request) {
 	writeJsonQuiet(w, r, m)
 }
 
-func parseURLPath(path string) (vid, fid, filename, ext string) {
+func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) {
 	switch strings.Count(path, "/") {
 	case 3:
 		parts := strings.Split(path, "/")
@@ -284,9 +284,7 @@ func parseURLPath(path string) (vid, fid, filename, ext string) {
 		sepIndex := strings.LastIndex(path, "/")
 		commaIndex := strings.LastIndex(path[sepIndex:], ",")
 		if commaIndex <= 0 {
-			if "favicon.ico" != path[sepIndex+1:] {
-				glog.V(0).Infoln("unknown file id", path[sepIndex+1:])
-			}
+			vid, isVolumeIdOnly = path[sepIndex+1:], true
 			return
 		}
 		dotIndex := strings.LastIndex(path[sepIndex:], ".")

go/weed/weed.go

@@ -7,7 +7,7 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
-  "net"
+	"net"
 	"net/http"
 	"os"
 	"strings"
@@ -225,20 +225,20 @@ func debug(params ...interface{}) {
 	}
 }
 func secure(whiteList []string, f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
-  return func(w http.ResponseWriter, r *http.Request) {
-    if len(whiteList) == 0 {
-      f(w, r)
-      return
-    }
-    host, _, err := net.SplitHostPort(r.RemoteAddr)
-    if err == nil {
-      for _, ip := range whiteList {
-        if ip == host {
-          f(w, r)
-          return
-        }
-      }
-    }
-    writeJsonQuiet(w, r, map[string]interface{}{"error": "No write permisson from " + host})
-  }
+	return func(w http.ResponseWriter, r *http.Request) {
+		if len(whiteList) == 0 {
+			f(w, r)
+			return
+		}
+		host, _, err := net.SplitHostPort(r.RemoteAddr)
+		if err == nil {
+			for _, ip := range whiteList {
+				if ip == host {
+					f(w, r)
+					return
+				}
+			}
+		}
+		writeJsonQuiet(w, r, map[string]interface{}{"error": "No write permisson from " + host})
+	}
 }