add cmd/dump - a dumper

Walk needed to be added to NeedleMap and CompactMap, to be able to add WalkKeys and WalkValues to volume. This is needed for iterating through all the stored needles in a volume - this was dump's purpose.

weed-fs/src/cmd/dump/main.go

@@ -0,0 +1,96 @@
+// Copyright Tamás Gulácsi 2013 All rights reserved
+// Use of this source is governed by the same rules as the weed-fs library.
+// If this would be ambigous, than Apache License 2.0 has to be used.
+//
+// dump dumps the files of a volume to tar or unique files.
+// Each file will have id#mimetype#original_name file format
+
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"flag"
+	"fmt"
+	// "io"
+	"log"
+	"os"
+	"pkg/storage"
+	"strings"
+	"time"
+)
+
+var (
+	volumePath = flag.String("dir", "/tmp", "volume directory")
+	volumeId   = flag.Int("id", 0, "volume Id")
+	dest       = flag.String("out", "-", "output path. Produces tar if path ends with .tar; creates files otherwise.")
+	tarFh      *tar.Writer
+	tarHeader  tar.Header
+	counter    int
+)
+
+func main() {
+	var err error
+
+	flag.Parse()
+
+	if *dest == "-" {
+		*dest = ""
+	}
+	if *dest == "" || strings.HasSuffix(*dest, ".tar") {
+		var fh *os.File
+		if *dest == "" {
+			fh = os.Stdout
+		} else {
+			if fh, err = os.Create(*dest); err != nil {
+				log.Printf("cannot open output tar %s: %s", *dest, err)
+				return
+			}
+		}
+		defer fh.Close()
+		tarFh = tar.NewWriter(fh)
+		defer tarFh.Close()
+		t := time.Now()
+		tarHeader = tar.Header{Mode: 0644,
+			ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
+			Typeflag:   tar.TypeReg,
+			AccessTime: t, ChangeTime: t}
+	}
+
+	v, err := storage.NewVolume(*volumePath, storage.VolumeId(*volumeId), storage.CopyNil)
+	if v == nil || v.Version() == 0 || err != nil {
+		log.Printf("cannot load volume %d from %s (%s): %s", *volumeId, *volumePath, v, err)
+		return
+	}
+	log.Printf("volume: %s (ver. %d)", v, v.Version())
+	if err := v.WalkValues(walker); err != nil {
+		log.Printf("error while walking: %s", err)
+		return
+	}
+
+	log.Printf("%d files written.", counter)
+}
+
+func walker(n *storage.Needle) (err error) {
+	// log.Printf("Id=%d Size=%d Name=%s mime=%s", n.Id, n.Size, n.Name, n.Mime)
+	nm := fmt.Sprintf("%d#%s#%s", n.Id, bytes.Replace(n.Mime, []byte{'/'}, []byte{'_'}, -1), n.Name)
+	// log.Print(nm)
+	if tarFh != nil {
+		tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
+		if err = tarFh.WriteHeader(&tarHeader); err != nil {
+			return err
+		}
+		_, err = tarFh.Write(n.Data)
+	} else {
+		if fh, e := os.Create(*dest + "/" + nm); e != nil {
+			return e
+		} else {
+			defer fh.Close()
+			_, err = fh.Write(n.Data)
+		}
+	}
+	if err == nil {
+		counter++
+	}
+	return
+}

weed-fs/src/cmd/weed/command.go

@@ -24,7 +24,6 @@ type Command struct {
 
 	// Flag is a set of flags specific to this command.
 	Flag flag.FlagSet
-
 }
 
 // Name returns the command's name: the first word in the usage line.

weed-fs/src/cmd/weed/shell.go

@@ -2,8 +2,8 @@ package main
 
 import (
 	"bufio"
-  "os"
 	"fmt"
+	"os"
 )
 
 func init() {
@@ -18,26 +18,25 @@ var cmdShell = &Command{
   `,
 }
 
-var (
-)
+var ()
 
 func runShell(command *Command, args []string) bool {
 	r := bufio.NewReader(os.Stdin)
 	o := bufio.NewWriter(os.Stdout)
 	e := bufio.NewWriter(os.Stderr)
-  prompt := func () { 
+	prompt := func() {
 		o.WriteString("> ")
 		o.Flush()
-  };
-  readLine := func () string { 
+	}
+	readLine := func() string {
 		ret, err := r.ReadString('\n')
 		if err != nil {
-       fmt.Fprint(e,err); 
+			fmt.Fprint(e, err)
 			os.Exit(1)
 		}
 		return ret
 	}
-  execCmd := func (cmd string) int { 
+	execCmd := func(cmd string) int {
 		if cmd != "" {
 			o.WriteString(cmd)
 		}

weed-fs/src/pkg/directory/file_id.go

@@ -3,8 +3,8 @@ package directory
 import (
 	"encoding/hex"
 	"pkg/storage"
-	"strings"
 	"pkg/util"
+	"strings"
 )
 
 type FileId struct {
@@ -16,7 +16,7 @@ type FileId struct {
 func NewFileId(VolumeId storage.VolumeId, Key uint64, Hashcode uint32) *FileId {
 	return &FileId{VolumeId: VolumeId, Key: Key, Hashcode: Hashcode}
 }
-func ParseFileId(fid string) *FileId{
+func ParseFileId(fid string) *FileId {
 	a := strings.Split(fid, ",")
 	if len(a) != 2 {
 		println("Invalid fid", fid, ", split length", len(a))

weed-fs/src/pkg/operation/delete_content.go

@@ -1,8 +1,8 @@
 package operation
 
 import (
-	"net/http"
 	"log"
+	"net/http"
 )
 
 func Delete(url string) error {

weed-fs/src/pkg/operation/lookup_volume_id.go

@@ -2,11 +2,11 @@ package operation
 
 import (
 	"encoding/json"
+	"errors"
+	_ "fmt"
 	"net/url"
 	"pkg/storage"
 	"pkg/util"
-  _ "fmt"
-  "errors"
 )
 
 type Location struct {
@@ -31,7 +31,7 @@ func Lookup(server string, vid storage.VolumeId) (*LookupResult, error) {
 	if err != nil {
 		return nil, err
 	}
-  if ret.Error != ""{
+	if ret.Error != "" {
 		return nil, errors.New(ret.Error)
 	}
 	return &ret, nil

weed-fs/src/pkg/operation/upload_content.go

@@ -3,13 +3,13 @@ package operation
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	_ "fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"mime/multipart"
 	"net/http"
-	"errors"
 )
 
 type UploadResult struct {
@@ -40,7 +40,7 @@ func Upload(uploadUrl string, filename string, reader io.Reader) (*UploadResult,
 		log.Println("failing to read upload resonse", uploadUrl, resp_body)
 		return nil, err
 	}
-	if ret.Error != ""{
+	if ret.Error != "" {
 		return nil, errors.New(ret.Error)
 	}
 	return &ret, nil

weed-fs/src/pkg/replication/volume_growth.go

@@ -182,7 +182,7 @@ func (vg *VolumeGrowth) GrowByCountAndType(count int, repType storage.Replicatio
 func (vg *VolumeGrowth) grow(topo *topology.Topology, vid storage.VolumeId, repType storage.ReplicationType, servers ...*topology.DataNode) error {
 	for _, server := range servers {
 		if err := operation.AllocateVolume(server, vid, repType); err == nil {
-			vi := storage.VolumeInfo{Id: vid, Size: 0, RepType: repType, Version:storage.CurrentVersion}
+			vi := storage.VolumeInfo{Id: vid, Size: 0, RepType: repType, Version: storage.CurrentVersion}
 			server.AddOrUpdateVolume(vi)
 			topo.RegisterVolumeLayout(&vi, server)
 			fmt.Println("Created Volume", vid, "on", server)

weed-fs/src/pkg/replication/volume_growth_test.go

@@ -80,7 +80,7 @@ func setup(topologyLayout string) *topology.Topology {
 	fmt.Println("data:", data)
 
 	//need to connect all nodes first before server adding volumes
-	topo := topology.NewTopology("mynetwork","/etc/weedfs/weedfs.conf","/tmp","testing",32*1024, 5)
+	topo := topology.NewTopology("mynetwork", "/etc/weedfs/weedfs.conf", "/tmp", "testing", 32*1024, 5)
 	mTopology := data.(map[string]interface{})
 	for dcKey, dcValue := range mTopology {
 		dc := topology.NewDataCenter(dcKey)
@@ -96,7 +96,7 @@ func setup(topologyLayout string) *topology.Topology {
 				rack.LinkChildNode(server)
 				for _, v := range serverMap["volumes"].([]interface{}) {
 					m := v.(map[string]interface{})
-					vi := storage.VolumeInfo{Id: storage.VolumeId(int64(m["id"].(float64))), Size: int64(m["size"].(float64)), Version:storage.CurrentVersion}
+					vi := storage.VolumeInfo{Id: storage.VolumeId(int64(m["id"].(float64))), Size: int64(m["size"].(float64)), Version: storage.CurrentVersion}
 					server.AddOrUpdateVolume(vi)
 				}
 				server.UpAdjustMaxVolumeCountDelta(int(serverMap["limit"].(float64)))
@@ -122,9 +122,8 @@ func TestRemoveDataCenter(t *testing.T) {
 func TestReserveOneVolume(t *testing.T) {
 	topo := setup(topologyLayout)
 	rand.Seed(time.Now().UnixNano())
-  vg:=&VolumeGrowth{copy1factor:3,copy2factor:2,copy3factor:1,copyAll:4}
-  if c, e := vg.GrowByCountAndType(1,storage.Copy000,topo);e==nil{
+	vg := &VolumeGrowth{copy1factor: 3, copy2factor: 2, copy3factor: 1, copyAll: 4}
+	if c, e := vg.GrowByCountAndType(1, storage.Copy000, topo); e == nil {
 		t.Log("reserved", c)
 	}
 }
-

weed-fs/src/pkg/sequence/sequence.go

@@ -2,10 +2,10 @@ package sequence
 
 import (
 	"encoding/gob"
+	"log"
 	"os"
 	"path"
 	"sync"
-	"log"
 )
 
 const (

weed-fs/src/pkg/storage/compact_map.go

@@ -175,3 +175,23 @@ func (cm *CompactMap) Peek() {
 		}
 	}
 }
+
+// iterate over the keys by calling iterate on each key till error is returned
+func (cm *CompactMap) Walk(pedestrian func(*NeedleValue) error) (err error) {
+	var i int
+	for _, cs := range cm.list {
+		for key := cs.start; key < cs.end; key++ {
+			if i = cs.binarySearchValues(key); i >= 0 {
+				if err = pedestrian(&cs.values[i]); err != nil {
+					return
+				}
+			}
+		}
+		for _, val := range cs.overflow {
+			if err = pedestrian(val); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

weed-fs/src/pkg/storage/compact_map_perf_test.go

@@ -1,10 +1,10 @@
 package storage
 
 import (
-	"testing"
 	"log"
 	"os"
 	"pkg/util"
+	"testing"
 )
 
 func TestMemoryUsage(t *testing.T) {

weed-fs/src/pkg/storage/compact_map_test.go

@@ -18,11 +18,11 @@ func TestXYZ(t *testing.T) {
 		m.Set(Key(i), i+11, i+5)
 	}
 
-//	for i := uint32(0); i < 100; i++ {
-//		if v := m.Get(Key(i)); v != nil {
-//			println(i, "=", v.Key, v.Offset, v.Size)
-//		}
-//	}
+	//	for i := uint32(0); i < 100; i++ {
+	//		if v := m.Get(Key(i)); v != nil {
+	//			println(i, "=", v.Key, v.Offset, v.Size)
+	//		}
+	//	}
 
 	for i := uint32(0); i < 10*batch; i++ {
 		v, ok := m.Get(Key(i))
@@ -51,7 +51,7 @@ func TestXYZ(t *testing.T) {
 				t.Fatal("key", i, "should have been deleted needle value", v)
 			}
 		} else if i%2 == 0 {
-		  if v==nil{
+			if v == nil {
 				t.Fatal("key", i, "missing")
 			}
 			if v.Size != i {

weed-fs/src/pkg/storage/needle_map.go

@@ -98,3 +98,8 @@ func (nm *NeedleMap) Close() {
 func (nm *NeedleMap) ContentSize() uint64 {
 	return nm.fileByteCounter
 }
+
+// iterate through all needles using the iterator function
+func (nm *NeedleMap) Walk(pedestrian func(*NeedleValue) error) (err error) {
+	return nm.m.Walk(pedestrian)
+}

weed-fs/src/pkg/storage/needle_read_write.go

@@ -2,10 +2,10 @@ package storage
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"os"
 	"pkg/util"
-	"fmt"
 )
 
 func (n *Needle) Append(w io.Writer, version Version) uint32 {
@@ -62,7 +62,8 @@ func (n *Needle) Append(w io.Writer, version Version) uint32 {
 	return n.Size
 }
 func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) {
-	if version == Version1 {
+	switch version {
+	case Version1:
 		bytes := make([]byte, NeedleHeaderSize+size+NeedleChecksumSize)
 		ret, e := r.Read(bytes)
 		n.readNeedleHeader(bytes)
@@ -72,7 +73,7 @@ func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) {
 			return 0, errors.New("CRC error! Data On Disk Corrupted!")
 		}
 		return ret, e
-	} else if version == Version2 {
+	case Version2:
 		if size == 0 {
 			return 0, nil
 		}
@@ -95,7 +96,7 @@ func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) {
 		}
 		return ret, e
 	}
-	return 0, errors.New("Unsupported Version!")
+	return 0, fmt.Errorf("Unsupported Version! (%d)", version)
 }
 func (n *Needle) readNeedleHeader(bytes []byte) {
 	n.Cookie = util.BytesToUint32(bytes[0:4])

weed-fs/src/pkg/storage/replication_type.go

@@ -32,7 +32,7 @@ func NewReplicationTypeFromString(t string) (ReplicationType, error) {
 	case "200":
 		return Copy200, nil
 	}
-  return Copy000, errors.New("Unknown Replication Type:"+t)
+	return Copy000, errors.New("Unknown Replication Type:" + t)
 }
 func NewReplicationTypeFromByte(b byte) (ReplicationType, error) {
 	switch b {
@@ -49,7 +49,7 @@ func NewReplicationTypeFromByte(b byte) (ReplicationType, error) {
 	case byte(200):
 		return Copy200, nil
 	}
-  return Copy000, errors.New("Unknown Replication Type:"+string(b))
+	return Copy000, errors.New("Unknown Replication Type:" + string(b))
 }
 
 func (r *ReplicationType) String() string {
@@ -87,7 +87,7 @@ func (r *ReplicationType) Byte() byte {
 	return byte(000)
 }
 
-func (repType ReplicationType)GetReplicationLevelIndex() int {
+func (repType ReplicationType) GetReplicationLevelIndex() int {
 	switch repType {
 	case Copy000:
 		return 0
@@ -104,7 +104,7 @@ func (repType ReplicationType)GetReplicationLevelIndex() int {
 	}
 	return -1
 }
-func (repType ReplicationType)GetCopyCount() int {
+func (repType ReplicationType) GetCopyCount() int {
 	switch repType {
 	case Copy000:
 		return 1

weed-fs/src/pkg/storage/store.go

@@ -65,13 +65,13 @@ func (s *Store) AddVolume(volumeListString string, replicationType string) error
 	}
 	return e
 }
-func (s *Store) addVolume(vid VolumeId, replicationType ReplicationType) error {
+func (s *Store) addVolume(vid VolumeId, replicationType ReplicationType) (err error) {
 	if s.volumes[vid] != nil {
 		return errors.New("Volume Id " + vid.String() + " already exists!")
 	}
 	log.Println("In dir", s.dir, "adds volume =", vid, ", replicationType =", replicationType)
-	s.volumes[vid] = NewVolume(s.dir, vid, replicationType)
-	return nil
+	s.volumes[vid], err = NewVolume(s.dir, vid, replicationType)
+	return err
 }
 
 func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) {
@@ -107,7 +107,7 @@ func (s *Store) loadExistingVolumes() {
 				base := name[:len(name)-len(".dat")]
 				if vid, err := NewVolumeId(base); err == nil {
 					if s.volumes[vid] == nil {
-						v := NewVolume(s.dir, vid, CopyNil)
+						if v, e := NewVolume(s.dir, vid, CopyNil); e == nil {
 							s.volumes[vid] = v
 							log.Println("In dir", s.dir, "read volume =", vid, "replicationType =", v.replicaType, "version =", v.version, "size =", v.Size())
 						}
@@ -115,6 +115,7 @@ func (s *Store) loadExistingVolumes() {
 				}
 			}
 		}
+	}
 }
 func (s *Store) Status() []*VolumeInfo {
 	var stats []*VolumeInfo

weed-fs/src/pkg/storage/volume.go

@@ -24,9 +24,9 @@ type Volume struct {
 	accessLock sync.Mutex
 }
 
-func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume) {
+func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume, e error) {
 	v = &Volume{dir: dirname, Id: id, replicaType: replicationType}
-	v.load()
+	e = v.load()
 	return
 }
 func (v *Volume) load() error {
@@ -43,6 +43,7 @@ func (v *Volume) load() error {
 	} else {
 		v.maybeWriteSuperBlock()
 	}
+	// TODO: if .idx not exists, but .cdb exists, then use (but don't load!) that
 	indexFile, ie := os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
 	if ie != nil {
 		return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
@@ -79,21 +80,23 @@ func (v *Volume) maybeWriteSuperBlock() {
 		v.dataFile.Write(header)
 	}
 }
-func (v *Volume) readSuperBlock() error {
+func (v *Volume) readSuperBlock() (err error) {
 	v.dataFile.Seek(0, 0)
 	header := make([]byte, SuperBlockSize)
 	if _, e := v.dataFile.Read(header); e != nil {
 		return fmt.Errorf("cannot read superblock: %s", e)
 	}
-	var err error
 	v.version, v.replicaType, err = ParseSuperBlock(header)
 	return err
 }
-func ParseSuperBlock(header []byte) (version Version, replicaType ReplicationType, e error) {
+func ParseSuperBlock(header []byte) (version Version, replicaType ReplicationType, err error) {
 	version = Version(header[0])
-	var err error
+	if version == 0 {
+		err = errors.New("Zero version impossible - bad superblock!")
+		return
+	}
 	if replicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
-		e = fmt.Errorf("cannot read replica type: %s", err)
+		err = fmt.Errorf("cannot read replica type: %s", err)
 	}
 	return
 }
@@ -221,3 +224,39 @@ func (v *Volume) copyDataAndGenerateIndexFile(srcName, dstName, idxName string)
 func (v *Volume) ContentSize() uint64 {
 	return v.nm.fileByteCounter
 }
+
+// Walk over the contained needles (call the function with each NeedleValue till error is returned)
+func (v *Volume) WalkValues(pedestrian func(*Needle) error) error {
+	pedplus := func(nv *NeedleValue) (err error) {
+		n := new(Needle)
+		if nv.Offset > 0 {
+			v.dataFile.Seek(int64(nv.Offset)*NeedlePaddingSize, 0)
+			if _, err = n.Read(v.dataFile, nv.Size, v.version); err != nil {
+				return
+			}
+			if err = pedestrian(n); err != nil {
+				return
+			}
+		}
+		return nil
+	}
+	return v.nm.Walk(pedplus)
+}
+
+// Walk over the keys
+func (v *Volume) WalkKeys(pedestrian func(Key) error) error {
+	pedplus := func(nv *NeedleValue) (err error) {
+		if nv.Offset > 0 && nv.Key > 0 {
+			if err = pedestrian(nv.Key); err != nil {
+				return
+			}
+		}
+		return nil
+	}
+	return v.nm.Walk(pedplus)
+}
+
+func (v *Volume) String() string {
+	return fmt.Sprintf("%d@%s:v%d:r%s", v.Id, v.dataFile.Name(),
+		v.Version(), v.replicaType)
+}

weed-fs/src/pkg/storage/volume_id.go

@@ -5,13 +5,14 @@ import (
 )
 
 type VolumeId uint32
-func NewVolumeId(vid string) (VolumeId,error) {
+
+func NewVolumeId(vid string) (VolumeId, error) {
 	volumeId, err := strconv.ParseUint(vid, 10, 64)
 	return VolumeId(volumeId), err
 }
-func (vid *VolumeId) String() string{
+func (vid *VolumeId) String() string {
 	return strconv.FormatUint(uint64(*vid), 10)
 }
-func (vid *VolumeId) Next() VolumeId{
-  return VolumeId(uint32(*vid)+1)
+func (vid *VolumeId) Next() VolumeId {
+	return VolumeId(uint32(*vid) + 1)
 }

weed-fs/src/pkg/storage/volume_version.go

@@ -1,7 +1,6 @@
 package storage
 
-import (
-)
+import ()
 
 type Version uint8
 

weed-fs/src/pkg/topology/configuration_test.go

@@ -32,11 +32,11 @@ func TestLoadConfiguration(t *testing.T) {
 	c, err := NewConfiguration([]byte(confContent))
 
 	fmt.Printf("%s\n", c)
-	if err!=nil{
-	  t.Fatalf("unmarshal error:%s",err.Error())
+	if err != nil {
+		t.Fatalf("unmarshal error:%s", err.Error())
 	}
 
 	if len(c.Topo.DataCenters) <= 0 || c.Topo.DataCenters[0].Name != "dc1" {
-    t.Fatalf("unmarshal error:%s",c)
+		t.Fatalf("unmarshal error:%s", c)
 	}
 }

weed-fs/src/pkg/topology/data_center.go

@@ -1,7 +1,6 @@
 package topology
 
-import (
-)
+import ()
 
 type DataCenter struct {
 	NodeImpl
@@ -28,7 +27,7 @@ func (dc *DataCenter) GetOrCreateRack(rackName string) *Rack {
 	return rack
 }
 
-func (dc *DataCenter) ToMap() interface{}{
+func (dc *DataCenter) ToMap() interface{} {
 	m := make(map[string]interface{})
 	m["Max"] = dc.GetMaxVolumeCount()
 	m["Free"] = dc.FreeSpace()

weed-fs/src/pkg/topology/node_list.go

@@ -37,8 +37,8 @@ func (nl *NodeList) RandomlyPickN(n int, min int) ([]Node, bool) {
 			list = append(list, n)
 		}
 	}
-	if n > len(list){
-	  return nil,false
+	if n > len(list) {
+		return nil, false
 	}
 	for i := n; i > 0; i-- {
 		r := rand.Intn(i)

weed-fs/src/pkg/topology/node_list_test.go

@@ -1,38 +1,38 @@
 package topology
 
 import (
+	_ "fmt"
 	"strconv"
 	"testing"
-  _ "fmt"
 )
 
 func TestXYZ(t *testing.T) {
-	topo := NewTopology("topo","/etc/weed.conf", "/tmp","test",234,5)
+	topo := NewTopology("topo", "/etc/weed.conf", "/tmp", "test", 234, 5)
 	for i := 0; i < 5; i++ {
 		dc := NewDataCenter("dc" + strconv.Itoa(i))
 		dc.activeVolumeCount = i
 		dc.maxVolumeCount = 5
 		topo.LinkChildNode(dc)
 	}
-	nl := NewNodeList(topo.Children(),nil)
+	nl := NewNodeList(topo.Children(), nil)
 
 	picked, ret := nl.RandomlyPickN(1)
-  if !ret || len(picked)!=1 {
+	if !ret || len(picked) != 1 {
 		t.Errorf("need to randomly pick 1 node")
 	}
 
 	picked, ret = nl.RandomlyPickN(4)
-	if !ret || len(picked)!=4 {
+	if !ret || len(picked) != 4 {
 		t.Errorf("need to randomly pick 4 nodes")
 	}
 
 	picked, ret = nl.RandomlyPickN(5)
-  if !ret || len(picked)!=5 {
+	if !ret || len(picked) != 5 {
 		t.Errorf("need to randomly pick 5 nodes")
 	}
 
 	picked, ret = nl.RandomlyPickN(6)
-  if ret || len(picked)!=0 {
+	if ret || len(picked) != 0 {
 		t.Errorf("can not randomly pick 6 nodes:", ret, picked)
 	}
 

weed-fs/src/pkg/topology/topo_test.go

@@ -78,7 +78,7 @@ func setup(topologyLayout string) *Topology {
 	}
 
 	//need to connect all nodes first before server adding volumes
-	topo := NewTopology("mynetwork","/etc/weed.conf","/tmp","test",234,5)
+	topo := NewTopology("mynetwork", "/etc/weed.conf", "/tmp", "test", 234, 5)
 	mTopology := data.(map[string]interface{})
 	for dcKey, dcValue := range mTopology {
 		dc := NewDataCenter(dcKey)
@@ -94,7 +94,7 @@ func setup(topologyLayout string) *Topology {
 				rack.LinkChildNode(server)
 				for _, v := range serverMap["volumes"].([]interface{}) {
 					m := v.(map[string]interface{})
-					vi := storage.VolumeInfo{Id: storage.VolumeId(int64(m["id"].(float64))), Size: int64(m["size"].(float64)), Version:storage.CurrentVersion}
+					vi := storage.VolumeInfo{Id: storage.VolumeId(int64(m["id"].(float64))), Size: int64(m["size"].(float64)), Version: storage.CurrentVersion}
 					server.AddOrUpdateVolume(vi)
 				}
 				server.UpAdjustMaxVolumeCountDelta(int(serverMap["limit"].(float64)))
@@ -122,6 +122,6 @@ func TestReserveOneVolume(t *testing.T) {
 	rand.Seed(time.Now().UnixNano())
 	rand.Seed(1)
 	ret, node, vid := topo.RandomlyReserveOneVolume()
-  fmt.Println("assigned :", ret, ", node :", node,", volume id:", vid)
+	fmt.Println("assigned :", ret, ", node :", node, ", volume id:", vid)
 
 }

weed-fs/src/pkg/topology/topology_compact.go

@@ -104,7 +104,7 @@ func vacuumVolume_Check(urlLocation string, vid storage.VolumeId, garbageThresho
 	values.Add("garbageThreshold", garbageThreshold)
 	jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_check", values)
 	if err != nil {
-	    fmt.Println("parameters:",values)
+		fmt.Println("parameters:", values)
 		return err, false
 	}
 	var ret VacuumVolumeResult

weed-fs/src/pkg/topology/topology_map.go

@@ -1,7 +1,6 @@
 package topology
 
-import (
-)
+import ()
 
 func (t *Topology) ToMap() interface{} {
 	m := make(map[string]interface{})

weed-fs/src/pkg/topology/volume_location.go

@@ -31,7 +31,7 @@ func (dnll *VolumeLocationList) Add(loc *DataNode) bool {
 func (dnll *VolumeLocationList) Remove(loc *DataNode) bool {
 	for i, dnl := range dnll.list {
 		if loc.Ip == dnl.Ip && loc.Port == dnl.Port {
-      dnll.list = append(dnll.list[:i],dnll.list[i+1:]...)
+			dnll.list = append(dnll.list[:i], dnll.list[i+1:]...)
 			return true
 		}
 	}

weed-fs/src/pkg/util/bytes.go

@@ -1,34 +1,33 @@
 package util
 
-func BytesToUint64(b []byte)(v uint64){
+func BytesToUint64(b []byte) (v uint64) {
 	length := uint(len(b))
-  for i :=uint(0);i<length-1;i++ {
+	for i := uint(0); i < length-1; i++ {
 		v += uint64(b[i])
 		v <<= 8
 	}
-  v+=uint64(b[length-1])
+	v += uint64(b[length-1])
 	return
 }
-func BytesToUint32(b []byte)(v uint32){
+func BytesToUint32(b []byte) (v uint32) {
 	length := uint(len(b))
-  for i :=uint(0);i<length-1;i++ {
+	for i := uint(0); i < length-1; i++ {
 		v += uint32(b[i])
 		v <<= 8
 	}
-  v+=uint32(b[length-1])
+	v += uint32(b[length-1])
 	return
 }
-func Uint64toBytes(b []byte, v uint64){
-  for i :=uint(0);i<8;i++ {
-    b[7-i] = byte(v>>(i*8))
+func Uint64toBytes(b []byte, v uint64) {
+	for i := uint(0); i < 8; i++ {
+		b[7-i] = byte(v >> (i * 8))
 	}
 }
-func Uint32toBytes(b []byte, v uint32){
-  for i :=uint(0);i<4;i++ {
-    b[3-i] = byte(v>>(i*8))
+func Uint32toBytes(b []byte, v uint32) {
+	for i := uint(0); i < 4; i++ {
+		b[3-i] = byte(v >> (i * 8))
 	}
 }
-func Uint8toBytes(b []byte, v uint8){
+func Uint8toBytes(b []byte, v uint8) {
 	b[0] = byte(v)
 }
-

weed-fs/src/pkg/util/parse.go

@@ -4,10 +4,10 @@ import (
 	"strconv"
 )
 
-func ParseInt(text string, defaultValue int) int{
-  count, parseError := strconv.ParseUint(text,10,64)
-  if parseError!=nil {
-    if len(text)>0{
+func ParseInt(text string, defaultValue int) int {
+	count, parseError := strconv.ParseUint(text, 10, 64)
+	if parseError != nil {
+		if len(text) > 0 {
 			return 0
 		}
 		return defaultValue