compatible with Go1

git-svn-id: https://weed-fs.googlecode.com/svn/trunk@46 282b0af5-e82d-9cf1-ede4-77906d7719d0

weed-fs/build

@@ -1,19 +0,0 @@
-# Build script generated by gb: http://go-gb.googlecode.com
-# gb provides configuration-free building and distributing
-
-echo "Build script generated by gb: http://go-gb.googlecode.com" 
-if [ "$1" = "goinstall" ]; then
-echo Running goinstall \
-
-else
-echo Building \
-&& echo "(in src/pkg/util)" gomake $1 && cd src/pkg/util && gomake $1 && cd - > /dev/null \
-&& echo "(in src/pkg/storage)" gomake $1 && cd src/pkg/storage && gomake $1 && cd - > /dev/null \
-&& echo "(in src/cmd/weedvolume/fix_volume_index)" gomake $1 && cd src/cmd/weedvolume/fix_volume_index && gomake $1 && cd - > /dev/null \
-&& echo "(in src/cmd/weedvolume)" gomake $1 && cd src/cmd/weedvolume && gomake $1 && cd - > /dev/null \
-&& echo "(in src/pkg/directory)" gomake $1 && cd src/pkg/directory && gomake $1 && cd - > /dev/null \
-&& echo "(in src/cmd/weedmaster)" gomake $1 && cd src/cmd/weedmaster && gomake $1 && cd - > /dev/null \
-
-fi
-
-# The makefiles above are invoked in topological dependence order

weed-fs/src/cmd/weedmaster/Makefile

@@ -1,39 +0,0 @@
-# Makefile generated by gb: http://go-gb.googlecode.com
-# gb provides configuration-free building and distributing
-
-include $(GOROOT)/src/Make.inc
-
-TARG=weedmaster
-GOFILES=\
-	weedmaster.go\
-
-# gb: this is the local install
-GBROOT=../../..
-
-# gb: compile/link against local install
-GCIMPORTS+= -I $(GBROOT)/_obj
-LDIMPORTS+= -L $(GBROOT)/_obj
-
-# gb: compile/link against GOPATH entries
-GOPATHSEP=:
-ifeq ($(GOHOSTOS),windows)
-GOPATHSEP=;
-endif
-GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
-LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
-
-# gb: default target is in GBROOT this way
-command:
-
-include $(GOROOT)/src/Make.cmd
-
-# gb: copy to local install
-$(GBROOT)/bin/$(TARG): $(TARG)
-	mkdir -p $(dir $@); cp -f $< $@
-command: $(GBROOT)/bin/$(TARG)
-
-# gb: local dependencies
-$(TARG): $(GBROOT)/_obj/storage.a
-
-$(TARG): $(GBROOT)/_obj/directory.a
-

weed-fs/src/cmd/weedmaster/weedmaster.go

@@ -1,90 +0,0 @@
-package main
-
-import (
-	"storage"
-	"directory"
-	"flag"
-	"fmt"
-	"http"
-	"json"
-	"log"
-	"strconv"
-	"strings"
-)
-
-var (
-	port       = flag.Int("port", 9333, "http listen port")
-	metaFolder = flag.String("mdir", "/tmp", "data directory to store mappings")
-	capacity   = flag.Int("capacity", 100, "maximum number of volumes to hold")
-	mapper     *directory.Mapper
-	IsDebug    = flag.Bool("debug", false, "verbose debug information")
-	volumeSizeLimitMB = flag.Uint("volumeSizeLimitMB", 32*1024, "Default Volume Size in MegaBytes")
-)
-
-func dirLookupHandler(w http.ResponseWriter, r *http.Request) {
-	vid := r.FormValue("volumeId")
-	commaSep := strings.Index(vid, ",")
-	if commaSep > 0 {
-		vid = vid[0:commaSep]
-	}
-	volumeId, _ := strconv.Atoui64(vid)
-	machine, e := mapper.Get(uint32(volumeId))
-	if e == nil {
-		writeJson(w, r, machine.Server)
-	} else {
-		log.Println("Invalid volume id", volumeId)
-		writeJson(w, r, map[string]string{"error": "volume id " + strconv.Uitoa64(volumeId) + " not found"})
-	}
-}
-func dirAssignHandler(w http.ResponseWriter, r *http.Request) {
-	fid, machine, err := mapper.PickForWrite()
-	if err == nil {
-		writeJson(w, r, map[string]string{"fid": fid, "url": machine.Url})
-	} else {
-		log.Println(err)
-		writeJson(w, r, map[string]string{"error": err.String()})
-	}
-}
-func dirJoinHandler(w http.ResponseWriter, r *http.Request) {
-	s := r.RemoteAddr[0:strings.Index(r.RemoteAddr, ":")+1] + r.FormValue("port")
-	publicUrl := r.FormValue("publicUrl")
-	volumes := new([]storage.VolumeInfo)
-	json.Unmarshal([]byte(r.FormValue("volumes")), volumes)
-	if *IsDebug {
-		log.Println(s, "volumes", r.FormValue("volumes"))
-	}
-	mapper.Add(*directory.NewMachine(s, publicUrl, *volumes))
-}
-func dirStatusHandler(w http.ResponseWriter, r *http.Request) {
-	writeJson(w, r, mapper)
-}
-func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) {
-	w.Header().Set("Content-Type", "application/javascript")
-	bytes, _ := json.Marshal(obj)
-	callback := r.FormValue("callback")
-	if callback == "" {
-		w.Write(bytes)
-	} else {
-		w.Write([]uint8(callback))
-		w.Write([]uint8("("))
-		fmt.Fprint(w, string(bytes))
-		w.Write([]uint8(")"))
-	}
-}
-
-func main() {
-	flag.Parse()
-    log.Println("Volume Size Limit is", *volumeSizeLimitMB, "MB")
-	mapper = directory.NewMapper(*metaFolder, "directory", uint64(*volumeSizeLimitMB)*1024*1024)
-	http.HandleFunc("/dir/assign", dirAssignHandler)
-	http.HandleFunc("/dir/lookup", dirLookupHandler)
-	http.HandleFunc("/dir/join", dirJoinHandler)
-	http.HandleFunc("/dir/status", dirStatusHandler)
-
-	log.Println("Start directory service at http://127.0.0.1:" + strconv.Itoa(*port))
-	e := http.ListenAndServe(":"+strconv.Itoa(*port), nil)
-	if e != nil {
-		log.Fatalf("Fail to start:", e.String())
-	}
-
-}

weed-fs/src/cmd/weedvolume/Makefile

@@ -1,37 +0,0 @@
-# Makefile generated by gb: http://go-gb.googlecode.com
-# gb provides configuration-free building and distributing
-
-include $(GOROOT)/src/Make.inc
-
-TARG=weedvolume
-GOFILES=\
-	weedvolume.go\
-
-# gb: this is the local install
-GBROOT=../../..
-
-# gb: compile/link against local install
-GCIMPORTS+= -I $(GBROOT)/_obj
-LDIMPORTS+= -L $(GBROOT)/_obj
-
-# gb: compile/link against GOPATH entries
-GOPATHSEP=:
-ifeq ($(GOHOSTOS),windows)
-GOPATHSEP=;
-endif
-GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
-LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
-
-# gb: default target is in GBROOT this way
-command:
-
-include $(GOROOT)/src/Make.cmd
-
-# gb: copy to local install
-$(GBROOT)/bin/$(TARG): $(TARG)
-	mkdir -p $(dir $@); cp -f $< $@
-command: $(GBROOT)/bin/$(TARG)
-
-# gb: local dependencies
-$(TARG): $(GBROOT)/_obj/storage.a
-

weed-fs/src/cmd/weedvolume/fix_volume_index/Makefile

@@ -1,37 +0,0 @@
-# Makefile generated by gb: http://go-gb.googlecode.com
-# gb provides configuration-free building and distributing
-
-include $(GOROOT)/src/Make.inc
-
-TARG=fix_volume_index
-GOFILES=\
-	fix_volume_index.go\
-
-# gb: this is the local install
-GBROOT=../../../..
-
-# gb: compile/link against local install
-GCIMPORTS+= -I $(GBROOT)/_obj
-LDIMPORTS+= -L $(GBROOT)/_obj
-
-# gb: compile/link against GOPATH entries
-GOPATHSEP=:
-ifeq ($(GOHOSTOS),windows)
-GOPATHSEP=;
-endif
-GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
-LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
-
-# gb: default target is in GBROOT this way
-command:
-
-include $(GOROOT)/src/Make.cmd
-
-# gb: copy to local install
-$(GBROOT)/bin/$(TARG): $(TARG)
-	mkdir -p $(dir $@); cp -f $< $@
-command: $(GBROOT)/bin/$(TARG)
-
-# gb: local dependencies
-$(TARG): $(GBROOT)/_obj/storage.a
-

weed-fs/src/cmd/weedvolume/fix_volume_index/fix_volume_index.go

@@ -1,59 +0,0 @@
-package main
-
-import (
-	"storage"
-	"flag"
-	"log"
-	"os"
-	"path"
-	"strconv"
-)
-
-var (
-	dir      = flag.String("dir", "/tmp", "data directory to store files")
-	volumeId = flag.Int("volumeId", -1, "a non-negative volume id. The volume should already exist in the dir. The volume index file should not exist.")
-	IsDebug  = flag.Bool("debug", false, "enable debug mode")
-
-	store *storage.Store
-)
-
-func main() {
-	flag.Parse()
-
-	if *volumeId == -1 {
-		flag.Usage()
-		return
-	}
-
-	fileName := strconv.Itoa(*volumeId)
-	dataFile, e := os.OpenFile(path.Join(*dir, fileName+".dat"), os.O_RDONLY, 0644)
-	if e != nil {
-		log.Fatalf("Read Volume [ERROR] %s\n", e)
-	}
-	defer dataFile.Close()
-	indexFile, ie := os.OpenFile(path.Join(*dir, fileName+".idx"), os.O_WRONLY|os.O_CREATE, 0644)
-	if ie != nil {
-		log.Fatalf("Create Volume Index [ERROR] %s\n", ie)
-	}
-	defer indexFile.Close()
-
-	//skip the volume super block
-	dataFile.Seek(storage.SuperBlockSize, 0)
-
-	n, length := storage.ReadNeedle(dataFile)
-	nm := storage.NewNeedleMap(indexFile)
-	offset := uint32(storage.SuperBlockSize)
-	for n != nil {
-		if *IsDebug {
-			log.Println("key", n.Key, "volume offset", offset, "data_size", n.Size, "length", length)
-		}
-		if n.Size > 0 {
-			count, pe := nm.Put(n.Key, offset/8, n.Size)
-			if *IsDebug {
-				log.Println("saved", count, "with error", pe)
-			}
-		}
-		offset += length
-		n, length = storage.ReadNeedle(dataFile)
-	}
-}

weed-fs/src/cmd/weedvolume/weedvolume.go

@@ -1,168 +0,0 @@
-package main
-
-import (
-	"storage"
-	"flag"
-	"fmt"
-	"http"
-	"json"
-	"log"
-	"mime"
-	"rand"
-	"strconv"
-	"strings"
-	"time"
-)
-
-var (
-	port        = flag.Int("port", 8080, "http listen port")
-	chunkFolder = flag.String("dir", "/tmp", "data directory to store files")
-	volumes     = flag.String("volumes", "0,1-3,4", "comma-separated list of volume ids or range of ids")
-	publicUrl   = flag.String("publicUrl", "localhost:8080", "public url to serve data read")
-	metaServer  = flag.String("mserver", "localhost:9333", "master directory server to store mappings")
-	IsDebug     = flag.Bool("debug", false, "enable debug mode")
-	pulse       = flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
-
-	store *storage.Store
-)
-
-func statusHandler(w http.ResponseWriter, r *http.Request) {
-	writeJson(w, r, store.Status())
-}
-func addVolumeHandler(w http.ResponseWriter, r *http.Request) {
-	store.AddVolume(r.FormValue("volume"))
-	writeJson(w, r, store.Status())
-}
-func storeHandler(w http.ResponseWriter, r *http.Request) {
-	switch r.Method {
-	case "GET":
-		GetHandler(w, r)
-	case "DELETE":
-		DeleteHandler(w, r)
-	case "POST":
-		PostHandler(w, r)
-	}
-}
-func GetHandler(w http.ResponseWriter, r *http.Request) {
-	n := new(storage.Needle)
-	vid, fid, ext := parseURLPath(r.URL.Path)
-	volumeId, _ := strconv.Atoui64(vid)
-	n.ParsePath(fid)
-
-	if *IsDebug {
-		log.Println("volume", volumeId, "reading", n)
-	}
-	cookie := n.Cookie
-	count, e := store.Read(volumeId, n)
-	if *IsDebug {
-		log.Println("read bytes", count, "error", e)
-	}
-	if n.Cookie != cookie {
-		log.Println("request with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
-		return
-	}
-	if ext != "" {
-		w.Header().Set("Content-Type", mime.TypeByExtension(ext))
-	}
-	w.Write(n.Data)
-}
-func PostHandler(w http.ResponseWriter, r *http.Request) {
-	vid, _, _ := parseURLPath(r.URL.Path)
-	volumeId, e := strconv.Atoui64(vid)
-	if e != nil {
-		writeJson(w, r, e)
-	} else {
-		ret := store.Write(volumeId, storage.NewNeedle(r))
-		m := make(map[string]uint32)
-		m["size"] = ret
-		writeJson(w, r, m)
-	}
-}
-func DeleteHandler(w http.ResponseWriter, r *http.Request) {
-	n := new(storage.Needle)
-	vid, fid, _ := parseURLPath(r.URL.Path)
-	volumeId, _ := strconv.Atoui64(vid)
-	n.ParsePath(fid)
-
-	if *IsDebug {
-		log.Println("deleting", n)
-	}
-
-	cookie := n.Cookie
-	count, ok := store.Read(volumeId, n)
-
-	if ok!=nil {
-		m := make(map[string]uint32)
-		m["size"] = 0
-		writeJson(w, r, m)
-		return
-	}
-
-	if n.Cookie != cookie {
-		log.Println("delete with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
-		return
-	}
-
-	n.Size = 0
-	store.Delete(volumeId, n)
-	m := make(map[string]uint32)
-	m["size"] = uint32(count)
-	writeJson(w, r, m)
-}
-func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) {
-	w.Header().Set("Content-Type", "application/javascript")
-	bytes, _ := json.Marshal(obj)
-	callback := r.FormValue("callback")
-	if callback == "" {
-		w.Write(bytes)
-	} else {
-		w.Write([]uint8(callback))
-		w.Write([]uint8("("))
-		fmt.Fprint(w, string(bytes))
-		w.Write([]uint8(")"))
-	}
-	//log.Println("JSON Response", string(bytes))
-}
-func parseURLPath(path string) (vid, fid, ext string) {
-	sepIndex := strings.LastIndex(path, "/")
-	commaIndex := strings.LastIndex(path[sepIndex:], ",")
-	if commaIndex <= 0 {
-		log.Println("unknown file id", path[sepIndex+1:])
-		return
-	}
-	dotIndex := strings.LastIndex(path[sepIndex:], ".")
-	vid = path[sepIndex+1 : commaIndex]
-	fid = path[commaIndex+1:]
-	ext = ""
-	if dotIndex > 0 {
-		fid = path[commaIndex+1 : dotIndex]
-		ext = path[dotIndex+1:]
-	}
-	return
-}
-
-func main() {
-	flag.Parse()
-	//TODO: now default to 1G, this value should come from server?
-	store = storage.NewStore(*port, *publicUrl, *chunkFolder, *volumes)
-	defer store.Close()
-	http.HandleFunc("/", storeHandler)
-	http.HandleFunc("/status", statusHandler)
-	http.HandleFunc("/add_volume", addVolumeHandler)
-
-	go func() {
-		for {
-			store.Join(*metaServer)
-			ns := int64(*pulse) * 1e9
-			time.Sleep(ns + rand.Int63()%ns)
-		}
-	}()
-	log.Println("store joined at", *metaServer)
-
-	log.Println("Start storage service at http://127.0.0.1:"+strconv.Itoa(*port), "public url", *publicUrl)
-	e := http.ListenAndServe(":"+strconv.Itoa(*port), nil)
-	if e != nil {
-		log.Fatalf("Fail to start:", e.String())
-	}
-
-}

weed-fs/src/pkg/directory/Makefile

@@ -1,36 +0,0 @@
-# Makefile generated by gb: http://go-gb.googlecode.com
-# gb provides configuration-free building and distributing
-
-include $(GOROOT)/src/Make.inc
-
-TARG=directory
-GOFILES=\
-	file_id.go\
-	volume_mapping.go\
-
-# gb: this is the local install
-GBROOT=../../..
-
-# gb: compile/link against local install
-GCIMPORTS+= -I $(GBROOT)/_obj
-LDIMPORTS+= -L $(GBROOT)/_obj
-
-# gb: compile/link against GOPATH entries
-GOPATHSEP=:
-ifeq ($(GOHOSTOS),windows)
-GOPATHSEP=;
-endif
-GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
-LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
-
-# gb: copy to local install
-$(GBROOT)/_obj/$(TARG).a: _obj/$(TARG).a
-	mkdir -p $(dir $@); cp -f $< $@
-
-package: $(GBROOT)/_obj/$(TARG).a
-
-include $(GOROOT)/src/Make.pkg
-
-# gb: local dependencies
-_obj/$(TARG).a: $(GBROOT)/_obj/storage.a
-_obj/$(TARG).a: $(GBROOT)/_obj/util.a

weed-fs/src/pkg/directory/file_id.go

@@ -3,10 +3,10 @@ package directory
 import (
 	"encoding/hex"
 	"log"
-	"storage"
+	"pkg/storage"
 	"strconv"
 	"strings"
-	"util"
+	"pkg/util"
 )
 
 type FileId struct {
@@ -18,14 +18,14 @@ type FileId struct {
 func NewFileId(VolumeId uint32, Key uint64, Hashcode uint32) *FileId {
 	return &FileId{VolumeId: VolumeId, Key: Key, Hashcode: Hashcode}
 }
-func ParseFileId(fid string) *FileId {
+func ParseFileId(fid string) *FileId{
 	a := strings.Split(fid, ",")
 	if len(a) != 2 {
-        log.Println("Invalid fid", fid, ", split length", len(a))
+		log.Println("Invalid fid", fid, ", split length", len(a))
 		return nil
 	}
 	vid_string, key_hash_string := a[0], a[1]
-	vid, _ := strconv.Atoui64(vid_string)
+	vid, _ := strconv.ParseUint(vid_string, 10, 64)
 	key, hash := storage.ParseKeyHash(key_hash_string)
 	return &FileId{VolumeId: uint32(vid), Key: key, Hashcode: hash}
 }
@@ -36,5 +36,5 @@ func (n *FileId) String() string {
 	nonzero_index := 0
 	for ; bytes[nonzero_index] == 0; nonzero_index++ {
 	}
-	return strconv.Uitoa64(uint64(n.VolumeId)) + "," + hex.EncodeToString(bytes[nonzero_index:])
+	return strconv.FormatUint(uint64(n.VolumeId), 10) + "," + hex.EncodeToString(bytes[nonzero_index:])
 }

weed-fs/src/pkg/directory/volume_mapping.go

@@ -1,12 +1,13 @@
 package directory
 
 import (
-	"gob"
+	"encoding/gob"
+	"errors"
+	"log"
+	"math/rand"
 	"os"
 	"path"
-	"rand"
-	"log"
-	"storage"
+	"pkg/storage"
 	"strconv"
 	"sync"
 )
@@ -31,7 +32,7 @@ type Mapper struct {
 	lock          sync.Mutex
 	Machines      []*Machine
 	vid2machineId map[uint32]int //machineId is +1 of the index of []*Machine, to detect not found entries
-	Writers       []uint32 // transient array of Writers volume id
+	Writers       []uint32       // transient array of Writers volume id
 
 	FileIdSequence uint64
 	fileIdCounter  uint64
@@ -64,11 +65,11 @@ func NewMapper(dirname string, filename string, volumeSizeLimit uint64) (m *Mapp
 	}
 	return
 }
-func (m *Mapper) PickForWrite() (string, MachineInfo, os.Error) {
+func (m *Mapper) PickForWrite() (string, MachineInfo, error) {
 	len_writers := len(m.Writers)
 	if len_writers <= 0 {
 		log.Println("No more writable volumes!")
-		return "", m.Machines[rand.Intn(len(m.Machines))].Server, os.NewError("No more writable volumes!")
+		return "", m.Machines[rand.Intn(len(m.Machines))].Server, errors.New("No more writable volumes!")
 	}
 	vid := m.Writers[rand.Intn(len_writers)]
 	machine_id := m.vid2machineId[vid]
@@ -76,7 +77,7 @@ func (m *Mapper) PickForWrite() (string, MachineInfo, os.Error) {
 		machine := m.Machines[machine_id-1]
 		return NewFileId(vid, m.NextFileId(), rand.Uint32()).String(), machine.Server, nil
 	}
-	return "", m.Machines[rand.Intn(len(m.Machines))].Server, os.NewError("Strangely vid " + strconv.Uitoa64(uint64(vid)) + " is on no machine!")
+	return "", m.Machines[rand.Intn(len(m.Machines))].Server, errors.New("Strangely vid " + strconv.FormatUint(uint64(vid), 10) + " is on no machine!")
 }
 func (m *Mapper) NextFileId() uint64 {
 	if m.fileIdCounter <= 0 {
@@ -87,16 +88,16 @@ func (m *Mapper) NextFileId() uint64 {
 	m.fileIdCounter--
 	return m.FileIdSequence - m.fileIdCounter
 }
-func (m *Mapper) Get(vid uint32) (*Machine, os.Error) {
+func (m *Mapper) Get(vid uint32) (*Machine, error) {
 	machineId := m.vid2machineId[vid]
 	if machineId <= 0 {
-		return nil, os.NewError("invalid volume id " + strconv.Uitob64(uint64(vid), 10))
+		return nil, errors.New("invalid volume id " + strconv.FormatUint(uint64(vid), 10))
 	}
 	return m.Machines[machineId-1], nil
 }
 func (m *Mapper) Add(machine Machine) {
 	//check existing machine, linearly
-    //log.Println("Adding machine", machine.Server.Url)
+	//log.Println("Adding machine", machine.Server.Url)
 	m.lock.Lock()
 	foundExistingMachineId := -1
 	for index, entry := range m.Machines {

weed-fs/src/pkg/storage/Makefile

@@ -1,37 +0,0 @@
-# Makefile generated by gb: http://go-gb.googlecode.com
-# gb provides configuration-free building and distributing
-
-include $(GOROOT)/src/Make.inc
-
-TARG=storage
-GOFILES=\
-	needle.go\
-	needle_map.go\
-	store.go\
-	volume.go\
-
-# gb: this is the local install
-GBROOT=../../..
-
-# gb: compile/link against local install
-GCIMPORTS+= -I $(GBROOT)/_obj
-LDIMPORTS+= -L $(GBROOT)/_obj
-
-# gb: compile/link against GOPATH entries
-GOPATHSEP=:
-ifeq ($(GOHOSTOS),windows)
-GOPATHSEP=;
-endif
-GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
-LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
-
-# gb: copy to local install
-$(GBROOT)/_obj/$(TARG).a: _obj/$(TARG).a
-	mkdir -p $(dir $@); cp -f $< $@
-
-package: $(GBROOT)/_obj/$(TARG).a
-
-include $(GOROOT)/src/Make.pkg
-
-# gb: local dependencies
-_obj/$(TARG).a: $(GBROOT)/_obj/util.a

weed-fs/src/pkg/storage/needle.go

@@ -1,14 +1,14 @@
 package storage
 
 import (
-    "encoding/hex"
+	"encoding/hex"
 	"io"
 	"io/ioutil"
-	"http"
 	"log"
+	"net/http"
 	"os"
 	"strings"
-    . "util"
+  "pkg/util"
 )
 
 type Needle struct {
@@ -21,6 +21,7 @@ type Needle struct {
 }
 
 func NewNeedle(r *http.Request) (n *Needle) {
+
 	n = new(Needle)
 	form, fe := r.MultipartReader()
 	if fe != nil {
@@ -30,11 +31,11 @@ func NewNeedle(r *http.Request) (n *Needle) {
 	data, _ := ioutil.ReadAll(part)
 	n.Data = data
 
-    commaSep := strings.LastIndex(r.URL.Path, ",")
+	commaSep := strings.LastIndex(r.URL.Path, ",")
 	dotSep := strings.LastIndex(r.URL.Path, ".")
 	fid := r.URL.Path[commaSep+1:]
 	if dotSep > 0 {
-		fid = r.URL.Path[commaSep+1:dotSep]
+		fid = r.URL.Path[commaSep+1 : dotSep]
 	}
 
 	n.ParsePath(fid)
@@ -49,51 +50,51 @@ func (n *Needle) ParsePath(fid string) {
 	}
 	n.Key, n.Cookie = ParseKeyHash(fid)
 }
-func (n *Needle) Append(w io.Writer) (uint32){
+func (n *Needle) Append(w io.Writer) uint32 {
 	header := make([]byte, 16)
-	Uint32toBytes(header[0:4], n.Cookie)
-	Uint64toBytes(header[4:12], n.Key)
+	util.Uint32toBytes(header[0:4], n.Cookie)
+	util.Uint64toBytes(header[4:12], n.Key)
 	n.Size = uint32(len(n.Data))
-	Uint32toBytes(header[12:16], n.Size)
+	util.Uint32toBytes(header[12:16], n.Size)
 	w.Write(header)
 	w.Write(n.Data)
 	rest := 8 - ((n.Size + 16 + 4) % 8)
-	Uint32toBytes(header[0:4], uint32(n.Checksum))
+	util.Uint32toBytes(header[0:4], uint32(n.Checksum))
 	w.Write(header[0 : rest+4])
 	return n.Size
 }
-func (n *Needle) Read(r io.Reader, size uint32)(int, os.Error) {
+func (n *Needle) Read(r io.Reader, size uint32) (int, error) {
 	bytes := make([]byte, size+16+4)
 	ret, e := r.Read(bytes)
-	n.Cookie = BytesToUint32(bytes[0:4])
-	n.Key = BytesToUint64(bytes[4:12])
-	n.Size = BytesToUint32(bytes[12:16])
+	n.Cookie = util.BytesToUint32(bytes[0:4])
+	n.Key = util.BytesToUint64(bytes[4:12])
+	n.Size = util.BytesToUint32(bytes[12:16])
 	n.Data = bytes[16 : 16+size]
-	n.Checksum = int32(BytesToUint32(bytes[16+size : 16+size+4]))
+	n.Checksum = int32(util.BytesToUint32(bytes[16+size : 16+size+4]))
 	return ret, e
 }
 func ReadNeedle(r *os.File) (*Needle, uint32) {
-    n := new(Needle)
-    bytes := make([]byte, 16)
-    count, e := r.Read(bytes)
-    if count<=0 || e!=nil {
-        return nil, 0
-    }
-    n.Cookie = BytesToUint32(bytes[0:4])
-    n.Key = BytesToUint64(bytes[4:12])
-    n.Size = BytesToUint32(bytes[12:16])
-    rest := 8 - ((n.Size + 16 + 4) % 8)
-    r.Seek(int64(n.Size+4+rest), 1)
-    return n, 16+n.Size+4+rest
+	n := new(Needle)
+	bytes := make([]byte, 16)
+	count, e := r.Read(bytes)
+	if count <= 0 || e != nil {
+		return nil, 0
+	}
+	n.Cookie = util.BytesToUint32(bytes[0:4])
+	n.Key = util.BytesToUint64(bytes[4:12])
+	n.Size = util.BytesToUint32(bytes[12:16])
+	rest := 8 - ((n.Size + 16 + 4) % 8)
+	r.Seek(int64(n.Size+4+rest), 1)
+	return n, 16 + n.Size + 4 + rest
 }
-func ParseKeyHash(key_hash_string string)(uint64,uint32) {
-    key_hash_bytes, khe := hex.DecodeString(key_hash_string)
-    key_hash_len := len(key_hash_bytes)
-    if khe != nil || key_hash_len <= 4 {
-        log.Println("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe)
-        return 0, 0
-    }
-    key := BytesToUint64(key_hash_bytes[0 : key_hash_len-4])
-    hash := BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len])
-    return key, hash
+func ParseKeyHash(key_hash_string string) (uint64, uint32) {
+	key_hash_bytes, khe := hex.DecodeString(key_hash_string)
+	key_hash_len := len(key_hash_bytes)
+	if khe != nil || key_hash_len <= 4 {
+		log.Println("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe)
+		return 0, 0
+	}
+	key := util.BytesToUint64(key_hash_bytes[0 : key_hash_len-4])
+	hash := util.BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len])
+	return key, hash
 }

weed-fs/src/pkg/storage/needle_map.go

@@ -3,7 +3,7 @@ package storage
 import (
 	"log"
 	"os"
-	. "util"
+	"pkg/util"
 )
 
 type NeedleValue struct {
@@ -36,24 +36,30 @@ func LoadNeedleMap(file *os.File) *NeedleMap {
 	count, e := nm.indexFile.Read(bytes)
 	if count > 0 {
 		fstat, _ := file.Stat()
-		log.Println("Loading index file", fstat.Name, "size", fstat.Size)
+		log.Println("Loading index file", fstat.Name(), "size", fstat.Size())
 	}
 	for count > 0 && e == nil {
 		for i := 0; i < count; i += 16 {
-			key := BytesToUint64(bytes[i : i+8])
-			offset := BytesToUint32(bytes[i+8 : i+12])
-			size := BytesToUint32(bytes[i+12 : i+16])
-			nm.m[key] = &NeedleValue{Offset: offset, Size: size}, offset > 0
-		}
+			key := util.BytesToUint64(bytes[i : i+8])
+			offset := util.BytesToUint32(bytes[i+8 : i+12])
+			size := util.BytesToUint32(bytes[i+12 : i+16])
+			if offset>0 {
+   			nm.m[key] = &NeedleValue{util.Offset: offset, Size: size}
+   	  }else{
+        delete(nm.m, key)
+   	  }
+		}     
+		
 		count, e = nm.indexFile.Read(bytes)
-	}
+	}	
 	return nm
-}
-func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) (int, os.Error) {
-    nm.m[key] = &NeedleValue{Offset: offset, Size: size}
-	Uint64toBytes(nm.bytes[0:8], key)
-	Uint32toBytes(nm.bytes[8:12], offset)
-	Uint32toBytes(nm.bytes[12:16], size)
+}     
+
+func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) (int, error) {
+	nm.m[key] = &NeedleValue{Offset: offset, Size: size}
+	util.Uint64toBytes(nm.bytes[0:8], key)
+	util.Uint32toBytes(nm.bytes[8:12], offset)
+	util.Uint32toBytes(nm.bytes[12:16], size)
 	return nm.indexFile.Write(nm.bytes)
 }
 func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
@@ -61,10 +67,10 @@ func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
 	return
 }
 func (nm *NeedleMap) Delete(key uint64) {
-	nm.m[key] = nil, false
-	Uint64toBytes(nm.bytes[0:8], key)
-	Uint32toBytes(nm.bytes[8:12], 0)
-	Uint32toBytes(nm.bytes[12:16], 0)
+	delete(nm.m, key)
+	util.Uint64toBytes(nm.bytes[0:8], key)
+	util.Uint32toBytes(nm.bytes[8:12], 0)
+	util.Uint32toBytes(nm.bytes[12:16], 0)
 	nm.indexFile.Write(nm.bytes)
 }
 func (nm *NeedleMap) Close() {

weed-fs/src/pkg/storage/store.go

@@ -1,19 +1,19 @@
 package storage
 
 import (
+	"encoding/json"
+	"errors"
 	"log"
-	"json"
-	"os"
-	"strings"
+	"net/url"
 	"strconv"
-	"url"
-	"util"
+	"strings"
+	"pkg/util"
 )
 
 type Store struct {
-	volumes      map[uint64]*Volume
-	dir          string
-	Port         int
+	volumes   map[uint64]*Volume
+	dir       string
+	Port      int
 	PublicUrl string
 }
 type VolumeInfo struct {
@@ -24,44 +24,44 @@ type VolumeInfo struct {
 func NewStore(port int, publicUrl, dirname string, volumeListString string) (s *Store) {
 	s = &Store{Port: port, PublicUrl: publicUrl, dir: dirname}
 	s.volumes = make(map[uint64]*Volume)
-	
+
 	s.AddVolume(volumeListString)
-		
+
 	log.Println("Store started on dir:", dirname, "with", len(s.volumes), "volumes")
 	return
 }
-func (s *Store) AddVolume(volumeListString string) os.Error{
-    for _, range_string := range strings.Split(volumeListString, ",") {
-        if strings.Index(range_string, "-") < 0 {
-            id_string := range_string
-            id, err := strconv.Atoui64(id_string)
-            if err != nil {
-                return os.NewError("Volume Id " + id_string+ " is not a valid unsigned integer!")
-            }
-            s.addVolume(id)
-        } else {
-            pair := strings.Split(range_string, "-")
-            start, start_err := strconv.Atoui64(pair[0])
-            if start_err != nil {
-                return os.NewError("Volume Start Id" + pair[0] + " is not a valid unsigned integer!")
-            }
-            end, end_err := strconv.Atoui64(pair[1])
-            if end_err != nil {
-                return os.NewError("Volume End Id" + pair[1] + " is not a valid unsigned integer!")
-            }
-            for id := start; id<=end; id++ {
-                s.addVolume(id)
-            }
-        }
-    }
-    return nil
+func (s *Store) AddVolume(volumeListString string) error {
+	for _, range_string := range strings.Split(volumeListString, ",") {
+		if strings.Index(range_string, "-") < 0 {
+			id_string := range_string
+			id, err := strconv.ParseUint(id_string, 10, 64)
+			if err != nil {
+				return errors.New("Volume Id " + id_string + " is not a valid unsigned integer!")
+			}
+			s.addVolume(id)
+		} else {
+			pair := strings.Split(range_string, "-")
+			start, start_err := strconv.ParseUint(pair[0], 10, 64)
+			if start_err != nil {
+				return errors.New("Volume Start Id" + pair[0] + " is not a valid unsigned integer!")
+			}
+			end, end_err := strconv.ParseUint(pair[1], 10, 64)
+			if end_err != nil {
+				return errors.New("Volume End Id" + pair[1] + " is not a valid unsigned integer!")
+			}
+			for id := start; id <= end; id++ {
+				s.addVolume(id)
+			}
+		}
+	}
+	return nil
 }
-func (s *Store) addVolume(vid uint64) os.Error{
-    if s.volumes[vid]!=nil {
-        return os.NewError("Volume Id "+strconv.Uitoa64(vid)+" already exists!")
-    }
-    s.volumes[vid] = NewVolume(s.dir, uint32(vid))
-    return nil
+func (s *Store) addVolume(vid uint64) error {
+	if s.volumes[vid] != nil {
+		return errors.New("Volume Id " + strconv.FormatUint(vid, 10) + " already exists!")
+	}
+	s.volumes[vid] = NewVolume(s.dir, uint32(vid))
+	return nil
 }
 func (s *Store) Status() *[]*VolumeInfo {
 	stats := new([]*VolumeInfo)
@@ -92,11 +92,11 @@ func (s *Store) Close() {
 	}
 }
 func (s *Store) Write(i uint64, n *Needle) uint32 {
-    return s.volumes[i].write(n)
+	return s.volumes[i].write(n)
 }
 func (s *Store) Delete(i uint64, n *Needle) uint32 {
 	return s.volumes[i].delete(n)
 }
-func (s *Store) Read(i uint64, n *Needle) (int, os.Error) {
+func (s *Store) Read(i uint64, n *Needle) (int, error) {
 	return s.volumes[i].read(n)
 }

weed-fs/src/pkg/storage/volume.go

@@ -1,11 +1,12 @@
 package storage
 
 import (
+	"io"
+	"log"
 	"os"
 	"path"
 	"strconv"
 	"sync"
-	"log"
 )
 
 const (
@@ -22,9 +23,9 @@ type Volume struct {
 }
 
 func NewVolume(dirname string, id uint32) (v *Volume) {
-	var e os.Error
+	var e error
 	v = &Volume{dir: dirname, Id: id}
-	fileName := strconv.Uitoa64(uint64(v.Id))
+	fileName := strconv.FormatUint(uint64(v.Id), 10)
 	v.dataFile, e = os.OpenFile(path.Join(v.dir, fileName+".dat"), os.O_RDWR|os.O_CREATE, 0644)
 	if e != nil {
 		log.Fatalf("New Volume [ERROR] %s\n", e)
@@ -41,7 +42,7 @@ func NewVolume(dirname string, id uint32) (v *Volume) {
 func (v *Volume) Size() int64 {
 	stat, e := v.dataFile.Stat()
 	if e == nil {
-		return stat.Size
+		return stat.Size()
 	}
 	return -1
 }
@@ -51,7 +52,7 @@ func (v *Volume) Close() {
 }
 func (v *Volume) maybeWriteSuperBlock() {
 	stat, _ := v.dataFile.Stat()
-	if stat.Size == 0 {
+	if stat.Size() == 0 {
 		header := make([]byte, SuperBlockSize)
 		header[0] = 1
 		v.dataFile.Write(header)
@@ -82,7 +83,7 @@ func (v *Volume) delete(n *Needle) uint32 {
 	}
 	return 0
 }
-func (v *Volume) read(n *Needle) (int, os.Error) {
+func (v *Volume) read(n *Needle) (int, error) {
 	v.accessLock.Lock()
 	defer v.accessLock.Unlock()
 	nv, ok := v.nm.Get(n.Key)
@@ -90,5 +91,5 @@ func (v *Volume) read(n *Needle) (int, os.Error) {
 		v.dataFile.Seek(int64(nv.Offset)*8, 0)
 		return n.Read(v.dataFile, nv.Size)
 	}
-	return -1, os.EOF
+	return -1, io.EOF
 }

weed-fs/src/pkg/util/Makefile

@@ -1,32 +0,0 @@
-# Makefile generated by gb: http://go-gb.googlecode.com
-# gb provides configuration-free building and distributing
-
-include $(GOROOT)/src/Make.inc
-
-TARG=util
-GOFILES=\
-	bytes.go\
-	post.go\
-
-# gb: this is the local install
-GBROOT=../../..
-
-# gb: compile/link against local install
-GCIMPORTS+= -I $(GBROOT)/_obj
-LDIMPORTS+= -L $(GBROOT)/_obj
-
-# gb: compile/link against GOPATH entries
-GOPATHSEP=:
-ifeq ($(GOHOSTOS),windows)
-GOPATHSEP=;
-endif
-GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
-LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
-
-# gb: copy to local install
-$(GBROOT)/_obj/$(TARG).a: _obj/$(TARG).a
-	mkdir -p $(dir $@); cp -f $< $@
-
-package: $(GBROOT)/_obj/$(TARG).a
-
-include $(GOROOT)/src/Make.pkg

weed-fs/src/pkg/util/post.go

@@ -1,23 +1,23 @@
 package util
 
 import (
-    "http"
-    "io/ioutil"
-    "url"
-    "log"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
 )
 
-func Post(url string, values url.Values)[]byte{
-    r, err := http.PostForm(url, values)
-    if err != nil {
-        log.Println("post:", err)
-        return nil
-    }
-    defer r.Body.Close()
-    b, err := ioutil.ReadAll(r.Body)
-    if err != nil {
-        log.Println("post:", err)
-        return nil
-    }
-    return b
+func Post(url string, values url.Values) []byte {
+	r, err := http.PostForm(url, values)
+	if err != nil {
+		log.Println("post:", err)
+		return nil
+	}
+	defer r.Body.Close()
+	b, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		log.Println("post:", err)
+		return nil
+	}
+	return b
 }