diff --git a/weed-fs/build b/weed-fs/build deleted file mode 100755 index 477879ac4..000000000 --- a/weed-fs/build +++ /dev/null @@ -1,19 +0,0 @@ -# Build script generated by gb: http://go-gb.googlecode.com -# gb provides configuration-free building and distributing - -echo "Build script generated by gb: http://go-gb.googlecode.com" -if [ "$1" = "goinstall" ]; then -echo Running goinstall \ - -else -echo Building \ -&& echo "(in src/pkg/util)" gomake $1 && cd src/pkg/util && gomake $1 && cd - > /dev/null \ -&& echo "(in src/pkg/storage)" gomake $1 && cd src/pkg/storage && gomake $1 && cd - > /dev/null \ -&& echo "(in src/cmd/weedvolume/fix_volume_index)" gomake $1 && cd src/cmd/weedvolume/fix_volume_index && gomake $1 && cd - > /dev/null \ -&& echo "(in src/cmd/weedvolume)" gomake $1 && cd src/cmd/weedvolume && gomake $1 && cd - > /dev/null \ -&& echo "(in src/pkg/directory)" gomake $1 && cd src/pkg/directory && gomake $1 && cd - > /dev/null \ -&& echo "(in src/cmd/weedmaster)" gomake $1 && cd src/cmd/weedmaster && gomake $1 && cd - > /dev/null \ - -fi - -# The makefiles above are invoked in topological dependence order diff --git a/weed-fs/src/cmd/weedmaster/Makefile b/weed-fs/src/cmd/weedmaster/Makefile deleted file mode 100644 index 9acc3f421..000000000 --- a/weed-fs/src/cmd/weedmaster/Makefile +++ /dev/null @@ -1,39 +0,0 @@ -# Makefile generated by gb: http://go-gb.googlecode.com -# gb provides configuration-free building and distributing - -include $(GOROOT)/src/Make.inc - -TARG=weedmaster -GOFILES=\ - weedmaster.go\ - -# gb: this is the local install -GBROOT=../../.. - -# gb: compile/link against local install -GCIMPORTS+= -I $(GBROOT)/_obj -LDIMPORTS+= -L $(GBROOT)/_obj - -# gb: compile/link against GOPATH entries -GOPATHSEP=: -ifeq ($(GOHOSTOS),windows) -GOPATHSEP=; -endif -GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) -LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) - -# gb: default target is in GBROOT this way -command: - -include $(GOROOT)/src/Make.cmd - -# gb: copy to local install -$(GBROOT)/bin/$(TARG): $(TARG) - mkdir -p $(dir $@); cp -f $< $@ -command: $(GBROOT)/bin/$(TARG) - -# gb: local dependencies -$(TARG): $(GBROOT)/_obj/storage.a - -$(TARG): $(GBROOT)/_obj/directory.a - diff --git a/weed-fs/src/cmd/weedmaster/weedmaster.go b/weed-fs/src/cmd/weedmaster/weedmaster.go deleted file mode 100644 index 13733f2ca..000000000 --- a/weed-fs/src/cmd/weedmaster/weedmaster.go +++ /dev/null @@ -1,90 +0,0 @@ -package main - -import ( - "storage" - "directory" - "flag" - "fmt" - "http" - "json" - "log" - "strconv" - "strings" -) - -var ( - port = flag.Int("port", 9333, "http listen port") - metaFolder = flag.String("mdir", "/tmp", "data directory to store mappings") - capacity = flag.Int("capacity", 100, "maximum number of volumes to hold") - mapper *directory.Mapper - IsDebug = flag.Bool("debug", false, "verbose debug information") - volumeSizeLimitMB = flag.Uint("volumeSizeLimitMB", 32*1024, "Default Volume Size in MegaBytes") -) - -func dirLookupHandler(w http.ResponseWriter, r *http.Request) { - vid := r.FormValue("volumeId") - commaSep := strings.Index(vid, ",") - if commaSep > 0 { - vid = vid[0:commaSep] - } - volumeId, _ := strconv.Atoui64(vid) - machine, e := mapper.Get(uint32(volumeId)) - if e == nil { - writeJson(w, r, machine.Server) - } else { - log.Println("Invalid volume id", volumeId) - writeJson(w, r, map[string]string{"error": "volume id " + strconv.Uitoa64(volumeId) + " not found"}) - } -} -func dirAssignHandler(w http.ResponseWriter, r *http.Request) { - fid, machine, err := mapper.PickForWrite() - if err == nil { - writeJson(w, r, map[string]string{"fid": fid, "url": machine.Url}) - } else { - log.Println(err) - writeJson(w, r, map[string]string{"error": err.String()}) - } -} -func dirJoinHandler(w http.ResponseWriter, r *http.Request) { - s := r.RemoteAddr[0:strings.Index(r.RemoteAddr, ":")+1] + r.FormValue("port") - publicUrl := r.FormValue("publicUrl") - volumes := new([]storage.VolumeInfo) - json.Unmarshal([]byte(r.FormValue("volumes")), volumes) - if *IsDebug { - log.Println(s, "volumes", r.FormValue("volumes")) - } - mapper.Add(*directory.NewMachine(s, publicUrl, *volumes)) -} -func dirStatusHandler(w http.ResponseWriter, r *http.Request) { - writeJson(w, r, mapper) -} -func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) { - w.Header().Set("Content-Type", "application/javascript") - bytes, _ := json.Marshal(obj) - callback := r.FormValue("callback") - if callback == "" { - w.Write(bytes) - } else { - w.Write([]uint8(callback)) - w.Write([]uint8("(")) - fmt.Fprint(w, string(bytes)) - w.Write([]uint8(")")) - } -} - -func main() { - flag.Parse() - log.Println("Volume Size Limit is", *volumeSizeLimitMB, "MB") - mapper = directory.NewMapper(*metaFolder, "directory", uint64(*volumeSizeLimitMB)*1024*1024) - http.HandleFunc("/dir/assign", dirAssignHandler) - http.HandleFunc("/dir/lookup", dirLookupHandler) - http.HandleFunc("/dir/join", dirJoinHandler) - http.HandleFunc("/dir/status", dirStatusHandler) - - log.Println("Start directory service at http://127.0.0.1:" + strconv.Itoa(*port)) - e := http.ListenAndServe(":"+strconv.Itoa(*port), nil) - if e != nil { - log.Fatalf("Fail to start:", e.String()) - } - -} diff --git a/weed-fs/src/cmd/weedvolume/Makefile b/weed-fs/src/cmd/weedvolume/Makefile deleted file mode 100644 index 62d8088b5..000000000 --- a/weed-fs/src/cmd/weedvolume/Makefile +++ /dev/null @@ -1,37 +0,0 @@ -# Makefile generated by gb: http://go-gb.googlecode.com -# gb provides configuration-free building and distributing - -include $(GOROOT)/src/Make.inc - -TARG=weedvolume -GOFILES=\ - weedvolume.go\ - -# gb: this is the local install -GBROOT=../../.. - -# gb: compile/link against local install -GCIMPORTS+= -I $(GBROOT)/_obj -LDIMPORTS+= -L $(GBROOT)/_obj - -# gb: compile/link against GOPATH entries -GOPATHSEP=: -ifeq ($(GOHOSTOS),windows) -GOPATHSEP=; -endif -GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) -LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) - -# gb: default target is in GBROOT this way -command: - -include $(GOROOT)/src/Make.cmd - -# gb: copy to local install -$(GBROOT)/bin/$(TARG): $(TARG) - mkdir -p $(dir $@); cp -f $< $@ -command: $(GBROOT)/bin/$(TARG) - -# gb: local dependencies -$(TARG): $(GBROOT)/_obj/storage.a - diff --git a/weed-fs/src/cmd/weedvolume/fix_volume_index/Makefile b/weed-fs/src/cmd/weedvolume/fix_volume_index/Makefile deleted file mode 100644 index 516117f0e..000000000 --- a/weed-fs/src/cmd/weedvolume/fix_volume_index/Makefile +++ /dev/null @@ -1,37 +0,0 @@ -# Makefile generated by gb: http://go-gb.googlecode.com -# gb provides configuration-free building and distributing - -include $(GOROOT)/src/Make.inc - -TARG=fix_volume_index -GOFILES=\ - fix_volume_index.go\ - -# gb: this is the local install -GBROOT=../../../.. - -# gb: compile/link against local install -GCIMPORTS+= -I $(GBROOT)/_obj -LDIMPORTS+= -L $(GBROOT)/_obj - -# gb: compile/link against GOPATH entries -GOPATHSEP=: -ifeq ($(GOHOSTOS),windows) -GOPATHSEP=; -endif -GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) -LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) - -# gb: default target is in GBROOT this way -command: - -include $(GOROOT)/src/Make.cmd - -# gb: copy to local install -$(GBROOT)/bin/$(TARG): $(TARG) - mkdir -p $(dir $@); cp -f $< $@ -command: $(GBROOT)/bin/$(TARG) - -# gb: local dependencies -$(TARG): $(GBROOT)/_obj/storage.a - diff --git a/weed-fs/src/cmd/weedvolume/fix_volume_index/fix_volume_index.go b/weed-fs/src/cmd/weedvolume/fix_volume_index/fix_volume_index.go deleted file mode 100644 index 9f4c089fb..000000000 --- a/weed-fs/src/cmd/weedvolume/fix_volume_index/fix_volume_index.go +++ /dev/null @@ -1,59 +0,0 @@ -package main - -import ( - "storage" - "flag" - "log" - "os" - "path" - "strconv" -) - -var ( - dir = flag.String("dir", "/tmp", "data directory to store files") - volumeId = flag.Int("volumeId", -1, "a non-negative volume id. The volume should already exist in the dir. The volume index file should not exist.") - IsDebug = flag.Bool("debug", false, "enable debug mode") - - store *storage.Store -) - -func main() { - flag.Parse() - - if *volumeId == -1 { - flag.Usage() - return - } - - fileName := strconv.Itoa(*volumeId) - dataFile, e := os.OpenFile(path.Join(*dir, fileName+".dat"), os.O_RDONLY, 0644) - if e != nil { - log.Fatalf("Read Volume [ERROR] %s\n", e) - } - defer dataFile.Close() - indexFile, ie := os.OpenFile(path.Join(*dir, fileName+".idx"), os.O_WRONLY|os.O_CREATE, 0644) - if ie != nil { - log.Fatalf("Create Volume Index [ERROR] %s\n", ie) - } - defer indexFile.Close() - - //skip the volume super block - dataFile.Seek(storage.SuperBlockSize, 0) - - n, length := storage.ReadNeedle(dataFile) - nm := storage.NewNeedleMap(indexFile) - offset := uint32(storage.SuperBlockSize) - for n != nil { - if *IsDebug { - log.Println("key", n.Key, "volume offset", offset, "data_size", n.Size, "length", length) - } - if n.Size > 0 { - count, pe := nm.Put(n.Key, offset/8, n.Size) - if *IsDebug { - log.Println("saved", count, "with error", pe) - } - } - offset += length - n, length = storage.ReadNeedle(dataFile) - } -} diff --git a/weed-fs/src/cmd/weedvolume/weedvolume.go b/weed-fs/src/cmd/weedvolume/weedvolume.go deleted file mode 100644 index b026189c0..000000000 --- a/weed-fs/src/cmd/weedvolume/weedvolume.go +++ /dev/null @@ -1,168 +0,0 @@ -package main - -import ( - "storage" - "flag" - "fmt" - "http" - "json" - "log" - "mime" - "rand" - "strconv" - "strings" - "time" -) - -var ( - port = flag.Int("port", 8080, "http listen port") - chunkFolder = flag.String("dir", "/tmp", "data directory to store files") - volumes = flag.String("volumes", "0,1-3,4", "comma-separated list of volume ids or range of ids") - publicUrl = flag.String("publicUrl", "localhost:8080", "public url to serve data read") - metaServer = flag.String("mserver", "localhost:9333", "master directory server to store mappings") - IsDebug = flag.Bool("debug", false, "enable debug mode") - pulse = flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") - - store *storage.Store -) - -func statusHandler(w http.ResponseWriter, r *http.Request) { - writeJson(w, r, store.Status()) -} -func addVolumeHandler(w http.ResponseWriter, r *http.Request) { - store.AddVolume(r.FormValue("volume")) - writeJson(w, r, store.Status()) -} -func storeHandler(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case "GET": - GetHandler(w, r) - case "DELETE": - DeleteHandler(w, r) - case "POST": - PostHandler(w, r) - } -} -func GetHandler(w http.ResponseWriter, r *http.Request) { - n := new(storage.Needle) - vid, fid, ext := parseURLPath(r.URL.Path) - volumeId, _ := strconv.Atoui64(vid) - n.ParsePath(fid) - - if *IsDebug { - log.Println("volume", volumeId, "reading", n) - } - cookie := n.Cookie - count, e := store.Read(volumeId, n) - if *IsDebug { - log.Println("read bytes", count, "error", e) - } - if n.Cookie != cookie { - log.Println("request with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent()) - return - } - if ext != "" { - w.Header().Set("Content-Type", mime.TypeByExtension(ext)) - } - w.Write(n.Data) -} -func PostHandler(w http.ResponseWriter, r *http.Request) { - vid, _, _ := parseURLPath(r.URL.Path) - volumeId, e := strconv.Atoui64(vid) - if e != nil { - writeJson(w, r, e) - } else { - ret := store.Write(volumeId, storage.NewNeedle(r)) - m := make(map[string]uint32) - m["size"] = ret - writeJson(w, r, m) - } -} -func DeleteHandler(w http.ResponseWriter, r *http.Request) { - n := new(storage.Needle) - vid, fid, _ := parseURLPath(r.URL.Path) - volumeId, _ := strconv.Atoui64(vid) - n.ParsePath(fid) - - if *IsDebug { - log.Println("deleting", n) - } - - cookie := n.Cookie - count, ok := store.Read(volumeId, n) - - if ok!=nil { - m := make(map[string]uint32) - m["size"] = 0 - writeJson(w, r, m) - return - } - - if n.Cookie != cookie { - log.Println("delete with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent()) - return - } - - n.Size = 0 - store.Delete(volumeId, n) - m := make(map[string]uint32) - m["size"] = uint32(count) - writeJson(w, r, m) -} -func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) { - w.Header().Set("Content-Type", "application/javascript") - bytes, _ := json.Marshal(obj) - callback := r.FormValue("callback") - if callback == "" { - w.Write(bytes) - } else { - w.Write([]uint8(callback)) - w.Write([]uint8("(")) - fmt.Fprint(w, string(bytes)) - w.Write([]uint8(")")) - } - //log.Println("JSON Response", string(bytes)) -} -func parseURLPath(path string) (vid, fid, ext string) { - sepIndex := strings.LastIndex(path, "/") - commaIndex := strings.LastIndex(path[sepIndex:], ",") - if commaIndex <= 0 { - log.Println("unknown file id", path[sepIndex+1:]) - return - } - dotIndex := strings.LastIndex(path[sepIndex:], ".") - vid = path[sepIndex+1 : commaIndex] - fid = path[commaIndex+1:] - ext = "" - if dotIndex > 0 { - fid = path[commaIndex+1 : dotIndex] - ext = path[dotIndex+1:] - } - return -} - -func main() { - flag.Parse() - //TODO: now default to 1G, this value should come from server? - store = storage.NewStore(*port, *publicUrl, *chunkFolder, *volumes) - defer store.Close() - http.HandleFunc("/", storeHandler) - http.HandleFunc("/status", statusHandler) - http.HandleFunc("/add_volume", addVolumeHandler) - - go func() { - for { - store.Join(*metaServer) - ns := int64(*pulse) * 1e9 - time.Sleep(ns + rand.Int63()%ns) - } - }() - log.Println("store joined at", *metaServer) - - log.Println("Start storage service at http://127.0.0.1:"+strconv.Itoa(*port), "public url", *publicUrl) - e := http.ListenAndServe(":"+strconv.Itoa(*port), nil) - if e != nil { - log.Fatalf("Fail to start:", e.String()) - } - -} diff --git a/weed-fs/src/pkg/directory/Makefile b/weed-fs/src/pkg/directory/Makefile deleted file mode 100644 index 49fc1b597..000000000 --- a/weed-fs/src/pkg/directory/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -# Makefile generated by gb: http://go-gb.googlecode.com -# gb provides configuration-free building and distributing - -include $(GOROOT)/src/Make.inc - -TARG=directory -GOFILES=\ - file_id.go\ - volume_mapping.go\ - -# gb: this is the local install -GBROOT=../../.. - -# gb: compile/link against local install -GCIMPORTS+= -I $(GBROOT)/_obj -LDIMPORTS+= -L $(GBROOT)/_obj - -# gb: compile/link against GOPATH entries -GOPATHSEP=: -ifeq ($(GOHOSTOS),windows) -GOPATHSEP=; -endif -GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) -LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) - -# gb: copy to local install -$(GBROOT)/_obj/$(TARG).a: _obj/$(TARG).a - mkdir -p $(dir $@); cp -f $< $@ - -package: $(GBROOT)/_obj/$(TARG).a - -include $(GOROOT)/src/Make.pkg - -# gb: local dependencies -_obj/$(TARG).a: $(GBROOT)/_obj/storage.a -_obj/$(TARG).a: $(GBROOT)/_obj/util.a diff --git a/weed-fs/src/pkg/directory/file_id.go b/weed-fs/src/pkg/directory/file_id.go index 549fcdc53..4d9882897 100644 --- a/weed-fs/src/pkg/directory/file_id.go +++ b/weed-fs/src/pkg/directory/file_id.go @@ -3,10 +3,10 @@ package directory import ( "encoding/hex" "log" - "storage" + "pkg/storage" "strconv" "strings" - "util" + "pkg/util" ) type FileId struct { @@ -18,14 +18,14 @@ type FileId struct { func NewFileId(VolumeId uint32, Key uint64, Hashcode uint32) *FileId { return &FileId{VolumeId: VolumeId, Key: Key, Hashcode: Hashcode} } -func ParseFileId(fid string) *FileId { +func ParseFileId(fid string) *FileId{ a := strings.Split(fid, ",") if len(a) != 2 { - log.Println("Invalid fid", fid, ", split length", len(a)) + log.Println("Invalid fid", fid, ", split length", len(a)) return nil } vid_string, key_hash_string := a[0], a[1] - vid, _ := strconv.Atoui64(vid_string) + vid, _ := strconv.ParseUint(vid_string, 10, 64) key, hash := storage.ParseKeyHash(key_hash_string) return &FileId{VolumeId: uint32(vid), Key: key, Hashcode: hash} } @@ -36,5 +36,5 @@ func (n *FileId) String() string { nonzero_index := 0 for ; bytes[nonzero_index] == 0; nonzero_index++ { } - return strconv.Uitoa64(uint64(n.VolumeId)) + "," + hex.EncodeToString(bytes[nonzero_index:]) + return strconv.FormatUint(uint64(n.VolumeId), 10) + "," + hex.EncodeToString(bytes[nonzero_index:]) } diff --git a/weed-fs/src/pkg/directory/volume_mapping.go b/weed-fs/src/pkg/directory/volume_mapping.go index 527ecbe1a..267a38a86 100644 --- a/weed-fs/src/pkg/directory/volume_mapping.go +++ b/weed-fs/src/pkg/directory/volume_mapping.go @@ -1,12 +1,13 @@ package directory import ( - "gob" + "encoding/gob" + "errors" + "log" + "math/rand" "os" "path" - "rand" - "log" - "storage" + "pkg/storage" "strconv" "sync" ) @@ -31,7 +32,7 @@ type Mapper struct { lock sync.Mutex Machines []*Machine vid2machineId map[uint32]int //machineId is +1 of the index of []*Machine, to detect not found entries - Writers []uint32 // transient array of Writers volume id + Writers []uint32 // transient array of Writers volume id FileIdSequence uint64 fileIdCounter uint64 @@ -64,11 +65,11 @@ func NewMapper(dirname string, filename string, volumeSizeLimit uint64) (m *Mapp } return } -func (m *Mapper) PickForWrite() (string, MachineInfo, os.Error) { +func (m *Mapper) PickForWrite() (string, MachineInfo, error) { len_writers := len(m.Writers) if len_writers <= 0 { log.Println("No more writable volumes!") - return "", m.Machines[rand.Intn(len(m.Machines))].Server, os.NewError("No more writable volumes!") + return "", m.Machines[rand.Intn(len(m.Machines))].Server, errors.New("No more writable volumes!") } vid := m.Writers[rand.Intn(len_writers)] machine_id := m.vid2machineId[vid] @@ -76,7 +77,7 @@ func (m *Mapper) PickForWrite() (string, MachineInfo, os.Error) { machine := m.Machines[machine_id-1] return NewFileId(vid, m.NextFileId(), rand.Uint32()).String(), machine.Server, nil } - return "", m.Machines[rand.Intn(len(m.Machines))].Server, os.NewError("Strangely vid " + strconv.Uitoa64(uint64(vid)) + " is on no machine!") + return "", m.Machines[rand.Intn(len(m.Machines))].Server, errors.New("Strangely vid " + strconv.FormatUint(uint64(vid), 10) + " is on no machine!") } func (m *Mapper) NextFileId() uint64 { if m.fileIdCounter <= 0 { @@ -87,16 +88,16 @@ func (m *Mapper) NextFileId() uint64 { m.fileIdCounter-- return m.FileIdSequence - m.fileIdCounter } -func (m *Mapper) Get(vid uint32) (*Machine, os.Error) { +func (m *Mapper) Get(vid uint32) (*Machine, error) { machineId := m.vid2machineId[vid] if machineId <= 0 { - return nil, os.NewError("invalid volume id " + strconv.Uitob64(uint64(vid), 10)) + return nil, errors.New("invalid volume id " + strconv.FormatUint(uint64(vid), 10)) } return m.Machines[machineId-1], nil } func (m *Mapper) Add(machine Machine) { //check existing machine, linearly - //log.Println("Adding machine", machine.Server.Url) + //log.Println("Adding machine", machine.Server.Url) m.lock.Lock() foundExistingMachineId := -1 for index, entry := range m.Machines { diff --git a/weed-fs/src/pkg/storage/Makefile b/weed-fs/src/pkg/storage/Makefile deleted file mode 100644 index eceed56ba..000000000 --- a/weed-fs/src/pkg/storage/Makefile +++ /dev/null @@ -1,37 +0,0 @@ -# Makefile generated by gb: http://go-gb.googlecode.com -# gb provides configuration-free building and distributing - -include $(GOROOT)/src/Make.inc - -TARG=storage -GOFILES=\ - needle.go\ - needle_map.go\ - store.go\ - volume.go\ - -# gb: this is the local install -GBROOT=../../.. - -# gb: compile/link against local install -GCIMPORTS+= -I $(GBROOT)/_obj -LDIMPORTS+= -L $(GBROOT)/_obj - -# gb: compile/link against GOPATH entries -GOPATHSEP=: -ifeq ($(GOHOSTOS),windows) -GOPATHSEP=; -endif -GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) -LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) - -# gb: copy to local install -$(GBROOT)/_obj/$(TARG).a: _obj/$(TARG).a - mkdir -p $(dir $@); cp -f $< $@ - -package: $(GBROOT)/_obj/$(TARG).a - -include $(GOROOT)/src/Make.pkg - -# gb: local dependencies -_obj/$(TARG).a: $(GBROOT)/_obj/util.a diff --git a/weed-fs/src/pkg/storage/needle.go b/weed-fs/src/pkg/storage/needle.go index c667aa245..dff46b97a 100644 --- a/weed-fs/src/pkg/storage/needle.go +++ b/weed-fs/src/pkg/storage/needle.go @@ -1,14 +1,14 @@ package storage import ( - "encoding/hex" + "encoding/hex" "io" "io/ioutil" - "http" "log" + "net/http" "os" "strings" - . "util" + "pkg/util" ) type Needle struct { @@ -21,6 +21,7 @@ type Needle struct { } func NewNeedle(r *http.Request) (n *Needle) { + n = new(Needle) form, fe := r.MultipartReader() if fe != nil { @@ -30,11 +31,11 @@ func NewNeedle(r *http.Request) (n *Needle) { data, _ := ioutil.ReadAll(part) n.Data = data - commaSep := strings.LastIndex(r.URL.Path, ",") + commaSep := strings.LastIndex(r.URL.Path, ",") dotSep := strings.LastIndex(r.URL.Path, ".") fid := r.URL.Path[commaSep+1:] if dotSep > 0 { - fid = r.URL.Path[commaSep+1:dotSep] + fid = r.URL.Path[commaSep+1 : dotSep] } n.ParsePath(fid) @@ -49,51 +50,51 @@ func (n *Needle) ParsePath(fid string) { } n.Key, n.Cookie = ParseKeyHash(fid) } -func (n *Needle) Append(w io.Writer) (uint32){ +func (n *Needle) Append(w io.Writer) uint32 { header := make([]byte, 16) - Uint32toBytes(header[0:4], n.Cookie) - Uint64toBytes(header[4:12], n.Key) + util.Uint32toBytes(header[0:4], n.Cookie) + util.Uint64toBytes(header[4:12], n.Key) n.Size = uint32(len(n.Data)) - Uint32toBytes(header[12:16], n.Size) + util.Uint32toBytes(header[12:16], n.Size) w.Write(header) w.Write(n.Data) rest := 8 - ((n.Size + 16 + 4) % 8) - Uint32toBytes(header[0:4], uint32(n.Checksum)) + util.Uint32toBytes(header[0:4], uint32(n.Checksum)) w.Write(header[0 : rest+4]) return n.Size } -func (n *Needle) Read(r io.Reader, size uint32)(int, os.Error) { +func (n *Needle) Read(r io.Reader, size uint32) (int, error) { bytes := make([]byte, size+16+4) ret, e := r.Read(bytes) - n.Cookie = BytesToUint32(bytes[0:4]) - n.Key = BytesToUint64(bytes[4:12]) - n.Size = BytesToUint32(bytes[12:16]) + n.Cookie = util.BytesToUint32(bytes[0:4]) + n.Key = util.BytesToUint64(bytes[4:12]) + n.Size = util.BytesToUint32(bytes[12:16]) n.Data = bytes[16 : 16+size] - n.Checksum = int32(BytesToUint32(bytes[16+size : 16+size+4])) + n.Checksum = int32(util.BytesToUint32(bytes[16+size : 16+size+4])) return ret, e } func ReadNeedle(r *os.File) (*Needle, uint32) { - n := new(Needle) - bytes := make([]byte, 16) - count, e := r.Read(bytes) - if count<=0 || e!=nil { - return nil, 0 - } - n.Cookie = BytesToUint32(bytes[0:4]) - n.Key = BytesToUint64(bytes[4:12]) - n.Size = BytesToUint32(bytes[12:16]) - rest := 8 - ((n.Size + 16 + 4) % 8) - r.Seek(int64(n.Size+4+rest), 1) - return n, 16+n.Size+4+rest + n := new(Needle) + bytes := make([]byte, 16) + count, e := r.Read(bytes) + if count <= 0 || e != nil { + return nil, 0 + } + n.Cookie = util.BytesToUint32(bytes[0:4]) + n.Key = util.BytesToUint64(bytes[4:12]) + n.Size = util.BytesToUint32(bytes[12:16]) + rest := 8 - ((n.Size + 16 + 4) % 8) + r.Seek(int64(n.Size+4+rest), 1) + return n, 16 + n.Size + 4 + rest } -func ParseKeyHash(key_hash_string string)(uint64,uint32) { - key_hash_bytes, khe := hex.DecodeString(key_hash_string) - key_hash_len := len(key_hash_bytes) - if khe != nil || key_hash_len <= 4 { - log.Println("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe) - return 0, 0 - } - key := BytesToUint64(key_hash_bytes[0 : key_hash_len-4]) - hash := BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len]) - return key, hash +func ParseKeyHash(key_hash_string string) (uint64, uint32) { + key_hash_bytes, khe := hex.DecodeString(key_hash_string) + key_hash_len := len(key_hash_bytes) + if khe != nil || key_hash_len <= 4 { + log.Println("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe) + return 0, 0 + } + key := util.BytesToUint64(key_hash_bytes[0 : key_hash_len-4]) + hash := util.BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len]) + return key, hash } diff --git a/weed-fs/src/pkg/storage/needle_map.go b/weed-fs/src/pkg/storage/needle_map.go index 3cdcf6328..b92da7de1 100644 --- a/weed-fs/src/pkg/storage/needle_map.go +++ b/weed-fs/src/pkg/storage/needle_map.go @@ -3,7 +3,7 @@ package storage import ( "log" "os" - . "util" + "pkg/util" ) type NeedleValue struct { @@ -36,24 +36,30 @@ func LoadNeedleMap(file *os.File) *NeedleMap { count, e := nm.indexFile.Read(bytes) if count > 0 { fstat, _ := file.Stat() - log.Println("Loading index file", fstat.Name, "size", fstat.Size) + log.Println("Loading index file", fstat.Name(), "size", fstat.Size()) } for count > 0 && e == nil { for i := 0; i < count; i += 16 { - key := BytesToUint64(bytes[i : i+8]) - offset := BytesToUint32(bytes[i+8 : i+12]) - size := BytesToUint32(bytes[i+12 : i+16]) - nm.m[key] = &NeedleValue{Offset: offset, Size: size}, offset > 0 - } + key := util.BytesToUint64(bytes[i : i+8]) + offset := util.BytesToUint32(bytes[i+8 : i+12]) + size := util.BytesToUint32(bytes[i+12 : i+16]) + if offset>0 { + nm.m[key] = &NeedleValue{util.Offset: offset, Size: size} + }else{ + delete(nm.m, key) + } + } + count, e = nm.indexFile.Read(bytes) - } + } return nm -} -func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) (int, os.Error) { - nm.m[key] = &NeedleValue{Offset: offset, Size: size} - Uint64toBytes(nm.bytes[0:8], key) - Uint32toBytes(nm.bytes[8:12], offset) - Uint32toBytes(nm.bytes[12:16], size) +} + +func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) (int, error) { + nm.m[key] = &NeedleValue{Offset: offset, Size: size} + util.Uint64toBytes(nm.bytes[0:8], key) + util.Uint32toBytes(nm.bytes[8:12], offset) + util.Uint32toBytes(nm.bytes[12:16], size) return nm.indexFile.Write(nm.bytes) } func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { @@ -61,10 +67,10 @@ func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { return } func (nm *NeedleMap) Delete(key uint64) { - nm.m[key] = nil, false - Uint64toBytes(nm.bytes[0:8], key) - Uint32toBytes(nm.bytes[8:12], 0) - Uint32toBytes(nm.bytes[12:16], 0) + delete(nm.m, key) + util.Uint64toBytes(nm.bytes[0:8], key) + util.Uint32toBytes(nm.bytes[8:12], 0) + util.Uint32toBytes(nm.bytes[12:16], 0) nm.indexFile.Write(nm.bytes) } func (nm *NeedleMap) Close() { diff --git a/weed-fs/src/pkg/storage/store.go b/weed-fs/src/pkg/storage/store.go index f70321ab1..e5cbd546b 100644 --- a/weed-fs/src/pkg/storage/store.go +++ b/weed-fs/src/pkg/storage/store.go @@ -1,19 +1,19 @@ package storage import ( + "encoding/json" + "errors" "log" - "json" - "os" - "strings" + "net/url" "strconv" - "url" - "util" + "strings" + "pkg/util" ) type Store struct { - volumes map[uint64]*Volume - dir string - Port int + volumes map[uint64]*Volume + dir string + Port int PublicUrl string } type VolumeInfo struct { @@ -24,44 +24,44 @@ type VolumeInfo struct { func NewStore(port int, publicUrl, dirname string, volumeListString string) (s *Store) { s = &Store{Port: port, PublicUrl: publicUrl, dir: dirname} s.volumes = make(map[uint64]*Volume) - + s.AddVolume(volumeListString) - + log.Println("Store started on dir:", dirname, "with", len(s.volumes), "volumes") return } -func (s *Store) AddVolume(volumeListString string) os.Error{ - for _, range_string := range strings.Split(volumeListString, ",") { - if strings.Index(range_string, "-") < 0 { - id_string := range_string - id, err := strconv.Atoui64(id_string) - if err != nil { - return os.NewError("Volume Id " + id_string+ " is not a valid unsigned integer!") - } - s.addVolume(id) - } else { - pair := strings.Split(range_string, "-") - start, start_err := strconv.Atoui64(pair[0]) - if start_err != nil { - return os.NewError("Volume Start Id" + pair[0] + " is not a valid unsigned integer!") - } - end, end_err := strconv.Atoui64(pair[1]) - if end_err != nil { - return os.NewError("Volume End Id" + pair[1] + " is not a valid unsigned integer!") - } - for id := start; id<=end; id++ { - s.addVolume(id) - } - } - } - return nil +func (s *Store) AddVolume(volumeListString string) error { + for _, range_string := range strings.Split(volumeListString, ",") { + if strings.Index(range_string, "-") < 0 { + id_string := range_string + id, err := strconv.ParseUint(id_string, 10, 64) + if err != nil { + return errors.New("Volume Id " + id_string + " is not a valid unsigned integer!") + } + s.addVolume(id) + } else { + pair := strings.Split(range_string, "-") + start, start_err := strconv.ParseUint(pair[0], 10, 64) + if start_err != nil { + return errors.New("Volume Start Id" + pair[0] + " is not a valid unsigned integer!") + } + end, end_err := strconv.ParseUint(pair[1], 10, 64) + if end_err != nil { + return errors.New("Volume End Id" + pair[1] + " is not a valid unsigned integer!") + } + for id := start; id <= end; id++ { + s.addVolume(id) + } + } + } + return nil } -func (s *Store) addVolume(vid uint64) os.Error{ - if s.volumes[vid]!=nil { - return os.NewError("Volume Id "+strconv.Uitoa64(vid)+" already exists!") - } - s.volumes[vid] = NewVolume(s.dir, uint32(vid)) - return nil +func (s *Store) addVolume(vid uint64) error { + if s.volumes[vid] != nil { + return errors.New("Volume Id " + strconv.FormatUint(vid, 10) + " already exists!") + } + s.volumes[vid] = NewVolume(s.dir, uint32(vid)) + return nil } func (s *Store) Status() *[]*VolumeInfo { stats := new([]*VolumeInfo) @@ -92,11 +92,11 @@ func (s *Store) Close() { } } func (s *Store) Write(i uint64, n *Needle) uint32 { - return s.volumes[i].write(n) + return s.volumes[i].write(n) } func (s *Store) Delete(i uint64, n *Needle) uint32 { return s.volumes[i].delete(n) } -func (s *Store) Read(i uint64, n *Needle) (int, os.Error) { +func (s *Store) Read(i uint64, n *Needle) (int, error) { return s.volumes[i].read(n) } diff --git a/weed-fs/src/pkg/storage/volume.go b/weed-fs/src/pkg/storage/volume.go index 08c97af9d..01d93efb7 100644 --- a/weed-fs/src/pkg/storage/volume.go +++ b/weed-fs/src/pkg/storage/volume.go @@ -1,11 +1,12 @@ package storage import ( + "io" + "log" "os" "path" "strconv" "sync" - "log" ) const ( @@ -22,9 +23,9 @@ type Volume struct { } func NewVolume(dirname string, id uint32) (v *Volume) { - var e os.Error + var e error v = &Volume{dir: dirname, Id: id} - fileName := strconv.Uitoa64(uint64(v.Id)) + fileName := strconv.FormatUint(uint64(v.Id), 10) v.dataFile, e = os.OpenFile(path.Join(v.dir, fileName+".dat"), os.O_RDWR|os.O_CREATE, 0644) if e != nil { log.Fatalf("New Volume [ERROR] %s\n", e) @@ -41,7 +42,7 @@ func NewVolume(dirname string, id uint32) (v *Volume) { func (v *Volume) Size() int64 { stat, e := v.dataFile.Stat() if e == nil { - return stat.Size + return stat.Size() } return -1 } @@ -51,7 +52,7 @@ func (v *Volume) Close() { } func (v *Volume) maybeWriteSuperBlock() { stat, _ := v.dataFile.Stat() - if stat.Size == 0 { + if stat.Size() == 0 { header := make([]byte, SuperBlockSize) header[0] = 1 v.dataFile.Write(header) @@ -82,7 +83,7 @@ func (v *Volume) delete(n *Needle) uint32 { } return 0 } -func (v *Volume) read(n *Needle) (int, os.Error) { +func (v *Volume) read(n *Needle) (int, error) { v.accessLock.Lock() defer v.accessLock.Unlock() nv, ok := v.nm.Get(n.Key) @@ -90,5 +91,5 @@ func (v *Volume) read(n *Needle) (int, os.Error) { v.dataFile.Seek(int64(nv.Offset)*8, 0) return n.Read(v.dataFile, nv.Size) } - return -1, os.EOF + return -1, io.EOF } diff --git a/weed-fs/src/pkg/util/Makefile b/weed-fs/src/pkg/util/Makefile deleted file mode 100644 index 458e05408..000000000 --- a/weed-fs/src/pkg/util/Makefile +++ /dev/null @@ -1,32 +0,0 @@ -# Makefile generated by gb: http://go-gb.googlecode.com -# gb provides configuration-free building and distributing - -include $(GOROOT)/src/Make.inc - -TARG=util -GOFILES=\ - bytes.go\ - post.go\ - -# gb: this is the local install -GBROOT=../../.. - -# gb: compile/link against local install -GCIMPORTS+= -I $(GBROOT)/_obj -LDIMPORTS+= -L $(GBROOT)/_obj - -# gb: compile/link against GOPATH entries -GOPATHSEP=: -ifeq ($(GOHOSTOS),windows) -GOPATHSEP=; -endif -GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) -LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) - -# gb: copy to local install -$(GBROOT)/_obj/$(TARG).a: _obj/$(TARG).a - mkdir -p $(dir $@); cp -f $< $@ - -package: $(GBROOT)/_obj/$(TARG).a - -include $(GOROOT)/src/Make.pkg diff --git a/weed-fs/src/pkg/util/post.go b/weed-fs/src/pkg/util/post.go index 8b421c830..871448785 100644 --- a/weed-fs/src/pkg/util/post.go +++ b/weed-fs/src/pkg/util/post.go @@ -1,23 +1,23 @@ package util import ( - "http" - "io/ioutil" - "url" - "log" + "io/ioutil" + "log" + "net/http" + "net/url" ) -func Post(url string, values url.Values)[]byte{ - r, err := http.PostForm(url, values) - if err != nil { - log.Println("post:", err) - return nil - } - defer r.Body.Close() - b, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Println("post:", err) - return nil - } - return b +func Post(url string, values url.Values) []byte { + r, err := http.PostForm(url, values) + if err != nil { + log.Println("post:", err) + return nil + } + defer r.Body.Close() + b, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Println("post:", err) + return nil + } + return b }