diff --git a/weed-fs/src/cmd/weed/master.go b/weed-fs/src/cmd/weed/master.go index d359152f7..5b69b3d12 100644 --- a/weed-fs/src/cmd/weed/master.go +++ b/weed-fs/src/cmd/weed/master.go @@ -47,7 +47,7 @@ func dirLookupHandler(w http.ResponseWriter, r *http.Request) { } volumeId, _ := storage.NewVolumeId(vid) machines := topo.Lookup(volumeId) - if machines == nil { + if machines != nil { ret := []map[string]string{} for _, dn := range *machines { ret = append(ret, map[string]string{"url": dn.Ip + strconv.Itoa(dn.Port), "publicUrl": dn.PublicUrl}) diff --git a/weed-fs/src/cmd/weed/upload.go b/weed-fs/src/cmd/weed/upload.go index 01f3e1658..faed646f1 100644 --- a/weed-fs/src/cmd/weed/upload.go +++ b/weed-fs/src/cmd/weed/upload.go @@ -42,7 +42,7 @@ func assign(count int) (*AssignResult, error) { values := make(url.Values) values.Add("count", strconv.Itoa(count)) values.Add("replication", *uploadReplication) - jsonBlob, err := util.Post("http://"+*server+"/dir/assign2", values) + jsonBlob, err := util.Post("http://"+*server+"/dir/assign", values) if *IsDebug { fmt.Println("debug", *IsDebug, "assign result :", string(jsonBlob)) } diff --git a/weed-fs/src/cmd/weed/volume.go b/weed-fs/src/cmd/weed/volume.go index 9c8302629..a6d3e613a 100644 --- a/weed-fs/src/cmd/weed/volume.go +++ b/weed-fs/src/cmd/weed/volume.go @@ -79,6 +79,7 @@ func GetHandler(w http.ResponseWriter, r *http.Request) { if err == nil { http.Redirect(w, r, "http://"+lookupResult.Locations[0].PublicUrl+r.URL.Path, http.StatusMovedPermanently) } else { + log.Println("lookup error:", err) w.WriteHeader(http.StatusNotFound) } return @@ -89,6 +90,7 @@ func GetHandler(w http.ResponseWriter, r *http.Request) { log.Println("read bytes", count, "error", e) } if e != nil || count <= 0 { + log.Println("read error:", e) w.WriteHeader(http.StatusNotFound) return } diff --git a/weed-fs/src/pkg/storage/crc.go b/weed-fs/src/pkg/storage/crc.go new file mode 100644 index 000000000..198352e68 --- /dev/null +++ b/weed-fs/src/pkg/storage/crc.go @@ -0,0 +1,21 @@ +package storage + +import ( + "hash/crc32" +) + +var table = crc32.MakeTable(crc32.Castagnoli) + +type CRC uint32 + +func NewCRC(b []byte) CRC { + return CRC(0).Update(b) +} + +func (c CRC) Update(b []byte) CRC { + return CRC(crc32.Update(uint32(c), table, b)) +} + +func (c CRC) Value() uint32 { + return uint32(c>>15|c<<17) + 0xa282ead8 +} diff --git a/weed-fs/src/pkg/storage/needle.go b/weed-fs/src/pkg/storage/needle.go index d4db281dc..49cceb1d6 100644 --- a/weed-fs/src/pkg/storage/needle.go +++ b/weed-fs/src/pkg/storage/needle.go @@ -2,6 +2,7 @@ package storage import ( "encoding/hex" + "errors" "fmt" "io" "io/ioutil" @@ -18,7 +19,7 @@ type Needle struct { Id uint64 "needle id" Size uint32 "Data size" Data []byte "The actual file data" - Checksum int32 "CRC32 to check integrity" + Checksum CRC "CRC32 to check integrity" Padding []byte "Aligned to 8 bytes" } @@ -44,6 +45,7 @@ func NewNeedle(r *http.Request) (n *Needle, fname string, e error) { } } n.Data = data + n.Checksum = NewCRC(data) commaSep := strings.LastIndex(r.URL.Path, ",") dotSep := strings.LastIndex(r.URL.Path, ".") @@ -86,7 +88,8 @@ func (n *Needle) Append(w io.Writer) uint32 { w.Write(header) w.Write(n.Data) rest := 8 - ((n.Size + 16 + 4) % 8) - util.Uint32toBytes(header[0:4], uint32(n.Checksum)) + util.Uint32toBytes(header[0:4], n.Checksum.Value()) + println("writing checksum", n.Checksum.Value(), "=>", util.BytesToUint32(header[0:4]), "for", n.Id) w.Write(header[0 : rest+4]) return n.Size } @@ -97,7 +100,10 @@ func (n *Needle) Read(r io.Reader, size uint32) (int, error) { n.Id = util.BytesToUint64(bytes[4:12]) n.Size = util.BytesToUint32(bytes[12:16]) n.Data = bytes[16 : 16+size] - n.Checksum = int32(util.BytesToUint32(bytes[16+size : 16+size+4])) + checksum := util.BytesToUint32(bytes[16+size : 16+size+4]) + if checksum != NewCRC(n.Data).Value() { + return 0, errors.New("CRC error! Data On Disk Corrupted!") + } return ret, e } func ReadNeedle(r *os.File) (*Needle, uint32) {