Chris Lu
12 years ago
36 changed files with 466 additions and 622 deletions
-
96weed-fs/src/cmd/dump/main.go
-
1weed-fs/src/cmd/weed/command.go
-
15weed-fs/src/cmd/weed/shell.go
-
4weed-fs/src/pkg/directory/file_id.go
-
2weed-fs/src/pkg/operation/delete_content.go
-
6weed-fs/src/pkg/operation/lookup_volume_id.go
-
4weed-fs/src/pkg/operation/upload_content.go
-
2weed-fs/src/pkg/replication/volume_growth.go
-
9weed-fs/src/pkg/replication/volume_growth_test.go
-
2weed-fs/src/pkg/sequence/sequence.go
-
20weed-fs/src/pkg/storage/compact_map.go
-
2weed-fs/src/pkg/storage/compact_map_perf_test.go
-
12weed-fs/src/pkg/storage/compact_map_test.go
-
5weed-fs/src/pkg/storage/needle_map.go
-
9weed-fs/src/pkg/storage/needle_read_write.go
-
8weed-fs/src/pkg/storage/replication_type.go
-
9weed-fs/src/pkg/storage/store.go
-
53weed-fs/src/pkg/storage/volume.go
-
9weed-fs/src/pkg/storage/volume_id.go
-
3weed-fs/src/pkg/storage/volume_version.go
-
6weed-fs/src/pkg/topology/configuration_test.go
-
5weed-fs/src/pkg/topology/data_center.go
-
4weed-fs/src/pkg/topology/node_list.go
-
14weed-fs/src/pkg/topology/node_list_test.go
-
6weed-fs/src/pkg/topology/topo_test.go
-
2weed-fs/src/pkg/topology/topology_compact.go
-
3weed-fs/src/pkg/topology/topology_map.go
-
2weed-fs/src/pkg/topology/volume_location.go
-
27weed-fs/src/pkg/util/bytes.go
-
8weed-fs/src/pkg/util/parse.go
@ -1,96 +0,0 @@ |
|||||
// Copyright Tamás Gulácsi 2013 All rights reserved
|
|
||||
// Use of this source is governed by the same rules as the weed-fs library.
|
|
||||
// If this would be ambigous, than Apache License 2.0 has to be used.
|
|
||||
//
|
|
||||
// dump dumps the files of a volume to tar or unique files.
|
|
||||
// Each file will have id#mimetype#original_name file format
|
|
||||
|
|
||||
package main |
|
||||
|
|
||||
import ( |
|
||||
"archive/tar" |
|
||||
"bytes" |
|
||||
"flag" |
|
||||
"fmt" |
|
||||
// "io"
|
|
||||
"log" |
|
||||
"os" |
|
||||
"pkg/storage" |
|
||||
"strings" |
|
||||
"time" |
|
||||
) |
|
||||
|
|
||||
var ( |
|
||||
volumePath = flag.String("dir", "/tmp", "volume directory") |
|
||||
volumeId = flag.Int("id", 0, "volume Id") |
|
||||
dest = flag.String("out", "-", "output path. Produces tar if path ends with .tar; creates files otherwise.") |
|
||||
tarFh *tar.Writer |
|
||||
tarHeader tar.Header |
|
||||
counter int |
|
||||
) |
|
||||
|
|
||||
func main() { |
|
||||
var err error |
|
||||
|
|
||||
flag.Parse() |
|
||||
|
|
||||
if *dest == "-" { |
|
||||
*dest = "" |
|
||||
} |
|
||||
if *dest == "" || strings.HasSuffix(*dest, ".tar") { |
|
||||
var fh *os.File |
|
||||
if *dest == "" { |
|
||||
fh = os.Stdout |
|
||||
} else { |
|
||||
if fh, err = os.Create(*dest); err != nil { |
|
||||
log.Printf("cannot open output tar %s: %s", *dest, err) |
|
||||
return |
|
||||
} |
|
||||
} |
|
||||
defer fh.Close() |
|
||||
tarFh = tar.NewWriter(fh) |
|
||||
defer tarFh.Close() |
|
||||
t := time.Now() |
|
||||
tarHeader = tar.Header{Mode: 0644, |
|
||||
ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(), |
|
||||
Typeflag: tar.TypeReg, |
|
||||
AccessTime: t, ChangeTime: t} |
|
||||
} |
|
||||
|
|
||||
v, err := storage.NewVolume(*volumePath, storage.VolumeId(*volumeId), storage.CopyNil) |
|
||||
if v == nil || v.Version() == 0 || err != nil { |
|
||||
log.Printf("cannot load volume %d from %s (%s): %s", *volumeId, *volumePath, v, err) |
|
||||
return |
|
||||
} |
|
||||
log.Printf("volume: %s (ver. %d)", v, v.Version()) |
|
||||
if err := v.WalkValues(walker); err != nil { |
|
||||
log.Printf("error while walking: %s", err) |
|
||||
return |
|
||||
} |
|
||||
|
|
||||
log.Printf("%d files written.", counter) |
|
||||
} |
|
||||
|
|
||||
func walker(n *storage.Needle) (err error) { |
|
||||
// log.Printf("Id=%d Size=%d Name=%s mime=%s", n.Id, n.Size, n.Name, n.Mime)
|
|
||||
nm := fmt.Sprintf("%d#%s#%s", n.Id, bytes.Replace(n.Mime, []byte{'/'}, []byte{'_'}, -1), n.Name) |
|
||||
// log.Print(nm)
|
|
||||
if tarFh != nil { |
|
||||
tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data)) |
|
||||
if err = tarFh.WriteHeader(&tarHeader); err != nil { |
|
||||
return err |
|
||||
} |
|
||||
_, err = tarFh.Write(n.Data) |
|
||||
} else { |
|
||||
if fh, e := os.Create(*dest + "/" + nm); e != nil { |
|
||||
return e |
|
||||
} else { |
|
||||
defer fh.Close() |
|
||||
_, err = fh.Write(n.Data) |
|
||||
} |
|
||||
} |
|
||||
if err == nil { |
|
||||
counter++ |
|
||||
} |
|
||||
return |
|
||||
} |
|
||||
@ -1,38 +1,38 @@ |
|||||
package topology |
package topology |
||||
|
|
||||
import ( |
import ( |
||||
_ "fmt" |
|
||||
"strconv" |
"strconv" |
||||
"testing" |
"testing" |
||||
|
_ "fmt" |
||||
) |
) |
||||
|
|
||||
func TestXYZ(t *testing.T) { |
func TestXYZ(t *testing.T) { |
||||
topo := NewTopology("topo", "/etc/weed.conf", "/tmp", "test", 234, 5) |
|
||||
|
topo := NewTopology("topo","/etc/weed.conf", "/tmp","test",234,5) |
||||
for i := 0; i < 5; i++ { |
for i := 0; i < 5; i++ { |
||||
dc := NewDataCenter("dc" + strconv.Itoa(i)) |
dc := NewDataCenter("dc" + strconv.Itoa(i)) |
||||
dc.activeVolumeCount = i |
dc.activeVolumeCount = i |
||||
dc.maxVolumeCount = 5 |
dc.maxVolumeCount = 5 |
||||
topo.LinkChildNode(dc) |
topo.LinkChildNode(dc) |
||||
} |
} |
||||
nl := NewNodeList(topo.Children(), nil) |
|
||||
|
nl := NewNodeList(topo.Children(),nil) |
||||
|
|
||||
picked, ret := nl.RandomlyPickN(1) |
picked, ret := nl.RandomlyPickN(1) |
||||
if !ret || len(picked) != 1 { |
|
||||
|
if !ret || len(picked)!=1 { |
||||
t.Errorf("need to randomly pick 1 node") |
t.Errorf("need to randomly pick 1 node") |
||||
} |
} |
||||
|
|
||||
picked, ret = nl.RandomlyPickN(4) |
picked, ret = nl.RandomlyPickN(4) |
||||
if !ret || len(picked) != 4 { |
|
||||
|
if !ret || len(picked)!=4 { |
||||
t.Errorf("need to randomly pick 4 nodes") |
t.Errorf("need to randomly pick 4 nodes") |
||||
} |
} |
||||
|
|
||||
picked, ret = nl.RandomlyPickN(5) |
picked, ret = nl.RandomlyPickN(5) |
||||
if !ret || len(picked) != 5 { |
|
||||
|
if !ret || len(picked)!=5 { |
||||
t.Errorf("need to randomly pick 5 nodes") |
t.Errorf("need to randomly pick 5 nodes") |
||||
} |
} |
||||
|
|
||||
picked, ret = nl.RandomlyPickN(6) |
picked, ret = nl.RandomlyPickN(6) |
||||
if ret || len(picked) != 0 { |
|
||||
|
if ret || len(picked)!=0 { |
||||
t.Errorf("can not randomly pick 6 nodes:", ret, picked) |
t.Errorf("can not randomly pick 6 nodes:", ret, picked) |
||||
} |
} |
||||
|
|
||||
|
|||||
@ -1,33 +1,34 @@ |
|||||
package util |
package util |
||||
|
|
||||
func BytesToUint64(b []byte) (v uint64) { |
|
||||
|
func BytesToUint64(b []byte)(v uint64){ |
||||
length := uint(len(b)) |
length := uint(len(b)) |
||||
for i := uint(0); i < length-1; i++ { |
|
||||
|
for i :=uint(0);i<length-1;i++ { |
||||
v += uint64(b[i]) |
v += uint64(b[i]) |
||||
v <<= 8 |
v <<= 8 |
||||
} |
} |
||||
v += uint64(b[length-1]) |
|
||||
|
v+=uint64(b[length-1]) |
||||
return |
return |
||||
} |
} |
||||
func BytesToUint32(b []byte) (v uint32) { |
|
||||
|
func BytesToUint32(b []byte)(v uint32){ |
||||
length := uint(len(b)) |
length := uint(len(b)) |
||||
for i := uint(0); i < length-1; i++ { |
|
||||
|
for i :=uint(0);i<length-1;i++ { |
||||
v += uint32(b[i]) |
v += uint32(b[i]) |
||||
v <<= 8 |
v <<= 8 |
||||
} |
} |
||||
v += uint32(b[length-1]) |
|
||||
|
v+=uint32(b[length-1]) |
||||
return |
return |
||||
} |
} |
||||
func Uint64toBytes(b []byte, v uint64) { |
|
||||
for i := uint(0); i < 8; i++ { |
|
||||
b[7-i] = byte(v >> (i * 8)) |
|
||||
|
func Uint64toBytes(b []byte, v uint64){ |
||||
|
for i :=uint(0);i<8;i++ { |
||||
|
b[7-i] = byte(v>>(i*8)) |
||||
} |
} |
||||
} |
} |
||||
func Uint32toBytes(b []byte, v uint32) { |
|
||||
for i := uint(0); i < 4; i++ { |
|
||||
b[3-i] = byte(v >> (i * 8)) |
|
||||
|
func Uint32toBytes(b []byte, v uint32){ |
||||
|
for i :=uint(0);i<4;i++ { |
||||
|
b[3-i] = byte(v>>(i*8)) |
||||
} |
} |
||||
} |
} |
||||
func Uint8toBytes(b []byte, v uint8) { |
|
||||
|
func Uint8toBytes(b []byte, v uint8){ |
||||
b[0] = byte(v) |
b[0] = byte(v) |
||||
} |
} |
||||
|
|
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue