Browse Source
add cmd/dump - a dumper
add cmd/dump - a dumper
Walk needed to be added to NeedleMap and CompactMap, to be able to add WalkKeys and WalkValues to volume. This is needed for iterating through all the stored needles in a volume - this was dump's purpose.pull/2/head
Tamás Gulácsi
12 years ago
36 changed files with 622 additions and 466 deletions
-
96weed-fs/src/cmd/dump/main.go
-
59weed-fs/src/cmd/weed/command.go
-
73weed-fs/src/cmd/weed/shell.go
-
2weed-fs/src/cmd/weed/upload.go
-
4weed-fs/src/cmd/weed/weed.go
-
6weed-fs/src/pkg/directory/file_id.go
-
44weed-fs/src/pkg/operation/allocate_volume.go
-
2weed-fs/src/pkg/operation/delete_content.go
-
50weed-fs/src/pkg/operation/lookup_volume_id.go
-
18weed-fs/src/pkg/operation/upload_content.go
-
10weed-fs/src/pkg/replication/volume_growth.go
-
17weed-fs/src/pkg/replication/volume_growth_test.go
-
52weed-fs/src/pkg/sequence/sequence.go
-
20weed-fs/src/pkg/storage/compact_map.go
-
58weed-fs/src/pkg/storage/compact_map_perf_test.go
-
36weed-fs/src/pkg/storage/compact_map_test.go
-
5weed-fs/src/pkg/storage/needle_map.go
-
9weed-fs/src/pkg/storage/needle_read_write.go
-
202weed-fs/src/pkg/storage/replication_type.go
-
13weed-fs/src/pkg/storage/store.go
-
53weed-fs/src/pkg/storage/volume.go
-
17weed-fs/src/pkg/storage/volume_id.go
-
9weed-fs/src/pkg/storage/volume_version.go
-
8weed-fs/src/pkg/topology/configuration_test.go
-
45weed-fs/src/pkg/topology/data_center.go
-
12weed-fs/src/pkg/topology/node_list.go
-
34weed-fs/src/pkg/topology/node_list_test.go
-
14weed-fs/src/pkg/topology/rack.go
-
10weed-fs/src/pkg/topology/topo_test.go
-
4weed-fs/src/pkg/topology/topology_compact.go
-
4weed-fs/src/pkg/topology/topology_event_handling.go
-
3weed-fs/src/pkg/topology/topology_map.go
-
16weed-fs/src/pkg/topology/volume_location.go
-
53weed-fs/src/pkg/util/bytes.go
-
20weed-fs/src/pkg/util/parse.go
-
2weed-fs/src/pkg/util/post.go
@ -0,0 +1,96 @@ |
|||
// Copyright Tamás Gulácsi 2013 All rights reserved
|
|||
// Use of this source is governed by the same rules as the weed-fs library.
|
|||
// If this would be ambigous, than Apache License 2.0 has to be used.
|
|||
//
|
|||
// dump dumps the files of a volume to tar or unique files.
|
|||
// Each file will have id#mimetype#original_name file format
|
|||
|
|||
package main |
|||
|
|||
import ( |
|||
"archive/tar" |
|||
"bytes" |
|||
"flag" |
|||
"fmt" |
|||
// "io"
|
|||
"log" |
|||
"os" |
|||
"pkg/storage" |
|||
"strings" |
|||
"time" |
|||
) |
|||
|
|||
var ( |
|||
volumePath = flag.String("dir", "/tmp", "volume directory") |
|||
volumeId = flag.Int("id", 0, "volume Id") |
|||
dest = flag.String("out", "-", "output path. Produces tar if path ends with .tar; creates files otherwise.") |
|||
tarFh *tar.Writer |
|||
tarHeader tar.Header |
|||
counter int |
|||
) |
|||
|
|||
func main() { |
|||
var err error |
|||
|
|||
flag.Parse() |
|||
|
|||
if *dest == "-" { |
|||
*dest = "" |
|||
} |
|||
if *dest == "" || strings.HasSuffix(*dest, ".tar") { |
|||
var fh *os.File |
|||
if *dest == "" { |
|||
fh = os.Stdout |
|||
} else { |
|||
if fh, err = os.Create(*dest); err != nil { |
|||
log.Printf("cannot open output tar %s: %s", *dest, err) |
|||
return |
|||
} |
|||
} |
|||
defer fh.Close() |
|||
tarFh = tar.NewWriter(fh) |
|||
defer tarFh.Close() |
|||
t := time.Now() |
|||
tarHeader = tar.Header{Mode: 0644, |
|||
ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(), |
|||
Typeflag: tar.TypeReg, |
|||
AccessTime: t, ChangeTime: t} |
|||
} |
|||
|
|||
v, err := storage.NewVolume(*volumePath, storage.VolumeId(*volumeId), storage.CopyNil) |
|||
if v == nil || v.Version() == 0 || err != nil { |
|||
log.Printf("cannot load volume %d from %s (%s): %s", *volumeId, *volumePath, v, err) |
|||
return |
|||
} |
|||
log.Printf("volume: %s (ver. %d)", v, v.Version()) |
|||
if err := v.WalkValues(walker); err != nil { |
|||
log.Printf("error while walking: %s", err) |
|||
return |
|||
} |
|||
|
|||
log.Printf("%d files written.", counter) |
|||
} |
|||
|
|||
func walker(n *storage.Needle) (err error) { |
|||
// log.Printf("Id=%d Size=%d Name=%s mime=%s", n.Id, n.Size, n.Name, n.Mime)
|
|||
nm := fmt.Sprintf("%d#%s#%s", n.Id, bytes.Replace(n.Mime, []byte{'/'}, []byte{'_'}, -1), n.Name) |
|||
// log.Print(nm)
|
|||
if tarFh != nil { |
|||
tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data)) |
|||
if err = tarFh.WriteHeader(&tarHeader); err != nil { |
|||
return err |
|||
} |
|||
_, err = tarFh.Write(n.Data) |
|||
} else { |
|||
if fh, e := os.Create(*dest + "/" + nm); e != nil { |
|||
return e |
|||
} else { |
|||
defer fh.Close() |
|||
_, err = fh.Write(n.Data) |
|||
} |
|||
} |
|||
if err == nil { |
|||
counter++ |
|||
} |
|||
return |
|||
} |
@ -1,53 +1,52 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"flag" |
|||
"fmt" |
|||
"os" |
|||
"strings" |
|||
"flag" |
|||
"fmt" |
|||
"os" |
|||
"strings" |
|||
) |
|||
|
|||
type Command struct { |
|||
// Run runs the command.
|
|||
// The args are the arguments after the command name.
|
|||
Run func(cmd *Command, args []string) bool |
|||
// Run runs the command.
|
|||
// The args are the arguments after the command name.
|
|||
Run func(cmd *Command, args []string) bool |
|||
|
|||
// UsageLine is the one-line usage message.
|
|||
// The first word in the line is taken to be the command name.
|
|||
UsageLine string |
|||
// UsageLine is the one-line usage message.
|
|||
// The first word in the line is taken to be the command name.
|
|||
UsageLine string |
|||
|
|||
// Short is the short description shown in the 'go help' output.
|
|||
Short string |
|||
// Short is the short description shown in the 'go help' output.
|
|||
Short string |
|||
|
|||
// Long is the long message shown in the 'go help <this-command>' output.
|
|||
Long string |
|||
|
|||
// Flag is a set of flags specific to this command.
|
|||
Flag flag.FlagSet |
|||
// Long is the long message shown in the 'go help <this-command>' output.
|
|||
Long string |
|||
|
|||
// Flag is a set of flags specific to this command.
|
|||
Flag flag.FlagSet |
|||
} |
|||
|
|||
// Name returns the command's name: the first word in the usage line.
|
|||
func (c *Command) Name() string { |
|||
name := c.UsageLine |
|||
i := strings.Index(name, " ") |
|||
if i >= 0 { |
|||
name = name[:i] |
|||
} |
|||
return name |
|||
name := c.UsageLine |
|||
i := strings.Index(name, " ") |
|||
if i >= 0 { |
|||
name = name[:i] |
|||
} |
|||
return name |
|||
} |
|||
|
|||
func (c *Command) Usage() { |
|||
fmt.Fprintf(os.Stderr, "Example: weed %s\n", c.UsageLine) |
|||
fmt.Fprintf(os.Stderr, "Default Usage:\n") |
|||
c.Flag.PrintDefaults() |
|||
fmt.Fprintf(os.Stderr, "Description:\n") |
|||
fmt.Fprintf(os.Stderr, " %s\n", strings.TrimSpace(c.Long)) |
|||
os.Exit(2) |
|||
fmt.Fprintf(os.Stderr, "Example: weed %s\n", c.UsageLine) |
|||
fmt.Fprintf(os.Stderr, "Default Usage:\n") |
|||
c.Flag.PrintDefaults() |
|||
fmt.Fprintf(os.Stderr, "Description:\n") |
|||
fmt.Fprintf(os.Stderr, " %s\n", strings.TrimSpace(c.Long)) |
|||
os.Exit(2) |
|||
} |
|||
|
|||
// Runnable reports whether the command can be run; otherwise
|
|||
// it is a documentation pseudo-command such as importpath.
|
|||
func (c *Command) Runnable() bool { |
|||
return c.Run != nil |
|||
return c.Run != nil |
|||
} |
@ -1,54 +1,53 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"bufio" |
|||
"os" |
|||
"fmt" |
|||
"bufio" |
|||
"fmt" |
|||
"os" |
|||
) |
|||
|
|||
func init() { |
|||
cmdShell.Run = runShell // break init cycle
|
|||
cmdShell.Run = runShell // break init cycle
|
|||
} |
|||
|
|||
var cmdShell = &Command{ |
|||
UsageLine: "shell", |
|||
Short: "run interactive commands, now just echo", |
|||
Long: `run interactive commands. |
|||
UsageLine: "shell", |
|||
Short: "run interactive commands, now just echo", |
|||
Long: `run interactive commands. |
|||
|
|||
`, |
|||
} |
|||
|
|||
var ( |
|||
) |
|||
var () |
|||
|
|||
func runShell(command *Command, args []string) bool { |
|||
r := bufio.NewReader(os.Stdin) |
|||
o := bufio.NewWriter(os.Stdout) |
|||
e := bufio.NewWriter(os.Stderr) |
|||
prompt := func () { |
|||
o.WriteString("> ") |
|||
o.Flush() |
|||
}; |
|||
readLine := func () string { |
|||
ret, err := r.ReadString('\n') |
|||
if err != nil { |
|||
fmt.Fprint(e,err); |
|||
os.Exit(1) |
|||
} |
|||
return ret |
|||
} |
|||
execCmd := func (cmd string) int { |
|||
if cmd != "" { |
|||
o.WriteString(cmd) |
|||
} |
|||
return 0 |
|||
} |
|||
r := bufio.NewReader(os.Stdin) |
|||
o := bufio.NewWriter(os.Stdout) |
|||
e := bufio.NewWriter(os.Stderr) |
|||
prompt := func() { |
|||
o.WriteString("> ") |
|||
o.Flush() |
|||
} |
|||
readLine := func() string { |
|||
ret, err := r.ReadString('\n') |
|||
if err != nil { |
|||
fmt.Fprint(e, err) |
|||
os.Exit(1) |
|||
} |
|||
return ret |
|||
} |
|||
execCmd := func(cmd string) int { |
|||
if cmd != "" { |
|||
o.WriteString(cmd) |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
cmd := "" |
|||
for { |
|||
prompt() |
|||
cmd = readLine() |
|||
execCmd(cmd) |
|||
} |
|||
return true |
|||
cmd := "" |
|||
for { |
|||
prompt() |
|||
cmd = readLine() |
|||
execCmd(cmd) |
|||
} |
|||
return true |
|||
} |
@ -1,32 +1,32 @@ |
|||
package operation |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
"errors" |
|||
"net/url" |
|||
"pkg/storage" |
|||
"pkg/topology" |
|||
"pkg/util" |
|||
"encoding/json" |
|||
"errors" |
|||
"net/url" |
|||
"pkg/storage" |
|||
"pkg/topology" |
|||
"pkg/util" |
|||
) |
|||
|
|||
type AllocateVolumeResult struct { |
|||
Error string |
|||
Error string |
|||
} |
|||
|
|||
func AllocateVolume(dn *topology.DataNode, vid storage.VolumeId, repType storage.ReplicationType) error { |
|||
values := make(url.Values) |
|||
values.Add("volume", vid.String()) |
|||
values.Add("replicationType", repType.String()) |
|||
jsonBlob, err := util.Post("http://"+dn.Url()+"/admin/assign_volume", values) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
var ret AllocateVolumeResult |
|||
if err := json.Unmarshal(jsonBlob, &ret); err != nil { |
|||
return err |
|||
} |
|||
if ret.Error != "" { |
|||
return errors.New(ret.Error) |
|||
} |
|||
return nil |
|||
values := make(url.Values) |
|||
values.Add("volume", vid.String()) |
|||
values.Add("replicationType", repType.String()) |
|||
jsonBlob, err := util.Post("http://"+dn.Url()+"/admin/assign_volume", values) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
var ret AllocateVolumeResult |
|||
if err := json.Unmarshal(jsonBlob, &ret); err != nil { |
|||
return err |
|||
} |
|||
if ret.Error != "" { |
|||
return errors.New(ret.Error) |
|||
} |
|||
return nil |
|||
} |
@ -1,38 +1,38 @@ |
|||
package operation |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
"net/url" |
|||
"pkg/storage" |
|||
"pkg/util" |
|||
_ "fmt" |
|||
"errors" |
|||
"encoding/json" |
|||
"errors" |
|||
_ "fmt" |
|||
"net/url" |
|||
"pkg/storage" |
|||
"pkg/util" |
|||
) |
|||
|
|||
type Location struct { |
|||
Url string "url" |
|||
PublicUrl string "publicUrl" |
|||
Url string "url" |
|||
PublicUrl string "publicUrl" |
|||
} |
|||
type LookupResult struct { |
|||
Locations []Location "locations" |
|||
Error string "error" |
|||
Locations []Location "locations" |
|||
Error string "error" |
|||
} |
|||
|
|||
//TODO: Add a caching for vid here
|
|||
func Lookup(server string, vid storage.VolumeId) (*LookupResult, error) { |
|||
values := make(url.Values) |
|||
values.Add("volumeId", vid.String()) |
|||
jsonBlob, err := util.Post("http://"+server+"/dir/lookup", values) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
var ret LookupResult |
|||
err = json.Unmarshal(jsonBlob, &ret) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
if ret.Error != ""{ |
|||
return nil, errors.New(ret.Error) |
|||
} |
|||
return &ret, nil |
|||
values := make(url.Values) |
|||
values.Add("volumeId", vid.String()) |
|||
jsonBlob, err := util.Post("http://"+server+"/dir/lookup", values) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
var ret LookupResult |
|||
err = json.Unmarshal(jsonBlob, &ret) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
if ret.Error != "" { |
|||
return nil, errors.New(ret.Error) |
|||
} |
|||
return &ret, nil |
|||
} |
@ -1,43 +1,43 @@ |
|||
package storage |
|||
|
|||
import ( |
|||
"log" |
|||
"os" |
|||
"pkg/util" |
|||
"testing" |
|||
"log" |
|||
"os" |
|||
"pkg/util" |
|||
) |
|||
|
|||
func TestMemoryUsage(t *testing.T) { |
|||
|
|||
indexFile, ie := os.OpenFile("sample.idx", os.O_RDWR|os.O_RDONLY, 0644) |
|||
if ie != nil { |
|||
log.Fatalln(ie) |
|||
} |
|||
LoadNewNeedleMap(indexFile) |
|||
indexFile, ie := os.OpenFile("sample.idx", os.O_RDWR|os.O_RDONLY, 0644) |
|||
if ie != nil { |
|||
log.Fatalln(ie) |
|||
} |
|||
LoadNewNeedleMap(indexFile) |
|||
|
|||
} |
|||
|
|||
func LoadNewNeedleMap(file *os.File) CompactMap { |
|||
m := NewCompactMap() |
|||
bytes := make([]byte, 16*1024) |
|||
count, e := file.Read(bytes) |
|||
if count > 0 { |
|||
fstat, _ := file.Stat() |
|||
log.Println("Loading index file", fstat.Name(), "size", fstat.Size()) |
|||
} |
|||
for count > 0 && e == nil { |
|||
for i := 0; i < count; i += 16 { |
|||
key := util.BytesToUint64(bytes[i : i+8]) |
|||
offset := util.BytesToUint32(bytes[i+8 : i+12]) |
|||
size := util.BytesToUint32(bytes[i+12 : i+16]) |
|||
if offset > 0 { |
|||
m.Set(Key(key), offset, size) |
|||
} else { |
|||
//delete(m, key)
|
|||
} |
|||
} |
|||
m := NewCompactMap() |
|||
bytes := make([]byte, 16*1024) |
|||
count, e := file.Read(bytes) |
|||
if count > 0 { |
|||
fstat, _ := file.Stat() |
|||
log.Println("Loading index file", fstat.Name(), "size", fstat.Size()) |
|||
} |
|||
for count > 0 && e == nil { |
|||
for i := 0; i < count; i += 16 { |
|||
key := util.BytesToUint64(bytes[i : i+8]) |
|||
offset := util.BytesToUint32(bytes[i+8 : i+12]) |
|||
size := util.BytesToUint32(bytes[i+12 : i+16]) |
|||
if offset > 0 { |
|||
m.Set(Key(key), offset, size) |
|||
} else { |
|||
//delete(m, key)
|
|||
} |
|||
} |
|||
|
|||
count, e = file.Read(bytes) |
|||
} |
|||
return m |
|||
count, e = file.Read(bytes) |
|||
} |
|||
return m |
|||
} |
@ -1,123 +1,123 @@ |
|||
package storage |
|||
|
|||
import ( |
|||
"errors" |
|||
"errors" |
|||
) |
|||
|
|||
type ReplicationType string |
|||
|
|||
const ( |
|||
Copy000 = ReplicationType("000") // single copy
|
|||
Copy001 = ReplicationType("001") // 2 copies, both on the same racks, and same data center
|
|||
Copy010 = ReplicationType("010") // 2 copies, both on different racks, but same data center
|
|||
Copy100 = ReplicationType("100") // 2 copies, each on different data center
|
|||
Copy110 = ReplicationType("110") // 3 copies, 2 on different racks and local data center, 1 on different data center
|
|||
Copy200 = ReplicationType("200") // 3 copies, each on dffereint data center
|
|||
LengthRelicationType = 6 |
|||
CopyNil = ReplicationType(255) // nil value
|
|||
Copy000 = ReplicationType("000") // single copy
|
|||
Copy001 = ReplicationType("001") // 2 copies, both on the same racks, and same data center
|
|||
Copy010 = ReplicationType("010") // 2 copies, both on different racks, but same data center
|
|||
Copy100 = ReplicationType("100") // 2 copies, each on different data center
|
|||
Copy110 = ReplicationType("110") // 3 copies, 2 on different racks and local data center, 1 on different data center
|
|||
Copy200 = ReplicationType("200") // 3 copies, each on dffereint data center
|
|||
LengthRelicationType = 6 |
|||
CopyNil = ReplicationType(255) // nil value
|
|||
) |
|||
|
|||
func NewReplicationTypeFromString(t string) (ReplicationType, error) { |
|||
switch t { |
|||
case "000": |
|||
return Copy000, nil |
|||
case "001": |
|||
return Copy001, nil |
|||
case "010": |
|||
return Copy010, nil |
|||
case "100": |
|||
return Copy100, nil |
|||
case "110": |
|||
return Copy110, nil |
|||
case "200": |
|||
return Copy200, nil |
|||
} |
|||
return Copy000, errors.New("Unknown Replication Type:"+t) |
|||
switch t { |
|||
case "000": |
|||
return Copy000, nil |
|||
case "001": |
|||
return Copy001, nil |
|||
case "010": |
|||
return Copy010, nil |
|||
case "100": |
|||
return Copy100, nil |
|||
case "110": |
|||
return Copy110, nil |
|||
case "200": |
|||
return Copy200, nil |
|||
} |
|||
return Copy000, errors.New("Unknown Replication Type:" + t) |
|||
} |
|||
func NewReplicationTypeFromByte(b byte) (ReplicationType, error) { |
|||
switch b { |
|||
case byte(000): |
|||
return Copy000, nil |
|||
case byte(001): |
|||
return Copy001, nil |
|||
case byte(010): |
|||
return Copy010, nil |
|||
case byte(100): |
|||
return Copy100, nil |
|||
case byte(110): |
|||
return Copy110, nil |
|||
case byte(200): |
|||
return Copy200, nil |
|||
} |
|||
return Copy000, errors.New("Unknown Replication Type:"+string(b)) |
|||
switch b { |
|||
case byte(000): |
|||
return Copy000, nil |
|||
case byte(001): |
|||
return Copy001, nil |
|||
case byte(010): |
|||
return Copy010, nil |
|||
case byte(100): |
|||
return Copy100, nil |
|||
case byte(110): |
|||
return Copy110, nil |
|||
case byte(200): |
|||
return Copy200, nil |
|||
} |
|||
return Copy000, errors.New("Unknown Replication Type:" + string(b)) |
|||
} |
|||
|
|||
func (r *ReplicationType) String() string { |
|||
switch *r { |
|||
case Copy000: |
|||
return "000" |
|||
case Copy001: |
|||
return "001" |
|||
case Copy010: |
|||
return "010" |
|||
case Copy100: |
|||
return "100" |
|||
case Copy110: |
|||
return "110" |
|||
case Copy200: |
|||
return "200" |
|||
} |
|||
return "000" |
|||
switch *r { |
|||
case Copy000: |
|||
return "000" |
|||
case Copy001: |
|||
return "001" |
|||
case Copy010: |
|||
return "010" |
|||
case Copy100: |
|||
return "100" |
|||
case Copy110: |
|||
return "110" |
|||
case Copy200: |
|||
return "200" |
|||
} |
|||
return "000" |
|||
} |
|||
func (r *ReplicationType) Byte() byte { |
|||
switch *r { |
|||
case Copy000: |
|||
return byte(000) |
|||
case Copy001: |
|||
return byte(001) |
|||
case Copy010: |
|||
return byte(010) |
|||
case Copy100: |
|||
return byte(100) |
|||
case Copy110: |
|||
return byte(110) |
|||
case Copy200: |
|||
return byte(200) |
|||
} |
|||
return byte(000) |
|||
switch *r { |
|||
case Copy000: |
|||
return byte(000) |
|||
case Copy001: |
|||
return byte(001) |
|||
case Copy010: |
|||
return byte(010) |
|||
case Copy100: |
|||
return byte(100) |
|||
case Copy110: |
|||
return byte(110) |
|||
case Copy200: |
|||
return byte(200) |
|||
} |
|||
return byte(000) |
|||
} |
|||
|
|||
func (repType ReplicationType)GetReplicationLevelIndex() int { |
|||
switch repType { |
|||
case Copy000: |
|||
return 0 |
|||
case Copy001: |
|||
return 1 |
|||
case Copy010: |
|||
return 2 |
|||
case Copy100: |
|||
return 3 |
|||
case Copy110: |
|||
return 4 |
|||
case Copy200: |
|||
return 5 |
|||
} |
|||
return -1 |
|||
func (repType ReplicationType) GetReplicationLevelIndex() int { |
|||
switch repType { |
|||
case Copy000: |
|||
return 0 |
|||
case Copy001: |
|||
return 1 |
|||
case Copy010: |
|||
return 2 |
|||
case Copy100: |
|||
return 3 |
|||
case Copy110: |
|||
return 4 |
|||
case Copy200: |
|||
return 5 |
|||
} |
|||
return -1 |
|||
} |
|||
func (repType ReplicationType)GetCopyCount() int { |
|||
switch repType { |
|||
case Copy000: |
|||
return 1 |
|||
case Copy001: |
|||
return 2 |
|||
case Copy010: |
|||
return 2 |
|||
case Copy100: |
|||
return 2 |
|||
case Copy110: |
|||
return 3 |
|||
case Copy200: |
|||
return 3 |
|||
} |
|||
return 0 |
|||
func (repType ReplicationType) GetCopyCount() int { |
|||
switch repType { |
|||
case Copy000: |
|||
return 1 |
|||
case Copy001: |
|||
return 2 |
|||
case Copy010: |
|||
return 2 |
|||
case Copy100: |
|||
return 2 |
|||
case Copy110: |
|||
return 3 |
|||
case Copy200: |
|||
return 3 |
|||
} |
|||
return 0 |
|||
} |
@ -1,17 +1,18 @@ |
|||
package storage |
|||
|
|||
import ( |
|||
"strconv" |
|||
"strconv" |
|||
) |
|||
|
|||
type VolumeId uint32 |
|||
func NewVolumeId(vid string) (VolumeId,error) { |
|||
volumeId, err := strconv.ParseUint(vid, 10, 64) |
|||
return VolumeId(volumeId), err |
|||
|
|||
func NewVolumeId(vid string) (VolumeId, error) { |
|||
volumeId, err := strconv.ParseUint(vid, 10, 64) |
|||
return VolumeId(volumeId), err |
|||
} |
|||
func (vid *VolumeId) String() string{ |
|||
return strconv.FormatUint(uint64(*vid), 10) |
|||
func (vid *VolumeId) String() string { |
|||
return strconv.FormatUint(uint64(*vid), 10) |
|||
} |
|||
func (vid *VolumeId) Next() VolumeId{ |
|||
return VolumeId(uint32(*vid)+1) |
|||
func (vid *VolumeId) Next() VolumeId { |
|||
return VolumeId(uint32(*vid) + 1) |
|||
} |
@ -1,12 +1,11 @@ |
|||
package storage |
|||
|
|||
import ( |
|||
) |
|||
import () |
|||
|
|||
type Version uint8 |
|||
|
|||
const ( |
|||
Version1 = Version(1) |
|||
Version2 = Version(2) |
|||
CurrentVersion = Version2 |
|||
Version1 = Version(1) |
|||
Version2 = Version(2) |
|||
CurrentVersion = Version2 |
|||
) |
@ -1,39 +1,39 @@ |
|||
package topology |
|||
|
|||
import ( |
|||
_ "fmt" |
|||
"strconv" |
|||
"testing" |
|||
_ "fmt" |
|||
) |
|||
|
|||
func TestXYZ(t *testing.T) { |
|||
topo := NewTopology("topo","/etc/weed.conf", "/tmp","test",234,5) |
|||
topo := NewTopology("topo", "/etc/weed.conf", "/tmp", "test", 234, 5) |
|||
for i := 0; i < 5; i++ { |
|||
dc := NewDataCenter("dc" + strconv.Itoa(i)) |
|||
dc.activeVolumeCount = i |
|||
dc.maxVolumeCount = 5 |
|||
topo.LinkChildNode(dc) |
|||
} |
|||
nl := NewNodeList(topo.Children(),nil) |
|||
nl := NewNodeList(topo.Children(), nil) |
|||
|
|||
picked, ret := nl.RandomlyPickN(1) |
|||
if !ret || len(picked)!=1 { |
|||
t.Errorf("need to randomly pick 1 node") |
|||
} |
|||
picked, ret := nl.RandomlyPickN(1) |
|||
if !ret || len(picked) != 1 { |
|||
t.Errorf("need to randomly pick 1 node") |
|||
} |
|||
|
|||
picked, ret = nl.RandomlyPickN(4) |
|||
if !ret || len(picked)!=4 { |
|||
t.Errorf("need to randomly pick 4 nodes") |
|||
if !ret || len(picked) != 4 { |
|||
t.Errorf("need to randomly pick 4 nodes") |
|||
} |
|||
|
|||
picked, ret = nl.RandomlyPickN(5) |
|||
if !ret || len(picked)!=5 { |
|||
t.Errorf("need to randomly pick 5 nodes") |
|||
} |
|||
picked, ret = nl.RandomlyPickN(5) |
|||
if !ret || len(picked) != 5 { |
|||
t.Errorf("need to randomly pick 5 nodes") |
|||
} |
|||
|
|||
picked, ret = nl.RandomlyPickN(6) |
|||
if ret || len(picked)!=0 { |
|||
t.Errorf("can not randomly pick 6 nodes:", ret, picked) |
|||
} |
|||
picked, ret = nl.RandomlyPickN(6) |
|||
if ret || len(picked) != 0 { |
|||
t.Errorf("can not randomly pick 6 nodes:", ret, picked) |
|||
} |
|||
|
|||
} |
@ -1,34 +1,33 @@ |
|||
package util |
|||
|
|||
func BytesToUint64(b []byte)(v uint64){ |
|||
length := uint(len(b)) |
|||
for i :=uint(0);i<length-1;i++ { |
|||
v += uint64(b[i]) |
|||
v <<= 8 |
|||
} |
|||
v+=uint64(b[length-1]) |
|||
return |
|||
func BytesToUint64(b []byte) (v uint64) { |
|||
length := uint(len(b)) |
|||
for i := uint(0); i < length-1; i++ { |
|||
v += uint64(b[i]) |
|||
v <<= 8 |
|||
} |
|||
v += uint64(b[length-1]) |
|||
return |
|||
} |
|||
func BytesToUint32(b []byte)(v uint32){ |
|||
length := uint(len(b)) |
|||
for i :=uint(0);i<length-1;i++ { |
|||
v += uint32(b[i]) |
|||
v <<= 8 |
|||
} |
|||
v+=uint32(b[length-1]) |
|||
return |
|||
func BytesToUint32(b []byte) (v uint32) { |
|||
length := uint(len(b)) |
|||
for i := uint(0); i < length-1; i++ { |
|||
v += uint32(b[i]) |
|||
v <<= 8 |
|||
} |
|||
v += uint32(b[length-1]) |
|||
return |
|||
} |
|||
func Uint64toBytes(b []byte, v uint64){ |
|||
for i :=uint(0);i<8;i++ { |
|||
b[7-i] = byte(v>>(i*8)) |
|||
} |
|||
func Uint64toBytes(b []byte, v uint64) { |
|||
for i := uint(0); i < 8; i++ { |
|||
b[7-i] = byte(v >> (i * 8)) |
|||
} |
|||
} |
|||
func Uint32toBytes(b []byte, v uint32){ |
|||
for i :=uint(0);i<4;i++ { |
|||
b[3-i] = byte(v>>(i*8)) |
|||
} |
|||
func Uint32toBytes(b []byte, v uint32) { |
|||
for i := uint(0); i < 4; i++ { |
|||
b[3-i] = byte(v >> (i * 8)) |
|||
} |
|||
} |
|||
func Uint8toBytes(b []byte, v uint8){ |
|||
b[0] = byte(v) |
|||
func Uint8toBytes(b []byte, v uint8) { |
|||
b[0] = byte(v) |
|||
} |
|||
|
@ -1,16 +1,16 @@ |
|||
package util |
|||
|
|||
import ( |
|||
"strconv" |
|||
"strconv" |
|||
) |
|||
|
|||
func ParseInt(text string, defaultValue int) int{ |
|||
count, parseError := strconv.ParseUint(text,10,64) |
|||
if parseError!=nil { |
|||
if len(text)>0{ |
|||
return 0 |
|||
} |
|||
return defaultValue |
|||
} |
|||
return int(count) |
|||
func ParseInt(text string, defaultValue int) int { |
|||
count, parseError := strconv.ParseUint(text, 10, 64) |
|||
if parseError != nil { |
|||
if len(text) > 0 { |
|||
return 0 |
|||
} |
|||
return defaultValue |
|||
} |
|||
return int(count) |
|||
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue