Browse Source
Major:
Major:
change replication_type to ReplicaPlacement, hopefully cleaner code works for 9 possible ReplicaPlacement xyz x : number of copies on other data centers y : number of copies on other racks z : number of copies on current rack x y z each can be 0,1,2 Minor: weed server "-mdir" default to "-dir" if emptypull/2/head
Chris Lu
11 years ago
32 changed files with 371 additions and 595 deletions
-
6go/operation/list_masters.go
-
4go/replication/allocate_volume.go
-
220go/replication/volume_growth.go
-
28go/replication/volume_growth_test.go
-
2go/storage/cdb_map.go
-
2go/storage/compact_map_perf_test.go
-
61go/storage/replica_placement.go
-
123go/storage/replication_type.go
-
18go/storage/store.go
-
16go/storage/volume.go
-
2go/storage/volume_info.go
-
14go/topology/collection.go
-
1go/topology/data_center.go
-
69go/topology/node.go
-
83go/topology/node_list.go
-
60go/topology/node_list_test.go
-
1go/topology/rack.go
-
14go/topology/topo_test.go
-
20go/topology/topology.go
-
6go/topology/topology_event_handling.go
-
16go/topology/volume_layout.go
-
2go/weed/compact.go
-
4go/weed/master.go
-
9go/weed/server.go
-
2go/weed/version.go
-
6go/weed/weed_server/master_server.go
-
24go/weed/weed_server/master_server_handlers.go
-
37note/replication.txt
@ -0,0 +1,61 @@ |
|||
package storage |
|||
|
|||
import ( |
|||
"errors" |
|||
"fmt" |
|||
) |
|||
|
|||
const ( |
|||
ReplicaPlacementCount = 9 |
|||
) |
|||
|
|||
type ReplicaPlacement struct { |
|||
SameRackCount int |
|||
DiffRackCount int |
|||
DiffDataCenterCount int |
|||
} |
|||
|
|||
func NewReplicaPlacementFromString(t string) (*ReplicaPlacement, error) { |
|||
rp := &ReplicaPlacement{} |
|||
for i, c := range t { |
|||
count := int(c - '0') |
|||
if 0 <= count && count <= 2 { |
|||
switch i { |
|||
case 0: |
|||
rp.DiffDataCenterCount = count |
|||
case 1: |
|||
rp.DiffRackCount = count |
|||
case 2: |
|||
rp.SameRackCount = count |
|||
} |
|||
} else { |
|||
return rp, errors.New("Unknown Replication Type:" + t) |
|||
} |
|||
} |
|||
return rp, nil |
|||
} |
|||
|
|||
func NewReplicaPlacementFromByte(b byte) (*ReplicaPlacement, error) { |
|||
return NewReplicaPlacementFromString(fmt.Sprintf("%d", b)) |
|||
} |
|||
|
|||
func (rp *ReplicaPlacement) Byte() byte { |
|||
ret := rp.DiffDataCenterCount*100 + rp.DiffRackCount*10 + rp.SameRackCount |
|||
return byte(ret) |
|||
} |
|||
|
|||
func (rp *ReplicaPlacement) String() string { |
|||
b := make([]byte, 3) |
|||
b[0] = byte(rp.DiffDataCenterCount + '0') |
|||
b[1] = byte(rp.DiffRackCount + '0') |
|||
b[2] = byte(rp.SameRackCount + '0') |
|||
return string(b) |
|||
} |
|||
|
|||
func (rp *ReplicaPlacement) GetCopyCount() int { |
|||
return rp.DiffDataCenterCount + rp.DiffRackCount + rp.SameRackCount + 1 |
|||
} |
|||
|
|||
func (rp *ReplicaPlacement) GetReplicationLevelIndex() int { |
|||
return rp.DiffDataCenterCount*3 + rp.DiffRackCount*3 + rp.SameRackCount |
|||
} |
|||
@ -1,123 +0,0 @@ |
|||
package storage |
|||
|
|||
import ( |
|||
"errors" |
|||
) |
|||
|
|||
type ReplicationType string |
|||
|
|||
const ( |
|||
Copy000 = ReplicationType("000") // single copy
|
|||
Copy001 = ReplicationType("001") // 2 copies, both on the same racks, and same data center
|
|||
Copy010 = ReplicationType("010") // 2 copies, both on different racks, but same data center
|
|||
Copy100 = ReplicationType("100") // 2 copies, each on different data center
|
|||
Copy110 = ReplicationType("110") // 3 copies, 2 on different racks and local data center, 1 on different data center
|
|||
Copy200 = ReplicationType("200") // 3 copies, each on dffereint data center
|
|||
LengthRelicationType = 6 |
|||
CopyNil = ReplicationType(255) // nil value
|
|||
) |
|||
|
|||
func NewReplicationTypeFromString(t string) (ReplicationType, error) { |
|||
switch t { |
|||
case "000": |
|||
return Copy000, nil |
|||
case "001": |
|||
return Copy001, nil |
|||
case "010": |
|||
return Copy010, nil |
|||
case "100": |
|||
return Copy100, nil |
|||
case "110": |
|||
return Copy110, nil |
|||
case "200": |
|||
return Copy200, nil |
|||
} |
|||
return Copy000, errors.New("Unknown Replication Type:" + t) |
|||
} |
|||
func NewReplicationTypeFromByte(b byte) (ReplicationType, error) { |
|||
switch b { |
|||
case byte(000): |
|||
return Copy000, nil |
|||
case byte(001): |
|||
return Copy001, nil |
|||
case byte(010): |
|||
return Copy010, nil |
|||
case byte(100): |
|||
return Copy100, nil |
|||
case byte(110): |
|||
return Copy110, nil |
|||
case byte(200): |
|||
return Copy200, nil |
|||
} |
|||
return Copy000, errors.New("Unknown Replication Type:" + string(b)) |
|||
} |
|||
|
|||
func (r *ReplicationType) String() string { |
|||
switch *r { |
|||
case Copy000: |
|||
return "000" |
|||
case Copy001: |
|||
return "001" |
|||
case Copy010: |
|||
return "010" |
|||
case Copy100: |
|||
return "100" |
|||
case Copy110: |
|||
return "110" |
|||
case Copy200: |
|||
return "200" |
|||
} |
|||
return "000" |
|||
} |
|||
func (r *ReplicationType) Byte() byte { |
|||
switch *r { |
|||
case Copy000: |
|||
return byte(000) |
|||
case Copy001: |
|||
return byte(001) |
|||
case Copy010: |
|||
return byte(010) |
|||
case Copy100: |
|||
return byte(100) |
|||
case Copy110: |
|||
return byte(110) |
|||
case Copy200: |
|||
return byte(200) |
|||
} |
|||
return byte(000) |
|||
} |
|||
|
|||
func (repType ReplicationType) GetReplicationLevelIndex() int { |
|||
switch repType { |
|||
case Copy000: |
|||
return 0 |
|||
case Copy001: |
|||
return 1 |
|||
case Copy010: |
|||
return 2 |
|||
case Copy100: |
|||
return 3 |
|||
case Copy110: |
|||
return 4 |
|||
case Copy200: |
|||
return 5 |
|||
} |
|||
return -1 |
|||
} |
|||
func (repType ReplicationType) GetCopyCount() int { |
|||
switch repType { |
|||
case Copy000: |
|||
return 1 |
|||
case Copy001: |
|||
return 2 |
|||
case Copy010: |
|||
return 2 |
|||
case Copy100: |
|||
return 2 |
|||
case Copy110: |
|||
return 3 |
|||
case Copy200: |
|||
return 3 |
|||
} |
|||
return 0 |
|||
} |
|||
@ -1,83 +0,0 @@ |
|||
package topology |
|||
|
|||
import ( |
|||
"code.google.com/p/weed-fs/go/glog" |
|||
"code.google.com/p/weed-fs/go/storage" |
|||
"math/rand" |
|||
) |
|||
|
|||
type NodeList struct { |
|||
nodes map[NodeId]Node |
|||
except map[string]Node |
|||
} |
|||
|
|||
func NewNodeList(nodes map[NodeId]Node, except map[string]Node) *NodeList { |
|||
m := make(map[NodeId]Node, len(nodes)-len(except)) |
|||
for _, n := range nodes { |
|||
if except[n.String()] == nil { |
|||
m[n.Id()] = n |
|||
} |
|||
} |
|||
nl := &NodeList{nodes: m} |
|||
return nl |
|||
} |
|||
|
|||
func (nl *NodeList) FreeSpace() int { |
|||
freeSpace := 0 |
|||
for _, n := range nl.nodes { |
|||
freeSpace += n.FreeSpace() |
|||
} |
|||
return freeSpace |
|||
} |
|||
|
|||
func (nl *NodeList) RandomlyPickN(count int, minSpace int, firstNodeName string) ([]Node, bool) { |
|||
var list []Node |
|||
var preferredNode *Node |
|||
if firstNodeName != "" { |
|||
for _, n := range nl.nodes { |
|||
if n.Id() == NodeId(firstNodeName) && n.FreeSpace() >= minSpace { |
|||
preferredNode = &n |
|||
break |
|||
} |
|||
} |
|||
if preferredNode == nil { |
|||
return list, false |
|||
} |
|||
} |
|||
|
|||
for _, n := range nl.nodes { |
|||
if n.FreeSpace() >= minSpace && n.Id() != NodeId(firstNodeName) { |
|||
list = append(list, n) |
|||
} |
|||
} |
|||
if count > len(list) || count == len(list) && firstNodeName != "" { |
|||
return nil, false |
|||
} |
|||
for i := len(list); i > 0; i-- { |
|||
r := rand.Intn(i) |
|||
list[r], list[i-1] = list[i-1], list[r] |
|||
} |
|||
if firstNodeName != "" { |
|||
list[0] = *preferredNode |
|||
} |
|||
return list[:count], true |
|||
} |
|||
|
|||
func (nl *NodeList) ReserveOneVolume(randomVolumeIndex int, vid storage.VolumeId) (bool, *DataNode) { |
|||
for _, node := range nl.nodes { |
|||
freeSpace := node.FreeSpace() |
|||
if randomVolumeIndex >= freeSpace { |
|||
randomVolumeIndex -= freeSpace |
|||
} else { |
|||
if node.IsDataNode() && node.FreeSpace() > 0 { |
|||
glog.V(0).Infoln("vid =", vid, " assigned to node =", node, ", freeSpace =", node.FreeSpace()) |
|||
return true, node.(*DataNode) |
|||
} |
|||
children := node.Children() |
|||
newNodeList := NewNodeList(children, nl.except) |
|||
return newNodeList.ReserveOneVolume(randomVolumeIndex, vid) |
|||
} |
|||
} |
|||
return false, nil |
|||
|
|||
} |
|||
@ -1,60 +0,0 @@ |
|||
package topology |
|||
|
|||
import ( |
|||
"code.google.com/p/weed-fs/go/sequence" |
|||
_ "fmt" |
|||
"strconv" |
|||
"testing" |
|||
) |
|||
|
|||
func TestXYZ(t *testing.T) { |
|||
topo, err := NewTopology("topo", "/etc/weed.conf", sequence.NewMemorySequencer(), 234, 5) |
|||
if err != nil { |
|||
t.Error("cannot create new topology:", err) |
|||
t.FailNow() |
|||
} |
|||
for i := 0; i < 5; i++ { |
|||
dc := NewDataCenter("dc" + strconv.Itoa(i)) |
|||
dc.activeVolumeCount = i |
|||
dc.maxVolumeCount = 5 |
|||
topo.LinkChildNode(dc) |
|||
} |
|||
nl := NewNodeList(topo.Children(), nil) |
|||
|
|||
picked, ret := nl.RandomlyPickN(1, 0, "") |
|||
if !ret || len(picked) != 1 { |
|||
t.Error("need to randomly pick 1 node") |
|||
} |
|||
|
|||
picked, ret = nl.RandomlyPickN(1, 0, "dc1") |
|||
if !ret || len(picked) != 1 { |
|||
t.Error("need to randomly pick 1 node") |
|||
} |
|||
if picked[0].Id() != "dc1" { |
|||
t.Error("need to randomly pick 1 dc1 node") |
|||
} |
|||
|
|||
picked, ret = nl.RandomlyPickN(2, 0, "dc1") |
|||
if !ret || len(picked) != 2 { |
|||
t.Error("need to randomly pick 1 node") |
|||
} |
|||
if picked[0].Id() != "dc1" { |
|||
t.Error("need to randomly pick 2 with one dc1 node") |
|||
} |
|||
|
|||
picked, ret = nl.RandomlyPickN(4, 0, "") |
|||
if !ret || len(picked) != 4 { |
|||
t.Error("need to randomly pick 4 nodes") |
|||
} |
|||
|
|||
picked, ret = nl.RandomlyPickN(5, 0, "") |
|||
if !ret || len(picked) != 5 { |
|||
t.Error("need to randomly pick 5 nodes") |
|||
} |
|||
|
|||
picked, ret = nl.RandomlyPickN(6, 0, "") |
|||
if ret || len(picked) != 0 { |
|||
t.Error("can not randomly pick 6 nodes:", ret, picked) |
|||
} |
|||
|
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue