diff --git a/weed-fs/src/pkg/topology/data_center.go b/weed-fs/src/pkg/topology/data_center.go index df6500732..c274dcf33 100644 --- a/weed-fs/src/pkg/topology/data_center.go +++ b/weed-fs/src/pkg/topology/data_center.go @@ -7,3 +7,9 @@ type DataCenter struct { Node ipRange IpRange } +func NewDataCenter(id NodeId) *DataCenter{ + dc := &DataCenter{} + dc.Node = *NewNode() + dc.Node.Id = id + return dc +} diff --git a/weed-fs/src/pkg/topology/node.go b/weed-fs/src/pkg/topology/node.go index 401826e04..b3bbcbc95 100644 --- a/weed-fs/src/pkg/topology/node.go +++ b/weed-fs/src/pkg/topology/node.go @@ -1,12 +1,13 @@ package topology import ( + "fmt" "pkg/storage" ) type NodeId string type Node struct { - id NodeId + Id NodeId countVolumeCount int reservedVolumeCount int maxVolumeCount int @@ -15,6 +16,12 @@ type Node struct { maxVolumeId storage.VolumeId } +func NewNode() *Node { + n := &Node{} + n.children = make(map[NodeId]*Node) + return n +} + func (n *Node) ReserveOneVolume(r int, vid storage.VolumeId) bool { for _, node := range n.children { freeSpace := node.maxVolumeCount - node.countVolumeCount - node.reservedVolumeCount @@ -33,10 +40,11 @@ func (n *Node) ReserveOneVolume(r int, vid storage.VolumeId) bool { } func (n *Node) AddVolume(v *storage.VolumeInfo) { - if n.maxVolumeId < v.Id { - n.maxVolumeId = v.Id - } + if n.maxVolumeId < v.Id { + n.maxVolumeId = v.Id + } n.countVolumeCount++ + fmt.Println(n.Id, "adds 1, volumeCount =", n.countVolumeCount) if n.reservedVolumeCount > 0 { //if reserved n.reservedVolumeCount-- } @@ -46,23 +54,23 @@ func (n *Node) AddVolume(v *storage.VolumeInfo) { } func (n *Node) GetMaxVolumeId() storage.VolumeId { - return n.maxVolumeId + return n.maxVolumeId } func (n *Node) AddNode(node *Node) { - n.children[node.id] = node - n.countVolumeCount += node.countVolumeCount - n.maxVolumeCount += node.maxVolumeCount - if n.parent != nil { - n.parent.AddNode(node) + if n.children[node.Id] == nil { + n.children[node.Id] = node + n.countVolumeCount += node.countVolumeCount + n.maxVolumeCount += node.maxVolumeCount + fmt.Println(n.Id, "adds", node.Id, "volumeCount =", n.countVolumeCount) } } func (n *Node) RemoveNode(node *Node) { - delete(n.children, node.id) - n.countVolumeCount -= node.countVolumeCount - n.maxVolumeCount -= node.maxVolumeCount - if n.parent != nil { - n.parent.RemoveNode(node) + if n.children[node.Id] != nil { + delete(n.children, node.Id) + n.countVolumeCount -= node.countVolumeCount + n.maxVolumeCount -= node.maxVolumeCount + fmt.Println(n.Id, "removes", node.Id, "volumeCount =", n.countVolumeCount) } } diff --git a/weed-fs/src/pkg/topology/rack.go b/weed-fs/src/pkg/topology/rack.go index 3bcea6de5..31dac6218 100644 --- a/weed-fs/src/pkg/topology/rack.go +++ b/weed-fs/src/pkg/topology/rack.go @@ -7,3 +7,10 @@ type Rack struct { Node ipRange IpRange } + +func NewRack(id NodeId) *Rack { + r := &Rack{} + r.Node = *NewNode() + r.Node.Id = id + return r +} diff --git a/weed-fs/src/pkg/topology/server.go b/weed-fs/src/pkg/topology/server.go index 802aaebbb..14aed0e2e 100644 --- a/weed-fs/src/pkg/topology/server.go +++ b/weed-fs/src/pkg/topology/server.go @@ -2,6 +2,7 @@ package topology import ( "pkg/storage" + _ "fmt" ) type Server struct { @@ -11,6 +12,12 @@ type Server struct { Port int PublicUrl string } +func NewServer(id NodeId) *Server{ + s := &Server{} + s.Node.Id = id + s.volumes = make(map[storage.VolumeId]*storage.VolumeInfo) + return s +} func (s *Server) CreateOneVolume(r int, vid storage.VolumeId) storage.VolumeId { s.AddVolume(&storage.VolumeInfo{Id:vid, Size: 32*1024*1024*1024}) return vid diff --git a/weed-fs/src/pkg/topology/topo_test.go b/weed-fs/src/pkg/topology/topo_test.go index 92426fb5a..0458c699e 100644 --- a/weed-fs/src/pkg/topology/topo_test.go +++ b/weed-fs/src/pkg/topology/topo_test.go @@ -1,11 +1,146 @@ package topology import ( - "testing" + "encoding/json" + "fmt" + "pkg/storage" + "testing" ) -func TestAddVolume(t *testing.T) { +var topologyLayout = ` +{ + "dc1":{ + "rack1":{ + "server1":{ + "volumes":[ + {"id":1, "size":12312}, + {"id":2, "size":12312}, + {"id":3, "size":12312} + ], + "limit":3 + }, + "server2":{ + "volumes":[ + {"id":4, "size":12312}, + {"id":5, "size":12312}, + {"id":6, "size":12312} + ], + "limit":10 + } + }, + "rack2":{ + "server1":{ + "volumes":[ + {"id":4, "size":12312}, + {"id":5, "size":12312}, + {"id":6, "size":12312} + ], + "limit":4 + }, + "server2":{ + "volumes":[], + "limit":4 + }, + "server3":{ + "volumes":[ + {"id":2, "size":12312}, + {"id":3, "size":12312}, + {"id":4, "size":12312} + ], + "limit":2 + } + } + }, + "dc2":{ + }, + "dc3":{ + "rack2":{ + "server1":{ + "volumes":[ + {"id":1, "size":12312}, + {"id":3, "size":12312}, + {"id":5, "size":12312} + ], + "limit":4 + } + } + } +} +` + +func setup() *Topology { + var data interface{} + err := json.Unmarshal([]byte(topologyLayout), &data) + if err != nil { + fmt.Println("error:", err) + } + fmt.Println("data:", data) + printMap(data) + + //need to connect all nodes first before server adding volumes + topo := NewTopology(NodeId("mynetwork")) + mTopology := data.(map[string]interface{}) + for dcKey, dcValue := range mTopology { + dc := NewDataCenter(NodeId(dcKey)) + dc.Node.parent = &topo.Node + dcMap := dcValue.(map[string]interface{}) + topo.Node.AddNode(&dc.Node) + for rackKey, rackValue := range dcMap { + rack := NewRack(NodeId(rackKey)) + rack.Node.parent = &dc.Node + rackMap := rackValue.(map[string]interface{}) + dc.Node.AddNode(&rack.Node) + for serverKey, serverValue := range rackMap { + server := NewServer(NodeId(serverKey)) + server.Node.parent = &rack.Node + serverMap := serverValue.(map[string]interface{}) + rack.Node.AddNode(&server.Node) + for _, v := range serverMap["volumes"].([]interface{}) { + m := v.(map[string]interface{}) + vi := &storage.VolumeInfo{Id: storage.VolumeId(int64(m["id"].(float64))), Size: int64(m["size"].(float64))} + server.AddVolume(vi) + } + } + } + } + + fmt.Println("topology:", *topo) + + bytes, err := json.Marshal(topo.Node.children) + if err!=nil{ + fmt.Println("json error:", err) + } + fmt.Println("json topo:", string(bytes)) + + return topo +} +func printMap(mm interface{}) { + m := mm.(map[string]interface{}) + for k, v := range m { + switch vv := v.(type) { + case string: + fmt.Println(k, "\"", vv, "\"") + case int, float64: + fmt.Println(k, ":", vv) + case []interface{}: + fmt.Println(k, ":[") + for _, u := range vv { + fmt.Println(u) + fmt.Println(",") + } + fmt.Println("]") + default: + fmt.Println(k, ":") + printMap(vv) + } + } +} + +func TestAddVolume(t *testing.T) { + topo := setup() + v := &storage.VolumeInfo{} + topo.AddVolume(v) } func TestAddServer(t *testing.T) { diff --git a/weed-fs/src/pkg/topology/topology.go b/weed-fs/src/pkg/topology/topology.go index 7afa19803..3dd08a1fe 100644 --- a/weed-fs/src/pkg/topology/topology.go +++ b/weed-fs/src/pkg/topology/topology.go @@ -8,6 +8,12 @@ import ( type Topology struct { Node } +func NewTopology(id NodeId) *Topology{ + t := &Topology{} + t.Node = *NewNode() + t.Node.Id = id + return t +} func (t *Topology) RandomlyReserveOneVolume() (bool,storage.VolumeId) { r := rand.Intn(t.Node.maxVolumeCount-t.Node.countVolumeCount-t.Node.reservedVolumeCount)