Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

assign a free volume randomly

pull/2/head
Chris Lu 13 years ago
parent
8d0ea07f5f
commit
7a30a48027
4 changed files with 92 additions and 59 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      weed-fs/src/cmd/weed/weed.go
  2. 67
      weed-fs/src/pkg/topology/node.go
  3. 75
      weed-fs/src/pkg/topology/topo_test.go
  4. 6
      weed-fs/src/pkg/topology/topology.go

3
weed-fs/src/cmd/weed/weed.go
View File

@ -9,7 +9,9 @@ import (
"os"
"strings"
"sync"
"math/rand"
"text/template"
"time"
"unicode"
"unicode/utf8"
)
@ -39,6 +41,7 @@ func setExitStatus(n int) {
}
func main() {
rand.Seed(time.Now().UnixNano())
flag.Usage = usage
flag.Parse()

67
weed-fs/src/pkg/topology/node.go
View File

@ -8,7 +8,7 @@ import (
type NodeId string
type Node struct {
Id NodeId
countVolumeCount int
activeVolumeCount int
reservedVolumeCount int
maxVolumeCount int
parent *Node
@ -21,30 +21,45 @@ func NewNode() *Node {
n.children = make(map[NodeId]*Node)
return n
}
func (n *Node) String() string {
if n.parent!=nil {
return n.parent.String()+":"+string(n.Id)
}
return string(n.Id)
}
func (n *Node) ReserveOneVolume(r int, vid storage.VolumeId) bool {
ret := false
if n.children == nil {
return true
}
for _, node := range n.children {
freeSpace := node.maxVolumeCount - node.countVolumeCount - node.reservedVolumeCount
if r > freeSpace {
freeSpace := node.maxVolumeCount - node.activeVolumeCount - node.reservedVolumeCount
fmt.Println("r =", r, ", node =", node, ", freeSpace =", freeSpace)
if freeSpace <= 0 {
continue
}
if r >= freeSpace {
r -= freeSpace
} else {
if node.ReserveOneVolume(r, vid) {
node.reservedVolumeCount++
return true
} else {
return false
}
ret = node.ReserveOneVolume(r, vid)
if ret {
break
}
}
}
return false
if ret {
n.reservedVolumeCount++
}
return ret
}
func (n *Node) AddVolume(v *storage.VolumeInfo) {
if n.maxVolumeId < v.Id {
n.maxVolumeId = v.Id
}
n.countVolumeCount++
fmt.Println(n.Id, "adds 1, volumeCount =", n.countVolumeCount)
n.activeVolumeCount++
fmt.Println(n.Id, "adds 1, volumeCount =", n.activeVolumeCount)
if n.reservedVolumeCount > 0 { //if reserved
n.reservedVolumeCount--
}
@ -52,6 +67,12 @@ func (n *Node) AddVolume(v *storage.VolumeInfo) {
n.parent.AddVolume(v)
}
}
func (n *Node) AddMaxVolumeCount(maxVolumeCount int) {//can be negative
n.maxVolumeCount += maxVolumeCount
if n.parent != nil {
n.parent.AddMaxVolumeCount(maxVolumeCount)
}
}
func (n *Node) GetMaxVolumeId() storage.VolumeId {
return n.maxVolumeId
@ -60,17 +81,27 @@ func (n *Node) GetMaxVolumeId() storage.VolumeId {
func (n *Node) AddNode(node *Node) {
if n.children[node.Id] == nil {
n.children[node.Id] = node
n.countVolumeCount += node.countVolumeCount
n.activeVolumeCount += node.activeVolumeCount
n.reservedVolumeCount += node.reservedVolumeCount
n.maxVolumeCount += node.maxVolumeCount
fmt.Println(n.Id, "adds", node.Id, "volumeCount =", n.countVolumeCount)
fmt.Println(n.Id, "adds", node.Id, "volumeCount =", n.activeVolumeCount)
}
}
func (n *Node) RemoveNode(node *Node) {
if n.children[node.Id] != nil {
func (n *Node) RemoveNode(nodeId NodeId) {
node := n.children[nodeId]
if node != nil {
delete(n.children, node.Id)
n.countVolumeCount -= node.countVolumeCount
n.activeVolumeCount -= node.activeVolumeCount
n.maxVolumeCount -= node.maxVolumeCount
fmt.Println(n.Id, "removes", node.Id, "volumeCount =", n.countVolumeCount)
n.reservedVolumeCount -= node.reservedVolumeCount
p := n.parent
for p != nil {
p.activeVolumeCount -= node.activeVolumeCount
p.maxVolumeCount -= node.maxVolumeCount
p.reservedVolumeCount -= node.reservedVolumeCount
p = p.parent
}
fmt.Println(n.Id, "removes", node.Id, "volumeCount =", n.activeVolumeCount)
}
}

75
weed-fs/src/pkg/topology/topo_test.go
View File

@ -3,8 +3,10 @@ package topology
import (
"encoding/json"
"fmt"
"math/rand"
"pkg/storage"
"testing"
"time"
)
var topologyLayout = `
@ -77,40 +79,41 @@ func setup() *Topology {
fmt.Println("data:", data)
printMap(data)
//need to connect all nodes first before server adding volumes
//need to connect all nodes first before server adding volumes
topo := NewTopology(NodeId("mynetwork"))
mTopology := data.(map[string]interface{})
for dcKey, dcValue := range mTopology {
dc := NewDataCenter(NodeId(dcKey))
dc.Node.parent = &topo.Node
dcMap := dcValue.(map[string]interface{})
topo.Node.AddNode(&dc.Node)
topo.Node.AddNode(&dc.Node)
for rackKey, rackValue := range dcMap {
rack := NewRack(NodeId(rackKey))
rack.Node.parent = &dc.Node
rack.Node.parent = &dc.Node
rackMap := rackValue.(map[string]interface{})
dc.Node.AddNode(&rack.Node)
dc.Node.AddNode(&rack.Node)
for serverKey, serverValue := range rackMap {
server := NewServer(NodeId(serverKey))
server.Node.parent = &rack.Node
server.Node.parent = &rack.Node
serverMap := serverValue.(map[string]interface{})
rack.Node.AddNode(&server.Node)
rack.Node.AddNode(&server.Node)
for _, v := range serverMap["volumes"].([]interface{}) {
m := v.(map[string]interface{})
m := v.(map[string]interface{})
vi := &storage.VolumeInfo{Id: storage.VolumeId(int64(m["id"].(float64))), Size: int64(m["size"].(float64))}
server.AddVolume(vi)
}
server.Node.AddMaxVolumeCount(int(serverMap["limit"].(float64)))
}
}
}
fmt.Println("topology:", *topo)
bytes, err := json.Marshal(topo.Node.children)
if err!=nil{
fmt.Println("json error:", err)
}
fmt.Println("json topo:", string(bytes))
bytes, err := json.Marshal(topo.Node.children)
if err != nil {
fmt.Println("json error:", err)
}
fmt.Println("json topo:", string(bytes))
return topo
}
@ -143,34 +146,26 @@ func TestAddVolume(t *testing.T) {
topo.AddVolume(v)
}
func TestAddServer(t *testing.T) {
}
func TestAddRack(t *testing.T) {
}
func TestAddDataCenter(t *testing.T) {
}
func TestReserveVolume(t *testing.T) {
}
func TestRemoveVolume(t *testing.T) {
}
func TestRemoveServer(t *testing.T) {
}
func TestRemoveRack(t *testing.T) {
}
func TestRemoveDataCenter(t *testing.T) {
topo := setup()
topo.RemoveNode(NodeId("dc2"))
if topo.activeVolumeCount != 15 {
t.Fail()
}
topo.RemoveNode(NodeId("dc3"))
if topo.activeVolumeCount != 12 {
t.Fail()
}
}
func TestReserveOneVolume(t *testing.T) {
topo := setup()
rand.Seed(time.Now().UnixNano())
ret, vid := topo.RandomlyReserveOneVolume()
fmt.Println("topology:", topo.Node)
fmt.Println("assigned :", ret)
fmt.Println("assigned volume id:", vid)
if topo.reservedVolumeCount != 1 {
t.Fail()
}
}

6
weed-fs/src/pkg/topology/topology.go
View File

@ -3,6 +3,7 @@ package topology
import (
"math/rand"
"pkg/storage"
"fmt"
)
type Topology struct {
@ -16,7 +17,10 @@ func NewTopology(id NodeId) *Topology{
}
func (t *Topology) RandomlyReserveOneVolume() (bool,storage.VolumeId) {
r := rand.Intn(t.Node.maxVolumeCount-t.Node.countVolumeCount-t.Node.reservedVolumeCount)
slots := t.Node.maxVolumeCount-t.Node.activeVolumeCount-t.Node.reservedVolumeCount
r := rand.Intn(slots)
r = rand.Intn(slots)
fmt.Println("slots:", slots, "random :", r)
vid := t.nextVolumeId()
return t.Node.ReserveOneVolume(r,vid), vid
}

Write Preview
Loading…
Cancel
Save