Browse Source

load configuration file, with topology configured

pull/2/head
Chris Lu 12 years ago
parent
commit
73dbb3aed9
  1. 3
      weed-fs/src/cmd/weed/master.go
  2. 30
      weed-fs/src/pkg/topology/configuration.go
  3. 14
      weed-fs/src/pkg/topology/data_center.go
  4. 21
      weed-fs/src/pkg/topology/ip_range.go
  5. 2
      weed-fs/src/pkg/topology/node.go
  6. 10
      weed-fs/src/pkg/topology/rack.go
  7. 26
      weed-fs/src/pkg/topology/topology.go

3
weed-fs/src/cmd/weed/master.go

@ -34,6 +34,7 @@ var (
capacity = cmdMaster.Flag.Int("capacity", 100, "maximum number of volumes to hold") capacity = cmdMaster.Flag.Int("capacity", 100, "maximum number of volumes to hold")
volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 32*1024, "Default Volume Size in MegaBytes") volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 32*1024, "Default Volume Size in MegaBytes")
mpulse = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") mpulse = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
confFile = cmdMaster.Flag.String("conf", "./weed.xml", "configuration file")
) )
var mapper *directory.Mapper var mapper *directory.Mapper
@ -131,7 +132,7 @@ func volumeGrowHandler(w http.ResponseWriter, r *http.Request) {
} }
func runMaster(cmd *Command, args []string) bool { func runMaster(cmd *Command, args []string) bool {
topo = topology.NewTopology("topo", *metaFolder, "toposequence", uint64(*volumeSizeLimitMB)*1024*1024, *mpulse)
topo = topology.NewTopology("topo", *confFile, *metaFolder, "toposequence", uint64(*volumeSizeLimitMB)*1024*1024, *mpulse)
vg = replication.NewDefaultVolumeGrowth() vg = replication.NewDefaultVolumeGrowth()
log.Println("Volume Size Limit is", *volumeSizeLimitMB, "MB") log.Println("Volume Size Limit is", *volumeSizeLimitMB, "MB")
mapper = directory.NewMapper(*metaFolder, "directory", uint64(*volumeSizeLimitMB)*1024*1024, *mpulse) mapper = directory.NewMapper(*metaFolder, "directory", uint64(*volumeSizeLimitMB)*1024*1024, *mpulse)

30
weed-fs/src/pkg/topology/configuration.go

@ -4,6 +4,10 @@ import (
"encoding/xml" "encoding/xml"
) )
type loc struct {
dcName string
rackName string
}
type rack struct { type rack struct {
Name string `xml:"name,attr"` Name string `xml:"name,attr"`
Ips []string `xml:"Ip"` Ips []string `xml:"Ip"`
@ -15,20 +19,38 @@ type dataCenter struct {
type topology struct { type topology struct {
DataCenters []dataCenter `xml:"DataCenter"` DataCenters []dataCenter `xml:"DataCenter"`
} }
type configuration struct {
type Configuration struct {
XMLName xml.Name `xml:"Configuration"` XMLName xml.Name `xml:"Configuration"`
Topo topology `xml:"Topology"` Topo topology `xml:"Topology"`
ip2location map[string]loc
} }
func NewConfiguration(b []byte) (*configuration, error){
c := &configuration{}
func NewConfiguration(b []byte) (*Configuration, error) {
c := &Configuration{}
err := xml.Unmarshal(b, c) err := xml.Unmarshal(b, c)
c.ip2location = make(map[string]loc)
for _, dc := range c.Topo.DataCenters {
for _, rack := range dc.Racks {
for _, ip := range rack.Ips {
c.ip2location[ip] = loc{dcName: dc.Name, rackName: rack.Name}
}
}
}
return c, err return c, err
} }
func (c *configuration) String() string{
func (c *Configuration) String() string {
if b, e := xml.MarshalIndent(c, " ", " "); e == nil { if b, e := xml.MarshalIndent(c, " ", " "); e == nil {
return string(b) return string(b)
} }
return "" return ""
} }
func (c *Configuration) Locate(ip string) (dc string, rack string) {
if c != nil && c.ip2location != nil {
if loc, ok := c.ip2location[ip]; ok {
return loc.dcName, loc.rackName
}
}
return "DefaultDataCenter", "DefaultRack"
}

14
weed-fs/src/pkg/topology/data_center.go

@ -5,7 +5,6 @@ import (
type DataCenter struct { type DataCenter struct {
NodeImpl NodeImpl
ipRange *IpRange
} }
func NewDataCenter(id string) *DataCenter { func NewDataCenter(id string) *DataCenter {
@ -17,21 +16,14 @@ func NewDataCenter(id string) *DataCenter {
return dc return dc
} }
func (dc *DataCenter) MatchLocationRange(ip string) bool {
if dc.ipRange == nil {
return true
}
return dc.ipRange.Match(ip)
}
func (dc *DataCenter) GetOrCreateRack(ip string) *Rack {
func (dc *DataCenter) GetOrCreateRack(rackName string) *Rack {
for _, c := range dc.Children() { for _, c := range dc.Children() {
rack := c.(*Rack) rack := c.(*Rack)
if rack.MatchLocationRange(ip) {
if string(rack.Id()) == rackName {
return rack return rack
} }
} }
rack := NewRack("DefaultRack")
rack := NewRack(rackName)
dc.LinkChildNode(rack) dc.LinkChildNode(rack)
return rack return rack
} }

21
weed-fs/src/pkg/topology/ip_range.go

@ -1,21 +0,0 @@
package topology
import (
)
type IpRange struct {
inclusives []string
exclusives []string
}
func (r *IpRange) Match(ip string) bool {
// TODO
// for _, exc := range r.exclusives {
// if exc
// }
// for _, inc := range r.inclusives {
// }
return true
}

2
weed-fs/src/pkg/topology/node.go

@ -137,7 +137,7 @@ func (n *NodeImpl) LinkChildNode(node Node) {
n.UpAdjustMaxVolumeId(node.GetMaxVolumeId()) n.UpAdjustMaxVolumeId(node.GetMaxVolumeId())
n.UpAdjustActiveVolumeCountDelta(node.GetActiveVolumeCount()) n.UpAdjustActiveVolumeCountDelta(node.GetActiveVolumeCount())
node.SetParent(n) node.SetParent(n)
fmt.Println(n, "adds child", node)
fmt.Println(n, "adds child", node.Id())
} }
} }

10
weed-fs/src/pkg/topology/rack.go

@ -7,7 +7,6 @@ import (
type Rack struct { type Rack struct {
NodeImpl NodeImpl
ipRange *IpRange
} }
func NewRack(id string) *Rack { func NewRack(id string) *Rack {
@ -19,13 +18,6 @@ func NewRack(id string) *Rack {
return r return r
} }
func (r *Rack) MatchLocationRange(ip string) bool {
if r.ipRange == nil {
return true
}
return r.ipRange.Match(ip)
}
func (r *Rack) GetOrCreateDataNode(ip string, port int, publicUrl string, maxVolumeCount int) *DataNode { func (r *Rack) GetOrCreateDataNode(ip string, port int, publicUrl string, maxVolumeCount int) *DataNode {
for _, c := range r.Children() { for _, c := range r.Children() {
dn := c.(*DataNode) dn := c.(*DataNode)
@ -39,7 +31,7 @@ func (r *Rack) GetOrCreateDataNode(ip string, port int, publicUrl string, maxVol
return dn return dn
} }
} }
dn := NewDataNode("DataNode" + ip + ":" + strconv.Itoa(port))
dn := NewDataNode(ip + ":" + strconv.Itoa(port))
dn.Ip = ip dn.Ip = ip
dn.Port = port dn.Port = port
dn.PublicUrl = publicUrl dn.PublicUrl = publicUrl

26
weed-fs/src/pkg/topology/topology.go

@ -6,6 +6,7 @@ import (
"pkg/directory" "pkg/directory"
"pkg/sequence" "pkg/sequence"
"pkg/storage" "pkg/storage"
"io/ioutil"
) )
type Topology struct { type Topology struct {
@ -23,9 +24,11 @@ type Topology struct {
chanDeadDataNodes chan *DataNode chanDeadDataNodes chan *DataNode
chanRecoveredDataNodes chan *DataNode chanRecoveredDataNodes chan *DataNode
chanFullVolumes chan *storage.VolumeInfo chanFullVolumes chan *storage.VolumeInfo
configuration *Configuration
} }
func NewTopology(id string, dirname string, filename string, volumeSizeLimit uint64, pulse int) *Topology {
func NewTopology(id string, confFile string, dirname string, filename string, volumeSizeLimit uint64, pulse int) *Topology {
t := &Topology{} t := &Topology{}
t.id = NodeId(id) t.id = NodeId(id)
t.nodeType = "Topology" t.nodeType = "Topology"
@ -41,9 +44,19 @@ func NewTopology(id string, dirname string, filename string, volumeSizeLimit uin
t.chanRecoveredDataNodes = make(chan *DataNode) t.chanRecoveredDataNodes = make(chan *DataNode)
t.chanFullVolumes = make(chan *storage.VolumeInfo) t.chanFullVolumes = make(chan *storage.VolumeInfo)
t.loadConfiguration(confFile)
return t return t
} }
func (t *Topology) loadConfiguration(configurationFile string)error{
b, e := ioutil.ReadFile(configurationFile);
if e ==nil{
t.configuration, e = NewConfiguration(b)
}
return e
}
func (t *Topology) RandomlyReserveOneVolume() (bool, *DataNode, *storage.VolumeId) { func (t *Topology) RandomlyReserveOneVolume() (bool, *DataNode, *storage.VolumeId) {
if t.FreeSpace() <= 0 { if t.FreeSpace() <= 0 {
return false, nil, nil return false, nil, nil
@ -97,8 +110,9 @@ func (t *Topology) RegisterVolumeLayout(v *storage.VolumeInfo, dn *DataNode) {
} }
func (t *Topology) RegisterVolumes(volumeInfos []storage.VolumeInfo, ip string, port int, publicUrl string, maxVolumeCount int) { func (t *Topology) RegisterVolumes(volumeInfos []storage.VolumeInfo, ip string, port int, publicUrl string, maxVolumeCount int) {
dc := t.GetOrCreateDataCenter(ip)
rack := dc.GetOrCreateRack(ip)
dcName, rackName := t.configuration.Locate(ip)
dc := t.GetOrCreateDataCenter(dcName)
rack := dc.GetOrCreateRack(rackName)
dn := rack.GetOrCreateDataNode(ip, port, publicUrl, maxVolumeCount) dn := rack.GetOrCreateDataNode(ip, port, publicUrl, maxVolumeCount)
for _, v := range volumeInfos { for _, v := range volumeInfos {
dn.AddOrUpdateVolume(v) dn.AddOrUpdateVolume(v)
@ -106,14 +120,14 @@ func (t *Topology) RegisterVolumes(volumeInfos []storage.VolumeInfo, ip string,
} }
} }
func (t *Topology) GetOrCreateDataCenter(ip string) *DataCenter {
func (t *Topology) GetOrCreateDataCenter(dcName string) *DataCenter {
for _, c := range t.Children() { for _, c := range t.Children() {
dc := c.(*DataCenter) dc := c.(*DataCenter)
if dc.MatchLocationRange(ip) {
if string(dc.Id()) == dcName {
return dc return dc
} }
} }
dc := NewDataCenter("DefaultDataCenter")
dc := NewDataCenter(dcName)
t.LinkChildNode(dc) t.LinkChildNode(dc)
return dc return dc
} }

Loading…
Cancel
Save