load configuration file, with topology configured

weed-fs/src/cmd/weed/master.go

@@ -34,6 +34,7 @@ var (
 	capacity          = cmdMaster.Flag.Int("capacity", 100, "maximum number of volumes to hold")
 	volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 32*1024, "Default Volume Size in MegaBytes")
 	mpulse            = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
+	confFile          = cmdMaster.Flag.String("conf", "./weed.xml", "configuration file")
 )
 
 var mapper *directory.Mapper
@@ -53,7 +54,7 @@ func dirLookupHandler(w http.ResponseWriter, r *http.Request) {
 		for _, machine := range machines {
 			ret = append(ret, map[string]string{"url": machine.Url, "publicUrl": machine.PublicUrl})
 		}
-		writeJson(w, r, map[string]interface{}{"locations":ret})
+		writeJson(w, r, map[string]interface{}{"locations": ret})
 	} else {
 		log.Println("Invalid volume id", volumeId)
 		writeJson(w, r, map[string]string{"error": "volume id " + volumeId.String() + " not found. " + e.Error()})
@@ -131,7 +132,7 @@ func volumeGrowHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func runMaster(cmd *Command, args []string) bool {
-	topo = topology.NewTopology("topo", *metaFolder, "toposequence", uint64(*volumeSizeLimitMB)*1024*1024, *mpulse)
+	topo = topology.NewTopology("topo", *confFile, *metaFolder, "toposequence", uint64(*volumeSizeLimitMB)*1024*1024, *mpulse)
 	vg = replication.NewDefaultVolumeGrowth()
 	log.Println("Volume Size Limit is", *volumeSizeLimitMB, "MB")
 	mapper = directory.NewMapper(*metaFolder, "directory", uint64(*volumeSizeLimitMB)*1024*1024, *mpulse)

weed-fs/src/pkg/topology/configuration.go

@@ -1,34 +1,56 @@
 package topology
 
 import (
-  "encoding/xml"
+	"encoding/xml"
 )
 
+type loc struct {
+	dcName   string
+	rackName string
+}
 type rack struct {
-  Name string `xml:"name,attr"`
-  Ips []string `xml:"Ip"`
+	Name string   `xml:"name,attr"`
+	Ips  []string `xml:"Ip"`
 }
 type dataCenter struct {
-  Name string `xml:"name,attr"`
-  Racks []rack `xml:"Rack"`
+	Name  string `xml:"name,attr"`
+	Racks []rack `xml:"Rack"`
 }
 type topology struct {
-  DataCenters []dataCenter `xml:"DataCenter"`
+	DataCenters []dataCenter `xml:"DataCenter"`
+}
+type Configuration struct {
+	XMLName     xml.Name `xml:"Configuration"`
+	Topo        topology `xml:"Topology"`
+	ip2location map[string]loc
 }
-type configuration struct {
-	XMLName xml.Name `xml:"Configuration"`
-	Topo topology `xml:"Topology"`
+
+func NewConfiguration(b []byte) (*Configuration, error) {
+	c := &Configuration{}
+	err := xml.Unmarshal(b, c)
+	c.ip2location = make(map[string]loc)
+	for _, dc := range c.Topo.DataCenters {
+		for _, rack := range dc.Racks {
+			for _, ip := range rack.Ips {
+				c.ip2location[ip] = loc{dcName: dc.Name, rackName: rack.Name}
+			}
+		}
+	}
+	return c, err
 }
 
-func NewConfiguration(b []byte) (*configuration, error){
-  c := &configuration{}
-  err := xml.Unmarshal(b, c)
-  return c, err
+func (c *Configuration) String() string {
+	if b, e := xml.MarshalIndent(c, "  ", "  "); e == nil {
+		return string(b)
+	}
+	return ""
 }
 
-func (c *configuration) String() string{
-  if b, e := xml.MarshalIndent(c, "  ", "  "); e==nil {
-    return string(b)
-  }
-  return ""
+func (c *Configuration) Locate(ip string) (dc string, rack string) {
+	if c != nil && c.ip2location != nil {
+		if loc, ok := c.ip2location[ip]; ok {
+			return loc.dcName, loc.rackName
+		}
+	}
+	return "DefaultDataCenter", "DefaultRack"
 }

weed-fs/src/pkg/topology/data_center.go

@@ -5,7 +5,6 @@ import (
 
 type DataCenter struct {
 	NodeImpl
-	ipRange *IpRange
 }
 
 func NewDataCenter(id string) *DataCenter {
@@ -17,21 +16,14 @@ func NewDataCenter(id string) *DataCenter {
 	return dc
 }
 
-func (dc *DataCenter) MatchLocationRange(ip string) bool {
-	if dc.ipRange == nil {
-		return true
-	}
-	return dc.ipRange.Match(ip)
-}
-
-func (dc *DataCenter) GetOrCreateRack(ip string) *Rack {
+func (dc *DataCenter) GetOrCreateRack(rackName string) *Rack {
   for _, c := range dc.Children() {
     rack := c.(*Rack)
-    if rack.MatchLocationRange(ip) {
+    if string(rack.Id()) == rackName {
       return rack
     }
   }
-  rack := NewRack("DefaultRack")
+  rack := NewRack(rackName)
   dc.LinkChildNode(rack)
   return rack
 }

weed-fs/src/pkg/topology/ip_range.go

@@ -1,21 +0,0 @@
-package topology
-
-import (
-
-)
-
-
-type IpRange struct {
-  inclusives   []string
-  exclusives   []string
-}
-
-func (r *IpRange) Match(ip string) bool {
-// TODO
-//  for _, exc := range r.exclusives {
-//    if exc
-//  }
-//  for _, inc := range r.inclusives {
-//  }
-  return true
-}

weed-fs/src/pkg/topology/node.go

@@ -137,7 +137,7 @@ func (n *NodeImpl) LinkChildNode(node Node) {
 		n.UpAdjustMaxVolumeId(node.GetMaxVolumeId())
 		n.UpAdjustActiveVolumeCountDelta(node.GetActiveVolumeCount())
 		node.SetParent(n)
-		fmt.Println(n, "adds child", node)
+		fmt.Println(n, "adds child", node.Id())
 	}
 }
 

weed-fs/src/pkg/topology/rack.go

@@ -7,7 +7,6 @@ import (
 
 type Rack struct {
 	NodeImpl
-	ipRange *IpRange
 }
 
 func NewRack(id string) *Rack {
@@ -19,13 +18,6 @@ func NewRack(id string) *Rack {
 	return r
 }
 
-func (r *Rack) MatchLocationRange(ip string) bool {
-	if r.ipRange == nil {
-		return true
-	}
-	return r.ipRange.Match(ip)
-}
-
 func (r *Rack) GetOrCreateDataNode(ip string, port int, publicUrl string, maxVolumeCount int) *DataNode {
 	for _, c := range r.Children() {
 		dn := c.(*DataNode)
@@ -39,7 +31,7 @@ func (r *Rack) GetOrCreateDataNode(ip string, port int, publicUrl string, maxVol
 			return dn
 		}
 	}
-	dn := NewDataNode("DataNode" + ip + ":" + strconv.Itoa(port))
+	dn := NewDataNode(ip + ":" + strconv.Itoa(port))
 	dn.Ip = ip
 	dn.Port = port
 	dn.PublicUrl = publicUrl

weed-fs/src/pkg/topology/topology.go

@@ -6,6 +6,7 @@ import (
 	"pkg/directory"
 	"pkg/sequence"
 	"pkg/storage"
+	"io/ioutil"
 )
 
 type Topology struct {
@@ -23,9 +24,11 @@ type Topology struct {
 	chanDeadDataNodes      chan *DataNode
 	chanRecoveredDataNodes chan *DataNode
 	chanFullVolumes        chan *storage.VolumeInfo
+	
+	configuration *Configuration
 }
 
-func NewTopology(id string, dirname string, filename string, volumeSizeLimit uint64, pulse int) *Topology {
+func NewTopology(id string, confFile string, dirname string, filename string, volumeSizeLimit uint64, pulse int) *Topology {
 	t := &Topology{}
 	t.id = NodeId(id)
 	t.nodeType = "Topology"
@@ -41,9 +44,19 @@ func NewTopology(id string, dirname string, filename string, volumeSizeLimit uin
 	t.chanRecoveredDataNodes = make(chan *DataNode)
 	t.chanFullVolumes = make(chan *storage.VolumeInfo)
 
+  t.loadConfiguration(confFile)
+	
 	return t
 }
 
+func (t *Topology) loadConfiguration(configurationFile string)error{
+  b, e := ioutil.ReadFile(configurationFile);
+  if e ==nil{
+    t.configuration, e = NewConfiguration(b)
+  }
+  return e 
+}
+
 func (t *Topology) RandomlyReserveOneVolume() (bool, *DataNode, *storage.VolumeId) {
 	if t.FreeSpace() <= 0 {
 		return false, nil, nil
@@ -97,8 +110,9 @@ func (t *Topology) RegisterVolumeLayout(v *storage.VolumeInfo, dn *DataNode) {
 }
 
 func (t *Topology) RegisterVolumes(volumeInfos []storage.VolumeInfo, ip string, port int, publicUrl string, maxVolumeCount int) {
-	dc := t.GetOrCreateDataCenter(ip)
-	rack := dc.GetOrCreateRack(ip)
+  dcName, rackName := t.configuration.Locate(ip)
+	dc := t.GetOrCreateDataCenter(dcName)
+	rack := dc.GetOrCreateRack(rackName)
 	dn := rack.GetOrCreateDataNode(ip, port, publicUrl, maxVolumeCount)
 	for _, v := range volumeInfos {
 		dn.AddOrUpdateVolume(v)
@@ -106,14 +120,14 @@ func (t *Topology) RegisterVolumes(volumeInfos []storage.VolumeInfo, ip string,
 	}
 }
 
-func (t *Topology) GetOrCreateDataCenter(ip string) *DataCenter {
+func (t *Topology) GetOrCreateDataCenter(dcName string) *DataCenter {
 	for _, c := range t.Children() {
 		dc := c.(*DataCenter)
-		if dc.MatchLocationRange(ip) {
+		if string(dc.Id()) == dcName {
 			return dc
 		}
 	}
-	dc := NewDataCenter("DefaultDataCenter")
+	dc := NewDataCenter(dcName)
 	t.LinkChildNode(dc)
 	return dc
 }