From eec67d60081e7435dcc2506093bceabf26f4938e Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Thu, 13 Mar 2014 12:13:39 -0700
Subject: [PATCH] fix bug on volume growing failure where free space equals
 what's required. more meaningful volume allocation errors

---
 go/replication/volume_growth.go | 27 ++++++++++++++++++---------
 go/topology/node.go             | 10 +++++++---
 2 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/go/replication/volume_growth.go b/go/replication/volume_growth.go
index 8466b149f..d5dd90b13 100644
--- a/go/replication/volume_growth.go
+++ b/go/replication/volume_growth.go
@@ -4,7 +4,7 @@ import (
 	"code.google.com/p/weed-fs/go/glog"
 	"code.google.com/p/weed-fs/go/storage"
 	"code.google.com/p/weed-fs/go/topology"
-	"errors"
+	"fmt"
 	"math/rand"
 	"sync"
 )
@@ -74,27 +74,36 @@ func (vg *VolumeGrowth) findAndGrow(topo *topology.Topology, preferredDataCenter
 
 func (vg *VolumeGrowth) findEmptySlotsForOneVolume(topo *topology.Topology, preferredDataCenter string, rp *storage.ReplicaPlacement) (servers []*topology.DataNode, err error) {
 	//find main datacenter and other data centers
-	mainDataCenter, otherDataCenters, dc_err := topo.RandomlyPickNodes(rp.DiffDataCenterCount+1, func(node topology.Node) bool {
+	mainDataCenter, otherDataCenters, dc_err := topo.RandomlyPickNodes(rp.DiffDataCenterCount+1, func(node topology.Node) error {
 		if preferredDataCenter != "" && node.IsDataCenter() && node.Id() != topology.NodeId(preferredDataCenter) {
-			return false
+			return fmt.Errorf("Not matching preferred:%s", preferredDataCenter)
 		}
-		return node.FreeSpace() > rp.DiffRackCount+rp.SameRackCount+1
+		if node.FreeSpace() < rp.DiffRackCount+rp.SameRackCount+1 {
+			return fmt.Errorf("Free:%d < Expected:%d", node.FreeSpace(), rp.DiffRackCount+rp.SameRackCount+1)
+		}
+		return nil
 	})
 	if dc_err != nil {
 		return nil, dc_err
 	}
 
 	//find main rack and other racks
-	mainRack, otherRacks, rack_err := mainDataCenter.(*topology.DataCenter).RandomlyPickNodes(rp.DiffRackCount+1, func(node topology.Node) bool {
-		return node.FreeSpace() > rp.SameRackCount+1
+	mainRack, otherRacks, rack_err := mainDataCenter.(*topology.DataCenter).RandomlyPickNodes(rp.DiffRackCount+1, func(node topology.Node) error {
+		if node.FreeSpace() < rp.SameRackCount+1 {
+			return fmt.Errorf("Free:%d < Expected:%d", node.FreeSpace(), rp.SameRackCount+1)
+		}
+		return nil
 	})
 	if rack_err != nil {
 		return nil, rack_err
 	}
 
 	//find main rack and other racks
-	mainServer, otherServers, server_err := mainRack.(*topology.Rack).RandomlyPickNodes(rp.SameRackCount+1, func(node topology.Node) bool {
-		return node.FreeSpace() > 1
+	mainServer, otherServers, server_err := mainRack.(*topology.Rack).RandomlyPickNodes(rp.SameRackCount+1, func(node topology.Node) error {
+		if node.FreeSpace() < 1 {
+			return fmt.Errorf("Free:%d < Expected:%d", node.FreeSpace(), 1)
+		}
+		return nil
 	})
 	if server_err != nil {
 		return nil, server_err
@@ -132,7 +141,7 @@ func (vg *VolumeGrowth) grow(topo *topology.Topology, vid storage.VolumeId, coll
 			glog.V(0).Infoln("Created Volume", vid, "on", server)
 		} else {
 			glog.V(0).Infoln("Failed to assign", vid, "to", servers, "error", err)
-			return errors.New("Failed to assign " + vid.String() + ", " + err.Error())
+			return fmt.Errorf("Failed to assign %s: %s", vid.String(), err.Error())
 		}
 	}
 	return nil
diff --git a/go/topology/node.go b/go/topology/node.go
index abe363b39..c52414008 100644
--- a/go/topology/node.go
+++ b/go/topology/node.go
@@ -5,6 +5,7 @@ import (
 	"code.google.com/p/weed-fs/go/storage"
 	"errors"
 	"math/rand"
+	"strings"
 )
 
 type NodeId string
@@ -50,15 +51,18 @@ type NodeImpl struct {
 }
 
 // the first node must satisfy filterFirstNodeFn(), the rest nodes must have one free slot
-func (n *NodeImpl) RandomlyPickNodes(numberOfNodes int, filterFirstNodeFn func(dn Node) bool) (firstNode Node, restNodes []Node, err error) {
+func (n *NodeImpl) RandomlyPickNodes(numberOfNodes int, filterFirstNodeFn func(dn Node) error) (firstNode Node, restNodes []Node, err error) {
 	candidates := make([]Node, 0, len(n.children))
+	errs := make([]string, 0)
 	for _, node := range n.children {
-		if filterFirstNodeFn(node) {
+		if err := filterFirstNodeFn(node); err == nil {
 			candidates = append(candidates, node)
+		} else {
+			errs = append(errs, string(node.Id())+":"+err.Error())
 		}
 	}
 	if len(candidates) == 0 {
-		return nil, nil, errors.New("No matching data node found!")
+		return nil, nil, errors.New("No matching data node found! \n" + strings.Join(errs, "\n"))
 	}
 	firstNode = candidates[rand.Intn(len(candidates))]
 	glog.V(2).Infoln(n.Id(), "picked main node:", firstNode.Id())