|
|
|
@ -107,10 +107,8 @@ func (c *commandEcBalance) Do(args []string, commandEnv *CommandEnv, writer io.W |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
ctx := context.Background() |
|
|
|
|
|
|
|
// collect all ec nodes
|
|
|
|
allEcNodes, totalFreeEcSlots, err := collectEcNodes(ctx, commandEnv, *dc) |
|
|
|
allEcNodes, totalFreeEcSlots, err := collectEcNodes(commandEnv, *dc) |
|
|
|
if err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
@ -138,7 +136,7 @@ func (c *commandEcBalance) Do(args []string, commandEnv *CommandEnv, writer io.W |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if err := balanceEcRacks(ctx, commandEnv, racks, *applyBalancing); err != nil { |
|
|
|
if err := balanceEcRacks(commandEnv, racks, *applyBalancing); err != nil { |
|
|
|
return fmt.Errorf("balance ec racks: %v", err) |
|
|
|
} |
|
|
|
|
|
|
|
@ -170,11 +168,11 @@ func balanceEcVolumes(commandEnv *CommandEnv, collection string, allEcNodes []*E |
|
|
|
return fmt.Errorf("delete duplicated collection %s ec shards: %v", collection, err) |
|
|
|
} |
|
|
|
|
|
|
|
if err := balanceEcShardsAcrossRacks(ctx, commandEnv, allEcNodes, racks, collection, applyBalancing); err != nil { |
|
|
|
if err := balanceEcShardsAcrossRacks(commandEnv, allEcNodes, racks, collection, applyBalancing); err != nil { |
|
|
|
return fmt.Errorf("balance across racks collection %s ec shards: %v", collection, err) |
|
|
|
} |
|
|
|
|
|
|
|
if err := balanceEcShardsWithinRacks(ctx, commandEnv, allEcNodes, racks, collection, applyBalancing); err != nil { |
|
|
|
if err := balanceEcShardsWithinRacks(commandEnv, allEcNodes, racks, collection, applyBalancing); err != nil { |
|
|
|
return fmt.Errorf("balance across racks collection %s ec shards: %v", collection, err) |
|
|
|
} |
|
|
|
|
|
|
|
@ -186,14 +184,14 @@ func deleteDuplicatedEcShards(ctx context.Context, commandEnv *CommandEnv, allEc |
|
|
|
vidLocations := collectVolumeIdToEcNodes(allEcNodes) |
|
|
|
// deduplicate ec shards
|
|
|
|
for vid, locations := range vidLocations { |
|
|
|
if err := doDeduplicateEcShards(ctx, commandEnv, collection, vid, locations, applyBalancing); err != nil { |
|
|
|
if err := doDeduplicateEcShards(commandEnv, collection, vid, locations, applyBalancing); err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
} |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func doDeduplicateEcShards(ctx context.Context, commandEnv *CommandEnv, collection string, vid needle.VolumeId, locations []*EcNode, applyBalancing bool) error { |
|
|
|
func doDeduplicateEcShards(commandEnv *CommandEnv, collection string, vid needle.VolumeId, locations []*EcNode, applyBalancing bool) error { |
|
|
|
|
|
|
|
// check whether this volume has ecNodes that are over average
|
|
|
|
shardToLocations := make([][]*EcNode, erasure_coding.TotalShardsCount) |
|
|
|
@ -215,10 +213,10 @@ func doDeduplicateEcShards(ctx context.Context, commandEnv *CommandEnv, collecti |
|
|
|
|
|
|
|
duplicatedShardIds := []uint32{uint32(shardId)} |
|
|
|
for _, ecNode := range ecNodes[1:] { |
|
|
|
if err := unmountEcShards(ctx, commandEnv.option.GrpcDialOption, vid, ecNode.info.Id, duplicatedShardIds); err != nil { |
|
|
|
if err := unmountEcShards(commandEnv.option.GrpcDialOption, vid, ecNode.info.Id, duplicatedShardIds); err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
if err := sourceServerDeleteEcShards(ctx, commandEnv.option.GrpcDialOption, collection, vid, ecNode.info.Id, duplicatedShardIds); err != nil { |
|
|
|
if err := sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, vid, ecNode.info.Id, duplicatedShardIds); err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
ecNode.deleteEcVolumeShards(vid, duplicatedShardIds) |
|
|
|
@ -227,19 +225,19 @@ func doDeduplicateEcShards(ctx context.Context, commandEnv *CommandEnv, collecti |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func balanceEcShardsAcrossRacks(ctx context.Context, commandEnv *CommandEnv, allEcNodes []*EcNode, racks map[RackId]*EcRack, collection string, applyBalancing bool) error { |
|
|
|
func balanceEcShardsAcrossRacks(commandEnv *CommandEnv, allEcNodes []*EcNode, racks map[RackId]*EcRack, collection string, applyBalancing bool) error { |
|
|
|
// collect vid => []ecNode, since previous steps can change the locations
|
|
|
|
vidLocations := collectVolumeIdToEcNodes(allEcNodes) |
|
|
|
// spread the ec shards evenly
|
|
|
|
for vid, locations := range vidLocations { |
|
|
|
if err := doBalanceEcShardsAcrossRacks(ctx, commandEnv, collection, vid, locations, racks, applyBalancing); err != nil { |
|
|
|
if err := doBalanceEcShardsAcrossRacks(commandEnv, collection, vid, locations, racks, applyBalancing); err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
} |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func doBalanceEcShardsAcrossRacks(ctx context.Context, commandEnv *CommandEnv, collection string, vid needle.VolumeId, locations []*EcNode, racks map[RackId]*EcRack, applyBalancing bool) error { |
|
|
|
func doBalanceEcShardsAcrossRacks(commandEnv *CommandEnv, collection string, vid needle.VolumeId, locations []*EcNode, racks map[RackId]*EcRack, applyBalancing bool) error { |
|
|
|
|
|
|
|
// calculate average number of shards an ec rack should have for one volume
|
|
|
|
averageShardsPerEcRack := ceilDivide(erasure_coding.TotalShardsCount, len(racks)) |
|
|
|
@ -274,7 +272,7 @@ func doBalanceEcShardsAcrossRacks(ctx context.Context, commandEnv *CommandEnv, c |
|
|
|
for _, n := range racks[rackId].ecNodes { |
|
|
|
possibleDestinationEcNodes = append(possibleDestinationEcNodes, n) |
|
|
|
} |
|
|
|
err := pickOneEcNodeAndMoveOneShard(ctx, commandEnv, averageShardsPerEcRack, ecNode, collection, vid, shardId, possibleDestinationEcNodes, applyBalancing) |
|
|
|
err := pickOneEcNodeAndMoveOneShard(commandEnv, averageShardsPerEcRack, ecNode, collection, vid, shardId, possibleDestinationEcNodes, applyBalancing) |
|
|
|
if err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
@ -306,7 +304,7 @@ func pickOneRack(rackToEcNodes map[RackId]*EcRack, rackToShardCount map[string]i |
|
|
|
return "" |
|
|
|
} |
|
|
|
|
|
|
|
func balanceEcShardsWithinRacks(ctx context.Context, commandEnv *CommandEnv, allEcNodes []*EcNode, racks map[RackId]*EcRack, collection string, applyBalancing bool) error { |
|
|
|
func balanceEcShardsWithinRacks(commandEnv *CommandEnv, allEcNodes []*EcNode, racks map[RackId]*EcRack, collection string, applyBalancing bool) error { |
|
|
|
// collect vid => []ecNode, since previous steps can change the locations
|
|
|
|
vidLocations := collectVolumeIdToEcNodes(allEcNodes) |
|
|
|
|
|
|
|
@ -330,7 +328,7 @@ func balanceEcShardsWithinRacks(ctx context.Context, commandEnv *CommandEnv, all |
|
|
|
} |
|
|
|
sourceEcNodes := rackEcNodesWithVid[rackId] |
|
|
|
averageShardsPerEcNode := ceilDivide(rackToShardCount[rackId], len(possibleDestinationEcNodes)) |
|
|
|
if err := doBalanceEcShardsWithinOneRack(ctx, commandEnv, averageShardsPerEcNode, collection, vid, sourceEcNodes, possibleDestinationEcNodes, applyBalancing); err != nil { |
|
|
|
if err := doBalanceEcShardsWithinOneRack(commandEnv, averageShardsPerEcNode, collection, vid, sourceEcNodes, possibleDestinationEcNodes, applyBalancing); err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
} |
|
|
|
@ -338,7 +336,7 @@ func balanceEcShardsWithinRacks(ctx context.Context, commandEnv *CommandEnv, all |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func doBalanceEcShardsWithinOneRack(ctx context.Context, commandEnv *CommandEnv, averageShardsPerEcNode int, collection string, vid needle.VolumeId, existingLocations, possibleDestinationEcNodes []*EcNode, applyBalancing bool) error { |
|
|
|
func doBalanceEcShardsWithinOneRack(commandEnv *CommandEnv, averageShardsPerEcNode int, collection string, vid needle.VolumeId, existingLocations, possibleDestinationEcNodes []*EcNode, applyBalancing bool) error { |
|
|
|
|
|
|
|
for _, ecNode := range existingLocations { |
|
|
|
|
|
|
|
@ -353,7 +351,7 @@ func doBalanceEcShardsWithinOneRack(ctx context.Context, commandEnv *CommandEnv, |
|
|
|
|
|
|
|
fmt.Printf("%s has %d overlimit, moving ec shard %d.%d\n", ecNode.info.Id, overLimitCount, vid, shardId) |
|
|
|
|
|
|
|
err := pickOneEcNodeAndMoveOneShard(ctx, commandEnv, averageShardsPerEcNode, ecNode, collection, vid, shardId, possibleDestinationEcNodes, applyBalancing) |
|
|
|
err := pickOneEcNodeAndMoveOneShard(commandEnv, averageShardsPerEcNode, ecNode, collection, vid, shardId, possibleDestinationEcNodes, applyBalancing) |
|
|
|
if err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
@ -365,18 +363,18 @@ func doBalanceEcShardsWithinOneRack(ctx context.Context, commandEnv *CommandEnv, |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func balanceEcRacks(ctx context.Context, commandEnv *CommandEnv, racks map[RackId]*EcRack, applyBalancing bool) error { |
|
|
|
func balanceEcRacks(commandEnv *CommandEnv, racks map[RackId]*EcRack, applyBalancing bool) error { |
|
|
|
|
|
|
|
// balance one rack for all ec shards
|
|
|
|
for _, ecRack := range racks { |
|
|
|
if err := doBalanceEcRack(ctx, commandEnv, ecRack, applyBalancing); err != nil { |
|
|
|
if err := doBalanceEcRack(commandEnv, ecRack, applyBalancing); err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
} |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func doBalanceEcRack(ctx context.Context, commandEnv *CommandEnv, ecRack *EcRack, applyBalancing bool) error { |
|
|
|
func doBalanceEcRack(commandEnv *CommandEnv, ecRack *EcRack, applyBalancing bool) error { |
|
|
|
|
|
|
|
if len(ecRack.ecNodes) <= 1 { |
|
|
|
return nil |
|
|
|
@ -421,7 +419,7 @@ func doBalanceEcRack(ctx context.Context, commandEnv *CommandEnv, ecRack *EcRack |
|
|
|
|
|
|
|
fmt.Printf("%s moves ec shards %d.%d to %s\n", fullNode.info.Id, shards.Id, shardId, emptyNode.info.Id) |
|
|
|
|
|
|
|
err := moveMountedShardToEcNode(ctx, commandEnv, fullNode, shards.Collection, needle.VolumeId(shards.Id), shardId, emptyNode, applyBalancing) |
|
|
|
err := moveMountedShardToEcNode(commandEnv, fullNode, shards.Collection, needle.VolumeId(shards.Id), shardId, emptyNode, applyBalancing) |
|
|
|
if err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
@ -440,7 +438,7 @@ func doBalanceEcRack(ctx context.Context, commandEnv *CommandEnv, ecRack *EcRack |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func pickOneEcNodeAndMoveOneShard(ctx context.Context, commandEnv *CommandEnv, averageShardsPerEcNode int, existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, possibleDestinationEcNodes []*EcNode, applyBalancing bool) error { |
|
|
|
func pickOneEcNodeAndMoveOneShard(commandEnv *CommandEnv, averageShardsPerEcNode int, existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, possibleDestinationEcNodes []*EcNode, applyBalancing bool) error { |
|
|
|
|
|
|
|
sortEcNodesByFreeslotsDecending(possibleDestinationEcNodes) |
|
|
|
|
|
|
|
@ -458,7 +456,7 @@ func pickOneEcNodeAndMoveOneShard(ctx context.Context, commandEnv *CommandEnv, a |
|
|
|
|
|
|
|
fmt.Printf("%s moves ec shard %d.%d to %s\n", existingLocation.info.Id, vid, shardId, destEcNode.info.Id) |
|
|
|
|
|
|
|
err := moveMountedShardToEcNode(ctx, commandEnv, existingLocation, collection, vid, shardId, destEcNode, applyBalancing) |
|
|
|
err := moveMountedShardToEcNode(commandEnv, existingLocation, collection, vid, shardId, destEcNode, applyBalancing) |
|
|
|
if err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
|
Chris Lu
5 years ago