You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

387 lines
12 KiB

4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
  1. package shell
  2. import (
  3. "flag"
  4. "fmt"
  5. "github.com/chrislusf/seaweedfs/weed/pb"
  6. "github.com/chrislusf/seaweedfs/weed/storage/super_block"
  7. "github.com/chrislusf/seaweedfs/weed/storage/types"
  8. "golang.org/x/exp/slices"
  9. "io"
  10. "os"
  11. "time"
  12. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  13. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  14. )
  15. func init() {
  16. Commands = append(Commands, &commandVolumeBalance{})
  17. }
  18. type commandVolumeBalance struct {
  19. }
  20. func (c *commandVolumeBalance) Name() string {
  21. return "volume.balance"
  22. }
  23. func (c *commandVolumeBalance) Help() string {
  24. return `balance all volumes among volume servers
  25. volume.balance [-collection ALL|EACH_COLLECTION|<collection_name>] [-force] [-dataCenter=<data_center_name>]
  26. Algorithm:
  27. For each type of volume server (different max volume count limit){
  28. for each collection {
  29. balanceWritableVolumes()
  30. balanceReadOnlyVolumes()
  31. }
  32. }
  33. func balanceWritableVolumes(){
  34. idealWritableVolumeRatio = totalWritableVolumes / totalNumberOfMaxVolumes
  35. for hasMovedOneVolume {
  36. sort all volume servers ordered by the localWritableVolumeRatio = localWritableVolumes to localVolumeMax
  37. pick the volume server B with the highest localWritableVolumeRatio y
  38. for any the volume server A with the number of writable volumes x + 1 <= idealWritableVolumeRatio * localVolumeMax {
  39. if y > localWritableVolumeRatio {
  40. if B has a writable volume id v that A does not have, and satisfy v replication requirements {
  41. move writable volume v from A to B
  42. }
  43. }
  44. }
  45. }
  46. }
  47. func balanceReadOnlyVolumes(){
  48. //similar to balanceWritableVolumes
  49. }
  50. `
  51. }
  52. func (c *commandVolumeBalance) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
  53. balanceCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
  54. collection := balanceCommand.String("collection", "EACH_COLLECTION", "collection name, or use \"ALL_COLLECTIONS\" across collections, \"EACH_COLLECTION\" for each collection")
  55. dc := balanceCommand.String("dataCenter", "", "only apply the balancing for this dataCenter")
  56. applyBalancing := balanceCommand.Bool("force", false, "apply the balancing plan.")
  57. if err = balanceCommand.Parse(args); err != nil {
  58. return nil
  59. }
  60. infoAboutSimulationMode(writer, *applyBalancing, "-force")
  61. if err = commandEnv.confirmIsLocked(args); err != nil {
  62. return
  63. }
  64. // collect topology information
  65. topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 15*time.Second)
  66. if err != nil {
  67. return err
  68. }
  69. volumeServers := collectVolumeServersByDc(topologyInfo, *dc)
  70. volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
  71. diskTypes := collectVolumeDiskTypes(topologyInfo)
  72. if *collection == "EACH_COLLECTION" {
  73. collections, err := ListCollectionNames(commandEnv, true, false)
  74. if err != nil {
  75. return err
  76. }
  77. for _, c := range collections {
  78. if err = balanceVolumeServers(commandEnv, diskTypes, volumeReplicas, volumeServers, volumeSizeLimitMb*1024*1024, c, *applyBalancing); err != nil {
  79. return err
  80. }
  81. }
  82. } else if *collection == "ALL_COLLECTIONS" {
  83. if err = balanceVolumeServers(commandEnv, diskTypes, volumeReplicas, volumeServers, volumeSizeLimitMb*1024*1024, "ALL_COLLECTIONS", *applyBalancing); err != nil {
  84. return err
  85. }
  86. } else {
  87. if err = balanceVolumeServers(commandEnv, diskTypes, volumeReplicas, volumeServers, volumeSizeLimitMb*1024*1024, *collection, *applyBalancing); err != nil {
  88. return err
  89. }
  90. }
  91. return nil
  92. }
  93. func balanceVolumeServers(commandEnv *CommandEnv, diskTypes []types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, volumeSizeLimit uint64, collection string, applyBalancing bool) error {
  94. for _, diskType := range diskTypes {
  95. if err := balanceVolumeServersByDiskType(commandEnv, diskType, volumeReplicas, nodes, volumeSizeLimit, collection, applyBalancing); err != nil {
  96. return err
  97. }
  98. }
  99. return nil
  100. }
  101. func balanceVolumeServersByDiskType(commandEnv *CommandEnv, diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, volumeSizeLimit uint64, collection string, applyBalancing bool) error {
  102. for _, n := range nodes {
  103. n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
  104. if collection != "ALL_COLLECTIONS" {
  105. if v.Collection != collection {
  106. return false
  107. }
  108. }
  109. return v.DiskType == string(diskType)
  110. })
  111. }
  112. if err := balanceSelectedVolume(commandEnv, diskType, volumeReplicas, nodes, capacityByMaxVolumeCount(diskType), sortWritableVolumes, applyBalancing); err != nil {
  113. return err
  114. }
  115. return nil
  116. }
  117. func collectVolumeServersByDc(t *master_pb.TopologyInfo, selectedDataCenter string) (nodes []*Node) {
  118. for _, dc := range t.DataCenterInfos {
  119. if selectedDataCenter != "" && dc.Id != selectedDataCenter {
  120. continue
  121. }
  122. for _, r := range dc.RackInfos {
  123. for _, dn := range r.DataNodeInfos {
  124. nodes = append(nodes, &Node{
  125. info: dn,
  126. dc: dc.Id,
  127. rack: r.Id,
  128. })
  129. }
  130. }
  131. }
  132. return
  133. }
  134. func collectVolumeDiskTypes(t *master_pb.TopologyInfo) (diskTypes []types.DiskType) {
  135. knownTypes := make(map[string]bool)
  136. for _, dc := range t.DataCenterInfos {
  137. for _, r := range dc.RackInfos {
  138. for _, dn := range r.DataNodeInfos {
  139. for diskType, _ := range dn.DiskInfos {
  140. if _, found := knownTypes[diskType]; !found {
  141. knownTypes[diskType] = true
  142. }
  143. }
  144. }
  145. }
  146. }
  147. for diskType, _ := range knownTypes {
  148. diskTypes = append(diskTypes, types.ToDiskType(diskType))
  149. }
  150. return
  151. }
  152. type Node struct {
  153. info *master_pb.DataNodeInfo
  154. selectedVolumes map[uint32]*master_pb.VolumeInformationMessage
  155. dc string
  156. rack string
  157. }
  158. type CapacityFunc func(*master_pb.DataNodeInfo) int
  159. func capacityByMaxVolumeCount(diskType types.DiskType) CapacityFunc {
  160. return func(info *master_pb.DataNodeInfo) int {
  161. diskInfo, found := info.DiskInfos[string(diskType)]
  162. if !found {
  163. return 0
  164. }
  165. return int(diskInfo.MaxVolumeCount)
  166. }
  167. }
  168. func capacityByFreeVolumeCount(diskType types.DiskType) CapacityFunc {
  169. return func(info *master_pb.DataNodeInfo) int {
  170. diskInfo, found := info.DiskInfos[string(diskType)]
  171. if !found {
  172. return 0
  173. }
  174. return int(diskInfo.MaxVolumeCount - diskInfo.VolumeCount)
  175. }
  176. }
  177. func (n *Node) localVolumeRatio(capacityFunc CapacityFunc) float64 {
  178. return divide(len(n.selectedVolumes), capacityFunc(n.info))
  179. }
  180. func (n *Node) localVolumeNextRatio(capacityFunc CapacityFunc) float64 {
  181. return divide(len(n.selectedVolumes)+1, capacityFunc(n.info))
  182. }
  183. func (n *Node) selectVolumes(fn func(v *master_pb.VolumeInformationMessage) bool) {
  184. n.selectedVolumes = make(map[uint32]*master_pb.VolumeInformationMessage)
  185. for _, diskInfo := range n.info.DiskInfos {
  186. for _, v := range diskInfo.VolumeInfos {
  187. if fn(v) {
  188. n.selectedVolumes[v.Id] = v
  189. }
  190. }
  191. }
  192. }
  193. func sortWritableVolumes(volumes []*master_pb.VolumeInformationMessage) {
  194. slices.SortFunc(volumes, func(a, b *master_pb.VolumeInformationMessage) bool {
  195. return a.Size < b.Size
  196. })
  197. }
  198. func sortReadOnlyVolumes(volumes []*master_pb.VolumeInformationMessage) {
  199. slices.SortFunc(volumes, func(a, b *master_pb.VolumeInformationMessage) bool {
  200. return a.Id < b.Id
  201. })
  202. }
  203. func balanceSelectedVolume(commandEnv *CommandEnv, diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, capacityFunc CapacityFunc, sortCandidatesFn func(volumes []*master_pb.VolumeInformationMessage), applyBalancing bool) (err error) {
  204. selectedVolumeCount, volumeMaxCount := 0, 0
  205. var nodesWithCapacity []*Node
  206. for _, dn := range nodes {
  207. selectedVolumeCount += len(dn.selectedVolumes)
  208. capacity := capacityFunc(dn.info)
  209. if capacity > 0 {
  210. nodesWithCapacity = append(nodesWithCapacity, dn)
  211. }
  212. volumeMaxCount += capacity
  213. }
  214. idealVolumeRatio := divide(selectedVolumeCount, volumeMaxCount)
  215. hasMoved := true
  216. // fmt.Fprintf(os.Stdout, " total %d volumes, max %d volumes, idealVolumeRatio %f\n", selectedVolumeCount, volumeMaxCount, idealVolumeRatio)
  217. for hasMoved {
  218. hasMoved = false
  219. slices.SortFunc(nodesWithCapacity, func(a, b *Node) bool {
  220. return a.localVolumeRatio(capacityFunc) < b.localVolumeRatio(capacityFunc)
  221. })
  222. fullNode := nodesWithCapacity[len(nodesWithCapacity)-1]
  223. var candidateVolumes []*master_pb.VolumeInformationMessage
  224. for _, v := range fullNode.selectedVolumes {
  225. candidateVolumes = append(candidateVolumes, v)
  226. }
  227. sortCandidatesFn(candidateVolumes)
  228. for i := 0; i < len(nodesWithCapacity)-1; i++ {
  229. emptyNode := nodesWithCapacity[i]
  230. if !(fullNode.localVolumeRatio(capacityFunc) > idealVolumeRatio && emptyNode.localVolumeNextRatio(capacityFunc) <= idealVolumeRatio) {
  231. // no more volume servers with empty slots
  232. break
  233. }
  234. fmt.Fprintf(os.Stdout, "%s %.2f %.2f:%.2f\t", diskType.ReadableString(), idealVolumeRatio, fullNode.localVolumeRatio(capacityFunc), emptyNode.localVolumeNextRatio(capacityFunc))
  235. hasMoved, err = attemptToMoveOneVolume(commandEnv, volumeReplicas, fullNode, candidateVolumes, emptyNode, applyBalancing)
  236. if err != nil {
  237. return
  238. }
  239. if hasMoved {
  240. // moved one volume
  241. break
  242. }
  243. }
  244. }
  245. return nil
  246. }
  247. func attemptToMoveOneVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*VolumeReplica, fullNode *Node, candidateVolumes []*master_pb.VolumeInformationMessage, emptyNode *Node, applyBalancing bool) (hasMoved bool, err error) {
  248. for _, v := range candidateVolumes {
  249. hasMoved, err = maybeMoveOneVolume(commandEnv, volumeReplicas, fullNode, v, emptyNode, applyBalancing)
  250. if err != nil {
  251. return
  252. }
  253. if hasMoved {
  254. break
  255. }
  256. }
  257. return
  258. }
  259. func maybeMoveOneVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*VolumeReplica, fullNode *Node, candidateVolume *master_pb.VolumeInformationMessage, emptyNode *Node, applyChange bool) (hasMoved bool, err error) {
  260. if candidateVolume.ReplicaPlacement > 0 {
  261. replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(candidateVolume.ReplicaPlacement))
  262. if !isGoodMove(replicaPlacement, volumeReplicas[candidateVolume.Id], fullNode, emptyNode) {
  263. return false, nil
  264. }
  265. }
  266. if _, found := emptyNode.selectedVolumes[candidateVolume.Id]; !found {
  267. if err = moveVolume(commandEnv, candidateVolume, fullNode, emptyNode, applyChange); err == nil {
  268. adjustAfterMove(candidateVolume, volumeReplicas, fullNode, emptyNode)
  269. return true, nil
  270. } else {
  271. return
  272. }
  273. }
  274. return
  275. }
  276. func moveVolume(commandEnv *CommandEnv, v *master_pb.VolumeInformationMessage, fullNode *Node, emptyNode *Node, applyChange bool) error {
  277. collectionPrefix := v.Collection + "_"
  278. if v.Collection == "" {
  279. collectionPrefix = ""
  280. }
  281. fmt.Fprintf(os.Stdout, " moving %s volume %s%d %s => %s\n", v.DiskType, collectionPrefix, v.Id, fullNode.info.Id, emptyNode.info.Id)
  282. if applyChange {
  283. return LiveMoveVolume(commandEnv.option.GrpcDialOption, os.Stderr, needle.VolumeId(v.Id), pb.NewServerAddressFromDataNode(fullNode.info), pb.NewServerAddressFromDataNode(emptyNode.info), 5*time.Second, v.DiskType, false)
  284. }
  285. return nil
  286. }
  287. func isGoodMove(placement *super_block.ReplicaPlacement, existingReplicas []*VolumeReplica, sourceNode, targetNode *Node) bool {
  288. for _, replica := range existingReplicas {
  289. if replica.location.dataNode.Id == targetNode.info.Id &&
  290. replica.location.rack == targetNode.rack &&
  291. replica.location.dc == targetNode.dc {
  292. // never move to existing nodes
  293. return false
  294. }
  295. }
  296. dcs, racks := make(map[string]bool), make(map[string]int)
  297. for _, replica := range existingReplicas {
  298. if replica.location.dataNode.Id != sourceNode.info.Id {
  299. dcs[replica.location.DataCenter()] = true
  300. racks[replica.location.Rack()]++
  301. }
  302. }
  303. dcs[targetNode.dc] = true
  304. racks[fmt.Sprintf("%s %s", targetNode.dc, targetNode.rack)]++
  305. if len(dcs) != placement.DiffDataCenterCount+1 {
  306. return false
  307. }
  308. if len(racks) != placement.DiffRackCount+placement.DiffDataCenterCount+1 {
  309. return false
  310. }
  311. for _, sameRackCount := range racks {
  312. if sameRackCount != placement.SameRackCount+1 {
  313. return false
  314. }
  315. }
  316. return true
  317. }
  318. func adjustAfterMove(v *master_pb.VolumeInformationMessage, volumeReplicas map[uint32][]*VolumeReplica, fullNode *Node, emptyNode *Node) {
  319. delete(fullNode.selectedVolumes, v.Id)
  320. if emptyNode.selectedVolumes != nil {
  321. emptyNode.selectedVolumes[v.Id] = v
  322. }
  323. existingReplicas := volumeReplicas[v.Id]
  324. for _, replica := range existingReplicas {
  325. if replica.location.dataNode.Id == fullNode.info.Id &&
  326. replica.location.rack == fullNode.rack &&
  327. replica.location.dc == fullNode.dc {
  328. loc := newLocation(emptyNode.dc, emptyNode.rack, emptyNode.info)
  329. replica.location = &loc
  330. return
  331. }
  332. }
  333. }