You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

446 lines
14 KiB

2 years ago
4 years ago
3 months ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
2 years ago
4 years ago
2 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
  1. package shell
  2. import (
  3. "cmp"
  4. "flag"
  5. "fmt"
  6. "io"
  7. "os"
  8. "strings"
  9. "time"
  10. "github.com/seaweedfs/seaweedfs/weed/pb"
  11. "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
  12. "github.com/seaweedfs/seaweedfs/weed/storage/super_block"
  13. "github.com/seaweedfs/seaweedfs/weed/storage/types"
  14. "golang.org/x/exp/slices"
  15. "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
  16. "github.com/seaweedfs/seaweedfs/weed/storage/needle"
  17. )
  18. func init() {
  19. Commands = append(Commands, &commandVolumeBalance{})
  20. }
  21. type commandVolumeBalance struct {
  22. volumeSizeLimitMb uint64
  23. commandEnv *CommandEnv
  24. writable bool
  25. applyBalancing bool
  26. }
  27. func (c *commandVolumeBalance) Name() string {
  28. return "volume.balance"
  29. }
  30. func (c *commandVolumeBalance) Help() string {
  31. return `balance all volumes among volume servers
  32. volume.balance [-collection ALL_COLLECTIONS|EACH_COLLECTION|<collection_name>] [-force] [-dataCenter=<data_center_name>] [-racks=rack_name_one,rack_name_two] [-nodes=192.168.0.1:8080,192.168.0.2:8080]
  33. Algorithm:
  34. For each type of volume server (different max volume count limit){
  35. for each collection {
  36. balanceWritableVolumes()
  37. balanceReadOnlyVolumes()
  38. }
  39. }
  40. func balanceWritableVolumes(){
  41. idealWritableVolumeRatio = totalWritableVolumes / totalNumberOfMaxVolumes
  42. for hasMovedOneVolume {
  43. sort all volume servers ordered by the localWritableVolumeRatio = localWritableVolumes to localVolumeMax
  44. pick the volume server B with the highest localWritableVolumeRatio y
  45. for any the volume server A with the number of writable volumes x + 1 <= idealWritableVolumeRatio * localVolumeMax {
  46. if y > localWritableVolumeRatio {
  47. if B has a writable volume id v that A does not have, and satisfy v replication requirements {
  48. move writable volume v from A to B
  49. }
  50. }
  51. }
  52. }
  53. }
  54. func balanceReadOnlyVolumes(){
  55. //similar to balanceWritableVolumes
  56. }
  57. `
  58. }
  59. func (c *commandVolumeBalance) HasTag(CommandTag) bool {
  60. return false
  61. }
  62. func (c *commandVolumeBalance) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
  63. balanceCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
  64. collection := balanceCommand.String("collection", "ALL_COLLECTIONS", "collection name, or use \"ALL_COLLECTIONS\" across collections, \"EACH_COLLECTION\" for each collection")
  65. dc := balanceCommand.String("dataCenter", "", "only apply the balancing for this dataCenter")
  66. racks := balanceCommand.String("racks", "", "only apply the balancing for this racks")
  67. nodes := balanceCommand.String("nodes", "", "only apply the balancing for this nodes")
  68. c.writable = *(balanceCommand.Bool("writable", false, "only apply the balancing for writable volumes"))
  69. noLock := balanceCommand.Bool("noLock", false, "do not lock the admin shell at one's own risk")
  70. c.applyBalancing = *(balanceCommand.Bool("force", false, "apply the balancing plan."))
  71. if err = balanceCommand.Parse(args); err != nil {
  72. return nil
  73. }
  74. infoAboutSimulationMode(writer, c.applyBalancing, "-force")
  75. if *noLock {
  76. commandEnv.noLock = true
  77. } else {
  78. if err = commandEnv.confirmIsLocked(args); err != nil {
  79. return
  80. }
  81. }
  82. c.commandEnv = commandEnv
  83. // collect topology information
  84. var topologyInfo *master_pb.TopologyInfo
  85. topologyInfo, c.volumeSizeLimitMb, err = collectTopologyInfo(commandEnv, 5*time.Second)
  86. if err != nil {
  87. return err
  88. }
  89. volumeServers := collectVolumeServersByDcRackNode(topologyInfo, *dc, *racks, *nodes)
  90. volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
  91. diskTypes := collectVolumeDiskTypes(topologyInfo)
  92. if *collection == "EACH_COLLECTION" {
  93. collections, err := ListCollectionNames(commandEnv, true, false)
  94. if err != nil {
  95. return err
  96. }
  97. for _, col := range collections {
  98. if err = c.balanceVolumeServers(diskTypes, volumeReplicas, volumeServers, col); err != nil {
  99. return err
  100. }
  101. }
  102. } else {
  103. if err = c.balanceVolumeServers(diskTypes, volumeReplicas, volumeServers, *collection); err != nil {
  104. return err
  105. }
  106. }
  107. return nil
  108. }
  109. func (c *commandVolumeBalance) balanceVolumeServers(diskTypes []types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, collection string) error {
  110. for _, diskType := range diskTypes {
  111. if err := c.balanceVolumeServersByDiskType(diskType, volumeReplicas, nodes, collection); err != nil {
  112. return err
  113. }
  114. }
  115. return nil
  116. }
  117. func (c *commandVolumeBalance) balanceVolumeServersByDiskType(diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, collection string) error {
  118. for _, n := range nodes {
  119. n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
  120. if collection != "ALL_COLLECTIONS" {
  121. if v.Collection != collection {
  122. return false
  123. }
  124. }
  125. if v.DiskType != string(diskType) {
  126. return false
  127. }
  128. if c.writable && v.Size > c.volumeSizeLimitMb {
  129. return false
  130. }
  131. return true
  132. })
  133. }
  134. if err := balanceSelectedVolume(c.commandEnv, diskType, volumeReplicas, nodes, sortWritableVolumes, c.applyBalancing); err != nil {
  135. return err
  136. }
  137. return nil
  138. }
  139. func collectVolumeServersByDcRackNode(t *master_pb.TopologyInfo, selectedDataCenter string, selectedRacks string, selectedNodes string) (nodes []*Node) {
  140. for _, dc := range t.DataCenterInfos {
  141. if selectedDataCenter != "" && dc.Id != selectedDataCenter {
  142. continue
  143. }
  144. for _, r := range dc.RackInfos {
  145. if selectedRacks != "" && !strings.Contains(selectedRacks, r.Id) {
  146. continue
  147. }
  148. for _, dn := range r.DataNodeInfos {
  149. if selectedNodes != "" && !strings.Contains(selectedNodes, dn.Id) {
  150. continue
  151. }
  152. nodes = append(nodes, &Node{
  153. info: dn,
  154. dc: dc.Id,
  155. rack: r.Id,
  156. })
  157. }
  158. }
  159. }
  160. return
  161. }
  162. func collectVolumeDiskTypes(t *master_pb.TopologyInfo) (diskTypes []types.DiskType) {
  163. knownTypes := make(map[string]bool)
  164. for _, dc := range t.DataCenterInfos {
  165. for _, r := range dc.RackInfos {
  166. for _, dn := range r.DataNodeInfos {
  167. for diskType := range dn.DiskInfos {
  168. if _, found := knownTypes[diskType]; !found {
  169. knownTypes[diskType] = true
  170. }
  171. }
  172. }
  173. }
  174. }
  175. for diskType := range knownTypes {
  176. diskTypes = append(diskTypes, types.ToDiskType(diskType))
  177. }
  178. return
  179. }
  180. type Node struct {
  181. info *master_pb.DataNodeInfo
  182. selectedVolumes map[uint32]*master_pb.VolumeInformationMessage
  183. dc string
  184. rack string
  185. }
  186. type CapacityFunc func(*master_pb.DataNodeInfo) float64
  187. func capacityByMaxVolumeCount(diskType types.DiskType) CapacityFunc {
  188. return func(info *master_pb.DataNodeInfo) float64 {
  189. diskInfo, found := info.DiskInfos[string(diskType)]
  190. if !found {
  191. return 0
  192. }
  193. return float64(diskInfo.MaxVolumeCount)
  194. }
  195. }
  196. func capacityByFreeVolumeCount(diskType types.DiskType) CapacityFunc {
  197. return func(info *master_pb.DataNodeInfo) float64 {
  198. diskInfo, found := info.DiskInfos[string(diskType)]
  199. if !found {
  200. return 0
  201. }
  202. var ecShardCount int
  203. for _, ecShardInfo := range diskInfo.EcShardInfos {
  204. ecShardCount += erasure_coding.ShardBits(ecShardInfo.EcIndexBits).ShardIdCount()
  205. }
  206. return float64(diskInfo.MaxVolumeCount-diskInfo.VolumeCount) - float64(ecShardCount)/erasure_coding.DataShardsCount
  207. }
  208. }
  209. func (n *Node) localVolumeRatio(capacityFunc CapacityFunc) float64 {
  210. return float64(len(n.selectedVolumes)) / capacityFunc(n.info)
  211. }
  212. func (n *Node) localVolumeNextRatio(capacityFunc CapacityFunc) float64 {
  213. return float64(len(n.selectedVolumes)+1) / capacityFunc(n.info)
  214. }
  215. func (n *Node) isOneVolumeOnly() bool {
  216. if len(n.selectedVolumes) != 1 {
  217. return false
  218. }
  219. for _, disk := range n.info.DiskInfos {
  220. if disk.VolumeCount == 1 && disk.MaxVolumeCount == 1 {
  221. return true
  222. }
  223. }
  224. return false
  225. }
  226. func (n *Node) selectVolumes(fn func(v *master_pb.VolumeInformationMessage) bool) {
  227. n.selectedVolumes = make(map[uint32]*master_pb.VolumeInformationMessage)
  228. for _, diskInfo := range n.info.DiskInfos {
  229. for _, v := range diskInfo.VolumeInfos {
  230. if fn(v) {
  231. n.selectedVolumes[v.Id] = v
  232. }
  233. }
  234. }
  235. }
  236. func sortWritableVolumes(volumes []*master_pb.VolumeInformationMessage) {
  237. slices.SortFunc(volumes, func(a, b *master_pb.VolumeInformationMessage) int {
  238. return cmp.Compare(a.Size, b.Size)
  239. })
  240. }
  241. func balanceSelectedVolume(commandEnv *CommandEnv, diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, sortCandidatesFn func(volumes []*master_pb.VolumeInformationMessage), applyBalancing bool) (err error) {
  242. selectedVolumeCount, volumeMaxCount := 0, float64(0)
  243. var nodesWithCapacity []*Node
  244. capacityFunc := capacityByMaxVolumeCount(diskType)
  245. for _, dn := range nodes {
  246. selectedVolumeCount += len(dn.selectedVolumes)
  247. capacity := capacityFunc(dn.info)
  248. if capacity > 0 {
  249. nodesWithCapacity = append(nodesWithCapacity, dn)
  250. }
  251. volumeMaxCount += capacity
  252. }
  253. idealVolumeRatio := float64(selectedVolumeCount) / volumeMaxCount
  254. hasMoved := true
  255. // fmt.Fprintf(os.Stdout, " total %d volumes, max %d volumes, idealVolumeRatio %f\n", selectedVolumeCount, volumeMaxCount, idealVolumeRatio)
  256. for hasMoved {
  257. hasMoved = false
  258. slices.SortFunc(nodesWithCapacity, func(a, b *Node) int {
  259. return cmp.Compare(a.localVolumeRatio(capacityFunc), b.localVolumeRatio(capacityFunc))
  260. })
  261. if len(nodesWithCapacity) == 0 {
  262. fmt.Printf("no volume server found with capacity for %s", diskType.ReadableString())
  263. return nil
  264. }
  265. var fullNode *Node
  266. var fullNodeIndex int
  267. for fullNodeIndex = len(nodesWithCapacity) - 1; fullNodeIndex >= 0; fullNodeIndex-- {
  268. fullNode = nodesWithCapacity[fullNodeIndex]
  269. if !fullNode.isOneVolumeOnly() {
  270. break
  271. }
  272. }
  273. var candidateVolumes []*master_pb.VolumeInformationMessage
  274. for _, v := range fullNode.selectedVolumes {
  275. candidateVolumes = append(candidateVolumes, v)
  276. }
  277. sortCandidatesFn(candidateVolumes)
  278. for _, emptyNode := range nodesWithCapacity[:fullNodeIndex] {
  279. if !(fullNode.localVolumeRatio(capacityFunc) > idealVolumeRatio && emptyNode.localVolumeNextRatio(capacityFunc) <= idealVolumeRatio) {
  280. // no more volume servers with empty slots
  281. break
  282. }
  283. fmt.Fprintf(os.Stdout, "%s %.2f %.2f:%.2f\t", diskType.ReadableString(), idealVolumeRatio, fullNode.localVolumeRatio(capacityFunc), emptyNode.localVolumeNextRatio(capacityFunc))
  284. hasMoved, err = attemptToMoveOneVolume(commandEnv, volumeReplicas, fullNode, candidateVolumes, emptyNode, applyBalancing)
  285. if err != nil {
  286. return
  287. }
  288. if hasMoved {
  289. // moved one volume
  290. break
  291. }
  292. }
  293. }
  294. return nil
  295. }
  296. func attemptToMoveOneVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*VolumeReplica, fullNode *Node, candidateVolumes []*master_pb.VolumeInformationMessage, emptyNode *Node, applyBalancing bool) (hasMoved bool, err error) {
  297. for _, v := range candidateVolumes {
  298. hasMoved, err = maybeMoveOneVolume(commandEnv, volumeReplicas, fullNode, v, emptyNode, applyBalancing)
  299. if err != nil {
  300. return
  301. }
  302. if hasMoved {
  303. break
  304. }
  305. }
  306. return
  307. }
  308. func maybeMoveOneVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*VolumeReplica, fullNode *Node, candidateVolume *master_pb.VolumeInformationMessage, emptyNode *Node, applyChange bool) (hasMoved bool, err error) {
  309. if !commandEnv.isLocked() {
  310. return false, fmt.Errorf("lock is lost")
  311. }
  312. if candidateVolume.RemoteStorageName != "" {
  313. return false, fmt.Errorf("does not move volume in remove storage")
  314. }
  315. if candidateVolume.ReplicaPlacement > 0 {
  316. replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(candidateVolume.ReplicaPlacement))
  317. if !isGoodMove(replicaPlacement, volumeReplicas[candidateVolume.Id], fullNode, emptyNode) {
  318. return false, nil
  319. }
  320. }
  321. if _, found := emptyNode.selectedVolumes[candidateVolume.Id]; !found {
  322. if err = moveVolume(commandEnv, candidateVolume, fullNode, emptyNode, applyChange); err == nil {
  323. adjustAfterMove(candidateVolume, volumeReplicas, fullNode, emptyNode)
  324. return true, nil
  325. } else {
  326. return
  327. }
  328. }
  329. return
  330. }
  331. func moveVolume(commandEnv *CommandEnv, v *master_pb.VolumeInformationMessage, fullNode *Node, emptyNode *Node, applyChange bool) error {
  332. collectionPrefix := v.Collection + "_"
  333. if v.Collection == "" {
  334. collectionPrefix = ""
  335. }
  336. fmt.Fprintf(os.Stdout, " moving %s volume %s%d %s => %s\n", v.DiskType, collectionPrefix, v.Id, fullNode.info.Id, emptyNode.info.Id)
  337. if applyChange {
  338. return LiveMoveVolume(commandEnv.option.GrpcDialOption, os.Stderr, needle.VolumeId(v.Id), pb.NewServerAddressFromDataNode(fullNode.info), pb.NewServerAddressFromDataNode(emptyNode.info), 5*time.Second, v.DiskType, 0, false)
  339. }
  340. return nil
  341. }
  342. func isGoodMove(placement *super_block.ReplicaPlacement, existingReplicas []*VolumeReplica, sourceNode, targetNode *Node) bool {
  343. for _, replica := range existingReplicas {
  344. if replica.location.dataNode.Id == targetNode.info.Id &&
  345. replica.location.rack == targetNode.rack &&
  346. replica.location.dc == targetNode.dc {
  347. // never move to existing nodes
  348. return false
  349. }
  350. }
  351. // existing replicas except the one on sourceNode
  352. existingReplicasExceptSourceNode := make([]*VolumeReplica, 0)
  353. for _, replica := range existingReplicas {
  354. if replica.location.dataNode.Id != sourceNode.info.Id {
  355. existingReplicasExceptSourceNode = append(existingReplicasExceptSourceNode, replica)
  356. }
  357. }
  358. // target location
  359. targetLocation := location{
  360. dc: targetNode.dc,
  361. rack: targetNode.rack,
  362. dataNode: targetNode.info,
  363. }
  364. // check if this satisfies replication requirements
  365. return satisfyReplicaPlacement(placement, existingReplicasExceptSourceNode, targetLocation)
  366. }
  367. func adjustAfterMove(v *master_pb.VolumeInformationMessage, volumeReplicas map[uint32][]*VolumeReplica, fullNode *Node, emptyNode *Node) {
  368. delete(fullNode.selectedVolumes, v.Id)
  369. if emptyNode.selectedVolumes != nil {
  370. emptyNode.selectedVolumes[v.Id] = v
  371. }
  372. existingReplicas := volumeReplicas[v.Id]
  373. for _, replica := range existingReplicas {
  374. if replica.location.dataNode.Id == fullNode.info.Id &&
  375. replica.location.rack == fullNode.rack &&
  376. replica.location.dc == fullNode.dc {
  377. loc := newLocation(emptyNode.dc, emptyNode.rack, emptyNode.info)
  378. replica.location = &loc
  379. for diskType, diskInfo := range fullNode.info.DiskInfos {
  380. if diskType == v.DiskType {
  381. addVolumeCount(diskInfo, -1)
  382. }
  383. }
  384. for diskType, diskInfo := range emptyNode.info.DiskInfos {
  385. if diskType == v.DiskType {
  386. addVolumeCount(diskInfo, 1)
  387. }
  388. }
  389. return
  390. }
  391. }
  392. }