You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

276 lines
8.6 KiB

  1. package shell
  2. import (
  3. "context"
  4. "flag"
  5. "fmt"
  6. "github.com/chrislusf/seaweedfs/weed/pb"
  7. "io"
  8. "github.com/chrislusf/seaweedfs/weed/operation"
  9. "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
  10. "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
  11. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  12. "google.golang.org/grpc"
  13. )
  14. func init() {
  15. Commands = append(Commands, &commandEcRebuild{})
  16. }
  17. type commandEcRebuild struct {
  18. }
  19. func (c *commandEcRebuild) Name() string {
  20. return "ec.rebuild"
  21. }
  22. func (c *commandEcRebuild) Help() string {
  23. return `find and rebuild missing ec shards among volume servers
  24. ec.rebuild [-c EACH_COLLECTION|<collection_name>] [-force]
  25. Algorithm:
  26. For each type of volume server (different max volume count limit){
  27. for each collection {
  28. rebuildEcVolumes()
  29. }
  30. }
  31. func rebuildEcVolumes(){
  32. idealWritableVolumes = totalWritableVolumes / numVolumeServers
  33. for {
  34. sort all volume servers ordered by the number of local writable volumes
  35. pick the volume server A with the lowest number of writable volumes x
  36. pick the volume server B with the highest number of writable volumes y
  37. if y > idealWritableVolumes and x +1 <= idealWritableVolumes {
  38. if B has a writable volume id v that A does not have {
  39. move writable volume v from A to B
  40. }
  41. }
  42. }
  43. }
  44. `
  45. }
  46. func (c *commandEcRebuild) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
  47. fixCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
  48. collection := fixCommand.String("collection", "EACH_COLLECTION", "collection name, or \"EACH_COLLECTION\" for each collection")
  49. applyChanges := fixCommand.Bool("force", false, "apply the changes")
  50. if err = fixCommand.Parse(args); err != nil {
  51. return nil
  52. }
  53. if err = commandEnv.confirmIsLocked(args); err != nil {
  54. return
  55. }
  56. // collect all ec nodes
  57. allEcNodes, _, err := collectEcNodes(commandEnv, "")
  58. if err != nil {
  59. return err
  60. }
  61. if *collection == "EACH_COLLECTION" {
  62. collections, err := ListCollectionNames(commandEnv, false, true)
  63. if err != nil {
  64. return err
  65. }
  66. fmt.Printf("rebuildEcVolumes collections %+v\n", len(collections))
  67. for _, c := range collections {
  68. fmt.Printf("rebuildEcVolumes collection %+v\n", c)
  69. if err = rebuildEcVolumes(commandEnv, allEcNodes, c, writer, *applyChanges); err != nil {
  70. return err
  71. }
  72. }
  73. } else {
  74. if err = rebuildEcVolumes(commandEnv, allEcNodes, *collection, writer, *applyChanges); err != nil {
  75. return err
  76. }
  77. }
  78. return nil
  79. }
  80. func rebuildEcVolumes(commandEnv *CommandEnv, allEcNodes []*EcNode, collection string, writer io.Writer, applyChanges bool) error {
  81. fmt.Printf("rebuildEcVolumes %s\n", collection)
  82. // collect vid => each shard locations, similar to ecShardMap in topology.go
  83. ecShardMap := make(EcShardMap)
  84. for _, ecNode := range allEcNodes {
  85. ecShardMap.registerEcNode(ecNode, collection)
  86. }
  87. for vid, locations := range ecShardMap {
  88. shardCount := locations.shardCount()
  89. if shardCount == erasure_coding.TotalShardsCount {
  90. continue
  91. }
  92. if shardCount < erasure_coding.DataShardsCount {
  93. return fmt.Errorf("ec volume %d is unrepairable with %d shards\n", vid, shardCount)
  94. }
  95. sortEcNodesByFreeslotsDecending(allEcNodes)
  96. if allEcNodes[0].freeEcSlot < erasure_coding.TotalShardsCount {
  97. return fmt.Errorf("disk space is not enough")
  98. }
  99. if err := rebuildOneEcVolume(commandEnv, allEcNodes[0], collection, vid, locations, writer, applyChanges); err != nil {
  100. return err
  101. }
  102. }
  103. return nil
  104. }
  105. func rebuildOneEcVolume(commandEnv *CommandEnv, rebuilder *EcNode, collection string, volumeId needle.VolumeId, locations EcShardLocations, writer io.Writer, applyChanges bool) error {
  106. fmt.Printf("rebuildOneEcVolume %s %d\n", collection, volumeId)
  107. // collect shard files to rebuilder local disk
  108. var generatedShardIds []uint32
  109. copiedShardIds, _, err := prepareDataToRecover(commandEnv, rebuilder, collection, volumeId, locations, writer, applyChanges)
  110. if err != nil {
  111. return err
  112. }
  113. defer func() {
  114. // clean up working files
  115. // ask the rebuilder to delete the copied shards
  116. err = sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.info), copiedShardIds)
  117. if err != nil {
  118. fmt.Fprintf(writer, "%s delete copied ec shards %s %d.%v\n", rebuilder.info.Id, collection, volumeId, copiedShardIds)
  119. }
  120. }()
  121. if !applyChanges {
  122. return nil
  123. }
  124. // generate ec shards, and maybe ecx file
  125. generatedShardIds, err = generateMissingShards(commandEnv.option.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.info))
  126. if err != nil {
  127. return err
  128. }
  129. // mount the generated shards
  130. err = mountEcShards(commandEnv.option.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.info), generatedShardIds)
  131. if err != nil {
  132. return err
  133. }
  134. rebuilder.addEcVolumeShards(volumeId, collection, generatedShardIds)
  135. return nil
  136. }
  137. func generateMissingShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress) (rebuiltShardIds []uint32, err error) {
  138. err = operation.WithVolumeServerClient(sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
  139. resp, rebultErr := volumeServerClient.VolumeEcShardsRebuild(context.Background(), &volume_server_pb.VolumeEcShardsRebuildRequest{
  140. VolumeId: uint32(volumeId),
  141. Collection: collection,
  142. })
  143. if rebultErr == nil {
  144. rebuiltShardIds = resp.RebuiltShardIds
  145. }
  146. return rebultErr
  147. })
  148. return
  149. }
  150. func prepareDataToRecover(commandEnv *CommandEnv, rebuilder *EcNode, collection string, volumeId needle.VolumeId, locations EcShardLocations, writer io.Writer, applyBalancing bool) (copiedShardIds []uint32, localShardIds []uint32, err error) {
  151. needEcxFile := true
  152. var localShardBits erasure_coding.ShardBits
  153. for _, diskInfo := range rebuilder.info.DiskInfos {
  154. for _, ecShardInfo := range diskInfo.EcShardInfos {
  155. if ecShardInfo.Collection == collection && needle.VolumeId(ecShardInfo.Id) == volumeId {
  156. needEcxFile = false
  157. localShardBits = erasure_coding.ShardBits(ecShardInfo.EcIndexBits)
  158. }
  159. }
  160. }
  161. for shardId, ecNodes := range locations {
  162. if len(ecNodes) == 0 {
  163. fmt.Fprintf(writer, "missing shard %d.%d\n", volumeId, shardId)
  164. continue
  165. }
  166. if localShardBits.HasShardId(erasure_coding.ShardId(shardId)) {
  167. localShardIds = append(localShardIds, uint32(shardId))
  168. fmt.Fprintf(writer, "use existing shard %d.%d\n", volumeId, shardId)
  169. continue
  170. }
  171. var copyErr error
  172. if applyBalancing {
  173. copyErr = operation.WithVolumeServerClient(pb.NewServerAddressFromDataNode(rebuilder.info), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
  174. _, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{
  175. VolumeId: uint32(volumeId),
  176. Collection: collection,
  177. ShardIds: []uint32{uint32(shardId)},
  178. CopyEcxFile: needEcxFile,
  179. CopyEcjFile: needEcxFile,
  180. CopyVifFile: needEcxFile,
  181. SourceDataNode: ecNodes[0].info.Id,
  182. })
  183. return copyErr
  184. })
  185. if copyErr == nil && needEcxFile {
  186. needEcxFile = false
  187. }
  188. }
  189. if copyErr != nil {
  190. fmt.Fprintf(writer, "%s failed to copy %d.%d from %s: %v\n", rebuilder.info.Id, volumeId, shardId, ecNodes[0].info.Id, copyErr)
  191. } else {
  192. fmt.Fprintf(writer, "%s copied %d.%d from %s\n", rebuilder.info.Id, volumeId, shardId, ecNodes[0].info.Id)
  193. copiedShardIds = append(copiedShardIds, uint32(shardId))
  194. }
  195. }
  196. if len(copiedShardIds)+len(localShardIds) >= erasure_coding.DataShardsCount {
  197. return copiedShardIds, localShardIds, nil
  198. }
  199. return nil, nil, fmt.Errorf("%d shards are not enough to recover volume %d", len(copiedShardIds)+len(localShardIds), volumeId)
  200. }
  201. type EcShardMap map[needle.VolumeId]EcShardLocations
  202. type EcShardLocations [][]*EcNode
  203. func (ecShardMap EcShardMap) registerEcNode(ecNode *EcNode, collection string) {
  204. for _, diskInfo := range ecNode.info.DiskInfos {
  205. for _, shardInfo := range diskInfo.EcShardInfos {
  206. if shardInfo.Collection == collection {
  207. existing, found := ecShardMap[needle.VolumeId(shardInfo.Id)]
  208. if !found {
  209. existing = make([][]*EcNode, erasure_coding.TotalShardsCount)
  210. ecShardMap[needle.VolumeId(shardInfo.Id)] = existing
  211. }
  212. for _, shardId := range erasure_coding.ShardBits(shardInfo.EcIndexBits).ShardIds() {
  213. existing[shardId] = append(existing[shardId], ecNode)
  214. }
  215. }
  216. }
  217. }
  218. }
  219. func (ecShardLocations EcShardLocations) shardCount() (count int) {
  220. for _, locations := range ecShardLocations {
  221. if len(locations) > 0 {
  222. count++
  223. }
  224. }
  225. return
  226. }