You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

643 lines
21 KiB

3 years ago
6 years ago
4 years ago
6 years ago
1 month ago
3 years ago
4 years ago
3 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
2 years ago
4 years ago
4 years ago
4 years ago
  1. package shell
  2. import (
  3. "context"
  4. "flag"
  5. "fmt"
  6. "io"
  7. "path/filepath"
  8. "strconv"
  9. "time"
  10. "github.com/seaweedfs/seaweedfs/weed/pb"
  11. "github.com/seaweedfs/seaweedfs/weed/storage/needle"
  12. "github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
  13. "github.com/seaweedfs/seaweedfs/weed/storage/types"
  14. "golang.org/x/exp/slices"
  15. "google.golang.org/grpc"
  16. "github.com/seaweedfs/seaweedfs/weed/operation"
  17. "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
  18. "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
  19. "github.com/seaweedfs/seaweedfs/weed/storage/super_block"
  20. )
  21. func init() {
  22. Commands = append(Commands, &commandVolumeFixReplication{})
  23. }
  24. type commandVolumeFixReplication struct {
  25. collectionPattern *string
  26. }
  27. func (c *commandVolumeFixReplication) Name() string {
  28. return "volume.fix.replication"
  29. }
  30. func (c *commandVolumeFixReplication) Help() string {
  31. return `add or remove replicas to volumes that are missing replicas or over-replicated
  32. This command finds all over-replicated volumes. If found, it will purge the oldest copies and stop.
  33. This command also finds all under-replicated volumes, and finds volume servers with free slots.
  34. If the free slots satisfy the replication requirement, the volume content is copied over and mounted.
  35. volume.fix.replication -n # do not take action
  36. volume.fix.replication # actually deleting or copying the volume files and mount the volume
  37. volume.fix.replication -collectionPattern=important* # fix any collections with prefix "important"
  38. Note:
  39. * each time this will only add back one replica for each volume id that is under replicated.
  40. If there are multiple replicas are missing, e.g. replica count is > 2, you may need to run this multiple times.
  41. * do not run this too quickly within seconds, since the new volume replica may take a few seconds
  42. to register itself to the master.
  43. `
  44. }
  45. func (c *commandVolumeFixReplication) HasTag(tag CommandTag) bool {
  46. return false && tag == ResourceHeavy // resource intensive only when deleting and checking with replicas.
  47. }
  48. func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
  49. volFixReplicationCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
  50. c.collectionPattern = volFixReplicationCommand.String("collectionPattern", "", "match with wildcard characters '*' and '?'")
  51. skipChange := volFixReplicationCommand.Bool("n", false, "skip the changes")
  52. doDelete := volFixReplicationCommand.Bool("doDelete", true, "Also delete over-replicated volumes besides fixing under-replication")
  53. doCheck := volFixReplicationCommand.Bool("doCheck", true, "Also check synchronization before deleting")
  54. retryCount := volFixReplicationCommand.Int("retry", 5, "how many times to retry")
  55. volumesPerStep := volFixReplicationCommand.Int("volumesPerStep", 0, "how many volumes to fix in one cycle")
  56. if err = volFixReplicationCommand.Parse(args); err != nil {
  57. return nil
  58. }
  59. commandEnv.noLock = *skipChange
  60. takeAction := !*skipChange
  61. if err = commandEnv.confirmIsLocked(args); takeAction && err != nil {
  62. return
  63. }
  64. underReplicatedVolumeIdsCount := 1
  65. for underReplicatedVolumeIdsCount > 0 {
  66. fixedVolumeReplicas := map[string]int{}
  67. // collect topology information
  68. topologyInfo, _, err := collectTopologyInfo(commandEnv, 15*time.Second)
  69. if err != nil {
  70. return err
  71. }
  72. // find all volumes that needs replication
  73. // collect all data nodes
  74. volumeReplicas, allLocations := collectVolumeReplicaLocations(topologyInfo)
  75. if len(allLocations) == 0 {
  76. return fmt.Errorf("no data nodes at all")
  77. }
  78. // find all under replicated volumes
  79. var underReplicatedVolumeIds, overReplicatedVolumeIds, misplacedVolumeIds []uint32
  80. for vid, replicas := range volumeReplicas {
  81. replica := replicas[0]
  82. replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement))
  83. switch {
  84. case replicaPlacement.GetCopyCount() > len(replicas) || !satisfyReplicaCurrentLocation(replicaPlacement, replicas):
  85. underReplicatedVolumeIds = append(underReplicatedVolumeIds, vid)
  86. case isMisplaced(replicas, replicaPlacement):
  87. misplacedVolumeIds = append(misplacedVolumeIds, vid)
  88. fmt.Fprintf(writer, "volume %d replication %s is not well placed %s\n", replica.info.Id, replicaPlacement, replica.location.dataNode.Id)
  89. case replicaPlacement.GetCopyCount() < len(replicas):
  90. overReplicatedVolumeIds = append(overReplicatedVolumeIds, vid)
  91. fmt.Fprintf(writer, "volume %d replication %s, but over replicated %+d\n", replica.info.Id, replicaPlacement, len(replicas))
  92. }
  93. }
  94. if !commandEnv.isLocked() {
  95. return fmt.Errorf("lock is lost")
  96. }
  97. if len(overReplicatedVolumeIds) > 0 && *doDelete {
  98. if err := c.deleteOneVolume(commandEnv, writer, takeAction, *doCheck, overReplicatedVolumeIds, volumeReplicas, allLocations, pickOneReplicaToDelete); err != nil {
  99. return err
  100. }
  101. }
  102. if len(misplacedVolumeIds) > 0 && *doDelete {
  103. if err := c.deleteOneVolume(commandEnv, writer, takeAction, *doCheck, misplacedVolumeIds, volumeReplicas, allLocations, pickOneMisplacedVolume); err != nil {
  104. return err
  105. }
  106. }
  107. underReplicatedVolumeIdsCount = len(underReplicatedVolumeIds)
  108. if underReplicatedVolumeIdsCount > 0 {
  109. // find the most underpopulated data nodes
  110. fixedVolumeReplicas, err = c.fixUnderReplicatedVolumes(commandEnv, writer, takeAction, underReplicatedVolumeIds, volumeReplicas, allLocations, *retryCount, *volumesPerStep)
  111. if err != nil {
  112. return err
  113. }
  114. }
  115. if *skipChange {
  116. break
  117. }
  118. // check that the topology has been updated
  119. if len(fixedVolumeReplicas) > 0 {
  120. fixedVolumes := make([]string, 0, len(fixedVolumeReplicas))
  121. for k, _ := range fixedVolumeReplicas {
  122. fixedVolumes = append(fixedVolumes, k)
  123. }
  124. volumeIdLocations, err := lookupVolumeIds(commandEnv, fixedVolumes)
  125. if err != nil {
  126. return err
  127. }
  128. for _, volumeIdLocation := range volumeIdLocations {
  129. volumeId := volumeIdLocation.VolumeOrFileId
  130. volumeIdLocationCount := len(volumeIdLocation.Locations)
  131. i := 0
  132. for fixedVolumeReplicas[volumeId] >= volumeIdLocationCount {
  133. fmt.Fprintf(writer, "the number of locations for volume %s has not increased yet, let's wait\n", volumeId)
  134. time.Sleep(time.Duration(i+1) * time.Second * 7)
  135. volumeLocIds, err := lookupVolumeIds(commandEnv, []string{volumeId})
  136. if err != nil {
  137. return err
  138. }
  139. volumeIdLocationCount = len(volumeLocIds[0].Locations)
  140. if *retryCount <= i {
  141. return fmt.Errorf("replicas volume %s mismatch in topology", volumeId)
  142. }
  143. i += 1
  144. }
  145. }
  146. }
  147. }
  148. return nil
  149. }
  150. func collectVolumeReplicaLocations(topologyInfo *master_pb.TopologyInfo) (map[uint32][]*VolumeReplica, []location) {
  151. volumeReplicas := make(map[uint32][]*VolumeReplica)
  152. var allLocations []location
  153. eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
  154. loc := newLocation(string(dc), string(rack), dn)
  155. for _, diskInfo := range dn.DiskInfos {
  156. for _, v := range diskInfo.VolumeInfos {
  157. volumeReplicas[v.Id] = append(volumeReplicas[v.Id], &VolumeReplica{
  158. location: &loc,
  159. info: v,
  160. })
  161. }
  162. }
  163. allLocations = append(allLocations, loc)
  164. })
  165. return volumeReplicas, allLocations
  166. }
  167. type SelectOneVolumeFunc func(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica
  168. func checkOneVolume(a *VolumeReplica, b *VolumeReplica, writer io.Writer, grpcDialOption grpc.DialOption) (err error) {
  169. aDB, bDB := needle_map.NewMemDb(), needle_map.NewMemDb()
  170. defer func() {
  171. aDB.Close()
  172. bDB.Close()
  173. }()
  174. // read index db
  175. readIndexDbCutoffFrom := uint64(time.Now().UnixNano())
  176. if err = readIndexDatabase(aDB, a.info.Collection, a.info.Id, pb.NewServerAddressFromDataNode(a.location.dataNode), false, writer, grpcDialOption); err != nil {
  177. return fmt.Errorf("readIndexDatabase %s volume %d: %v", a.location.dataNode, a.info.Id, err)
  178. }
  179. if err := readIndexDatabase(bDB, b.info.Collection, b.info.Id, pb.NewServerAddressFromDataNode(b.location.dataNode), false, writer, grpcDialOption); err != nil {
  180. return fmt.Errorf("readIndexDatabase %s volume %d: %v", b.location.dataNode, b.info.Id, err)
  181. }
  182. if _, err = doVolumeCheckDisk(aDB, bDB, a, b, false, writer, true, false, float64(1), readIndexDbCutoffFrom, grpcDialOption); err != nil {
  183. return fmt.Errorf("doVolumeCheckDisk source:%s target:%s volume %d: %v", a.location.dataNode.Id, b.location.dataNode.Id, a.info.Id, err)
  184. }
  185. return
  186. }
  187. func (c *commandVolumeFixReplication) deleteOneVolume(commandEnv *CommandEnv, writer io.Writer, takeAction bool, doCheck bool, overReplicatedVolumeIds []uint32, volumeReplicas map[uint32][]*VolumeReplica, allLocations []location, selectOneVolumeFn SelectOneVolumeFunc) error {
  188. for _, vid := range overReplicatedVolumeIds {
  189. replicas := volumeReplicas[vid]
  190. replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replicas[0].info.ReplicaPlacement))
  191. replica := selectOneVolumeFn(replicas, replicaPlacement)
  192. // check collection name pattern
  193. if *c.collectionPattern != "" {
  194. matched, err := filepath.Match(*c.collectionPattern, replica.info.Collection)
  195. if err != nil {
  196. return fmt.Errorf("match pattern %s with collection %s: %v", *c.collectionPattern, replica.info.Collection, err)
  197. }
  198. if !matched {
  199. break
  200. }
  201. }
  202. collectionIsMismatch := false
  203. for _, volumeReplica := range replicas {
  204. if volumeReplica.info.Collection != replica.info.Collection {
  205. fmt.Fprintf(writer, "skip delete volume %d as collection %s is mismatch: %s\n", replica.info.Id, replica.info.Collection, volumeReplica.info.Collection)
  206. collectionIsMismatch = true
  207. }
  208. }
  209. if collectionIsMismatch {
  210. continue
  211. }
  212. fmt.Fprintf(writer, "deleting volume %d from %s ...\n", replica.info.Id, replica.location.dataNode.Id)
  213. if !takeAction {
  214. break
  215. }
  216. if doCheck {
  217. for _, replicaB := range replicas {
  218. if replicaB.location.dataNode == replica.location.dataNode {
  219. continue
  220. }
  221. if err := checkOneVolume(replica, replicaB, writer, commandEnv.option.GrpcDialOption); err != nil {
  222. return fmt.Errorf("sync volume %d on %s and %s: %v\n", replica.info.Id, replica.location.dataNode.Id, replicaB.location.dataNode.Id, err)
  223. }
  224. }
  225. }
  226. if err := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(replica.info.Id),
  227. pb.NewServerAddressFromDataNode(replica.location.dataNode), false); err != nil {
  228. return fmt.Errorf("deleting volume %d from %s : %v", replica.info.Id, replica.location.dataNode.Id, err)
  229. }
  230. }
  231. return nil
  232. }
  233. func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *CommandEnv, writer io.Writer, takeAction bool, underReplicatedVolumeIds []uint32, volumeReplicas map[uint32][]*VolumeReplica, allLocations []location, retryCount int, volumesPerStep int) (fixedVolumes map[string]int, err error) {
  234. fixedVolumes = map[string]int{}
  235. if len(underReplicatedVolumeIds) > volumesPerStep && volumesPerStep > 0 {
  236. underReplicatedVolumeIds = underReplicatedVolumeIds[0:volumesPerStep]
  237. }
  238. for _, vid := range underReplicatedVolumeIds {
  239. for i := 0; i < retryCount+1; i++ {
  240. if err = c.fixOneUnderReplicatedVolume(commandEnv, writer, takeAction, volumeReplicas, vid, allLocations); err == nil {
  241. if takeAction {
  242. fixedVolumes[strconv.FormatUint(uint64(vid), 10)] = len(volumeReplicas[vid])
  243. }
  244. break
  245. } else {
  246. fmt.Fprintf(writer, "fixing under replicated volume %d: %v\n", vid, err)
  247. }
  248. }
  249. }
  250. return fixedVolumes, nil
  251. }
  252. func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *CommandEnv, writer io.Writer, takeAction bool, volumeReplicas map[uint32][]*VolumeReplica, vid uint32, allLocations []location) error {
  253. replicas := volumeReplicas[vid]
  254. replica := pickOneReplicaToCopyFrom(replicas)
  255. replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement))
  256. foundNewLocation := false
  257. hasSkippedCollection := false
  258. keepDataNodesSorted(allLocations, types.ToDiskType(replica.info.DiskType))
  259. fn := capacityByFreeVolumeCount(types.ToDiskType(replica.info.DiskType))
  260. for _, dst := range allLocations {
  261. // check whether data nodes satisfy the constraints
  262. if fn(dst.dataNode) > 0 && satisfyReplicaPlacement(replicaPlacement, replicas, dst) {
  263. // check collection name pattern
  264. if *c.collectionPattern != "" {
  265. matched, err := filepath.Match(*c.collectionPattern, replica.info.Collection)
  266. if err != nil {
  267. return fmt.Errorf("match pattern %s with collection %s: %v", *c.collectionPattern, replica.info.Collection, err)
  268. }
  269. if !matched {
  270. hasSkippedCollection = true
  271. break
  272. }
  273. }
  274. // ask the volume server to replicate the volume
  275. foundNewLocation = true
  276. fmt.Fprintf(writer, "replicating volume %d %s from %s to dataNode %s ...\n", replica.info.Id, replicaPlacement, replica.location.dataNode.Id, dst.dataNode.Id)
  277. if !takeAction {
  278. // adjust volume count
  279. addVolumeCount(dst.dataNode.DiskInfos[replica.info.DiskType], 1)
  280. break
  281. }
  282. err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(dst.dataNode), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
  283. stream, replicateErr := volumeServerClient.VolumeCopy(context.Background(), &volume_server_pb.VolumeCopyRequest{
  284. VolumeId: replica.info.Id,
  285. SourceDataNode: string(pb.NewServerAddressFromDataNode(replica.location.dataNode)),
  286. })
  287. if replicateErr != nil {
  288. return fmt.Errorf("copying from %s => %s : %v", replica.location.dataNode.Id, dst.dataNode.Id, replicateErr)
  289. }
  290. for {
  291. resp, recvErr := stream.Recv()
  292. if recvErr != nil {
  293. if recvErr == io.EOF {
  294. break
  295. } else {
  296. return recvErr
  297. }
  298. }
  299. if resp.ProcessedBytes > 0 {
  300. fmt.Fprintf(writer, "volume %d processed %d bytes\n", replica.info.Id, resp.ProcessedBytes)
  301. }
  302. }
  303. return nil
  304. })
  305. if err != nil {
  306. return err
  307. }
  308. // adjust volume count
  309. addVolumeCount(dst.dataNode.DiskInfos[replica.info.DiskType], 1)
  310. break
  311. }
  312. }
  313. if !foundNewLocation && !hasSkippedCollection {
  314. fmt.Fprintf(writer, "failed to place volume %d replica as %s, existing:%+v\n", replica.info.Id, replicaPlacement, len(replicas))
  315. }
  316. return nil
  317. }
  318. func addVolumeCount(info *master_pb.DiskInfo, count int) {
  319. if info == nil {
  320. return
  321. }
  322. info.VolumeCount += int64(count)
  323. info.FreeVolumeCount -= int64(count)
  324. }
  325. func keepDataNodesSorted(dataNodes []location, diskType types.DiskType) {
  326. fn := capacityByFreeVolumeCount(diskType)
  327. slices.SortFunc(dataNodes, func(a, b location) int {
  328. return int(fn(b.dataNode) - fn(a.dataNode))
  329. })
  330. }
  331. func satisfyReplicaCurrentLocation(replicaPlacement *super_block.ReplicaPlacement, replicas []*VolumeReplica) bool {
  332. existingDataCenters, existingRacks, _ := countReplicas(replicas)
  333. if replicaPlacement.DiffDataCenterCount+1 > len(existingDataCenters) {
  334. return false
  335. }
  336. if replicaPlacement.DiffRackCount+1 > len(existingRacks) {
  337. return false
  338. }
  339. if replicaPlacement.SameRackCount > 0 {
  340. foundSatisfyRack := false
  341. for _, rackCount := range existingRacks {
  342. if rackCount >= replicaPlacement.SameRackCount+1 {
  343. foundSatisfyRack = true
  344. }
  345. }
  346. return foundSatisfyRack
  347. }
  348. return true
  349. }
  350. /*
  351. if on an existing data node {
  352. return false
  353. }
  354. if different from existing dcs {
  355. if lack on different dcs {
  356. return true
  357. }else{
  358. return false
  359. }
  360. }
  361. if not on primary dc {
  362. return false
  363. }
  364. if different from existing racks {
  365. if lack on different racks {
  366. return true
  367. }else{
  368. return false
  369. }
  370. }
  371. if not on primary rack {
  372. return false
  373. }
  374. if lacks on same rack {
  375. return true
  376. } else {
  377. return false
  378. }
  379. */
  380. func satisfyReplicaPlacement(replicaPlacement *super_block.ReplicaPlacement, replicas []*VolumeReplica, possibleLocation location) bool {
  381. existingDataCenters, _, existingDataNodes := countReplicas(replicas)
  382. if _, found := existingDataNodes[possibleLocation.String()]; found {
  383. // avoid duplicated volume on the same data node
  384. return false
  385. }
  386. primaryDataCenters, _ := findTopKeys(existingDataCenters)
  387. // ensure data center count is within limit
  388. if _, found := existingDataCenters[possibleLocation.DataCenter()]; !found {
  389. // different from existing dcs
  390. if len(existingDataCenters) < replicaPlacement.DiffDataCenterCount+1 {
  391. // lack on different dcs
  392. return true
  393. } else {
  394. // adding this would go over the different dcs limit
  395. return false
  396. }
  397. }
  398. // now this is same as one of the existing data center
  399. if !isAmong(possibleLocation.DataCenter(), primaryDataCenters) {
  400. // not on one of the primary dcs
  401. return false
  402. }
  403. // now this is one of the primary dcs
  404. primaryDcRacks := make(map[string]int)
  405. for _, replica := range replicas {
  406. if replica.location.DataCenter() != possibleLocation.DataCenter() {
  407. continue
  408. }
  409. primaryDcRacks[replica.location.Rack()] += 1
  410. }
  411. primaryRacks, _ := findTopKeys(primaryDcRacks)
  412. sameRackCount := primaryDcRacks[possibleLocation.Rack()]
  413. // ensure rack count is within limit
  414. if _, found := primaryDcRacks[possibleLocation.Rack()]; !found {
  415. // different from existing racks
  416. if len(primaryDcRacks) < replicaPlacement.DiffRackCount+1 {
  417. // lack on different racks
  418. return true
  419. } else {
  420. // adding this would go over the different racks limit
  421. return false
  422. }
  423. }
  424. // now this is same as one of the existing racks
  425. if !isAmong(possibleLocation.Rack(), primaryRacks) {
  426. // not on the primary rack
  427. return false
  428. }
  429. // now this is on the primary rack
  430. // different from existing data nodes
  431. if sameRackCount < replicaPlacement.SameRackCount+1 {
  432. // lack on same rack
  433. return true
  434. } else {
  435. // adding this would go over the same data node limit
  436. return false
  437. }
  438. }
  439. func findTopKeys(m map[string]int) (topKeys []string, max int) {
  440. for k, c := range m {
  441. if max < c {
  442. topKeys = topKeys[:0]
  443. topKeys = append(topKeys, k)
  444. max = c
  445. } else if max == c {
  446. topKeys = append(topKeys, k)
  447. }
  448. }
  449. return
  450. }
  451. func isAmong(key string, keys []string) bool {
  452. for _, k := range keys {
  453. if k == key {
  454. return true
  455. }
  456. }
  457. return false
  458. }
  459. type VolumeReplica struct {
  460. location *location
  461. info *master_pb.VolumeInformationMessage
  462. }
  463. type location struct {
  464. dc string
  465. rack string
  466. dataNode *master_pb.DataNodeInfo
  467. }
  468. func newLocation(dc, rack string, dataNode *master_pb.DataNodeInfo) location {
  469. return location{
  470. dc: dc,
  471. rack: rack,
  472. dataNode: dataNode,
  473. }
  474. }
  475. func (l location) String() string {
  476. return fmt.Sprintf("%s %s %s", l.dc, l.rack, l.dataNode.Id)
  477. }
  478. func (l location) Rack() string {
  479. return fmt.Sprintf("%s %s", l.dc, l.rack)
  480. }
  481. func (l location) DataCenter() string {
  482. return l.dc
  483. }
  484. func pickOneReplicaToCopyFrom(replicas []*VolumeReplica) *VolumeReplica {
  485. mostRecent := replicas[0]
  486. for _, replica := range replicas {
  487. if replica.info.ModifiedAtSecond > mostRecent.info.ModifiedAtSecond {
  488. mostRecent = replica
  489. }
  490. }
  491. return mostRecent
  492. }
  493. func countReplicas(replicas []*VolumeReplica) (diffDc, diffRack, diffNode map[string]int) {
  494. diffDc = make(map[string]int)
  495. diffRack = make(map[string]int)
  496. diffNode = make(map[string]int)
  497. for _, replica := range replicas {
  498. diffDc[replica.location.DataCenter()] += 1
  499. diffRack[replica.location.Rack()] += 1
  500. diffNode[replica.location.String()] += 1
  501. }
  502. return
  503. }
  504. func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica {
  505. slices.SortFunc(replicas, func(a, b *VolumeReplica) int {
  506. if a.info.Size != b.info.Size {
  507. return int(a.info.Size - b.info.Size)
  508. }
  509. if a.info.ModifiedAtSecond != b.info.ModifiedAtSecond {
  510. return int(a.info.ModifiedAtSecond - b.info.ModifiedAtSecond)
  511. }
  512. if a.info.CompactRevision != b.info.CompactRevision {
  513. return int(a.info.CompactRevision - b.info.CompactRevision)
  514. }
  515. return 0
  516. })
  517. return replicas[0]
  518. }
  519. // check and fix misplaced volumes
  520. func isMisplaced(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) bool {
  521. for i := 0; i < len(replicas); i++ {
  522. others := otherThan(replicas, i)
  523. if !satisfyReplicaPlacement(replicaPlacement, others, *replicas[i].location) {
  524. return true
  525. }
  526. }
  527. return false
  528. }
  529. func otherThan(replicas []*VolumeReplica, index int) (others []*VolumeReplica) {
  530. for i := 0; i < len(replicas); i++ {
  531. if index != i {
  532. others = append(others, replicas[i])
  533. }
  534. }
  535. return
  536. }
  537. func pickOneMisplacedVolume(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) (toDelete *VolumeReplica) {
  538. var deletionCandidates []*VolumeReplica
  539. for i := 0; i < len(replicas); i++ {
  540. others := otherThan(replicas, i)
  541. if !isMisplaced(others, replicaPlacement) {
  542. deletionCandidates = append(deletionCandidates, replicas[i])
  543. }
  544. }
  545. if len(deletionCandidates) > 0 {
  546. return pickOneReplicaToDelete(deletionCandidates, replicaPlacement)
  547. }
  548. return pickOneReplicaToDelete(replicas, replicaPlacement)
  549. }