You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

224 lines
7.0 KiB

  1. package storage
  2. import (
  3. "context"
  4. "fmt"
  5. "io"
  6. "os"
  7. "sort"
  8. "github.com/chrislusf/seaweedfs/weed/glog"
  9. "github.com/chrislusf/seaweedfs/weed/operation"
  10. "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
  11. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  12. . "github.com/chrislusf/seaweedfs/weed/storage/types"
  13. )
  14. // The volume sync with a master volume via 2 steps:
  15. // 1. The slave checks master side to find subscription checkpoint
  16. // to setup the replication.
  17. // 2. The slave receives the updates from master
  18. /*
  19. Assume the slave volume needs to follow the master volume.
  20. The master volume could be compacted, and could be many files ahead of
  21. slave volume.
  22. Step 1:
  23. The slave volume will ask the master volume for a snapshot
  24. of (existing file entries, last offset, number of compacted times).
  25. For each entry x in master existing file entries:
  26. if x does not exist locally:
  27. add x locally
  28. For each entry y in local slave existing file entries:
  29. if y does not exist on master:
  30. delete y locally
  31. Step 2:
  32. After this, use the last offset and number of compacted times to request
  33. the master volume to send a new file, and keep looping. If the number of
  34. compacted times is changed, go back to step 1 (very likely this can be
  35. optimized more later).
  36. */
  37. func (v *Volume) Synchronize(volumeServer string) (err error) {
  38. var lastCompactRevision uint16 = 0
  39. var compactRevision uint16 = 0
  40. var masterMap *needle.CompactMap
  41. for i := 0; i < 3; i++ {
  42. if masterMap, _, compactRevision, err = fetchVolumeFileEntries(volumeServer, v.Id); err != nil {
  43. return fmt.Errorf("Failed to sync volume %d entries with %s: %v", v.Id, volumeServer, err)
  44. }
  45. if lastCompactRevision != compactRevision && lastCompactRevision != 0 {
  46. if err = v.Compact(0); err != nil {
  47. return fmt.Errorf("Compact Volume before synchronizing %v", err)
  48. }
  49. if err = v.commitCompact(); err != nil {
  50. return fmt.Errorf("Commit Compact before synchronizing %v", err)
  51. }
  52. }
  53. lastCompactRevision = compactRevision
  54. if err = v.trySynchronizing(volumeServer, masterMap, compactRevision); err == nil {
  55. return
  56. }
  57. }
  58. return
  59. }
  60. type ByOffset []needle.NeedleValue
  61. func (a ByOffset) Len() int { return len(a) }
  62. func (a ByOffset) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
  63. func (a ByOffset) Less(i, j int) bool { return a[i].Offset < a[j].Offset }
  64. // trySynchronizing sync with remote volume server incrementally by
  65. // make up the local and remote delta.
  66. func (v *Volume) trySynchronizing(volumeServer string, masterMap *needle.CompactMap, compactRevision uint16) error {
  67. slaveIdxFile, err := os.Open(v.nm.IndexFileName())
  68. if err != nil {
  69. return fmt.Errorf("Open volume %d index file: %v", v.Id, err)
  70. }
  71. defer slaveIdxFile.Close()
  72. slaveMap, err := LoadBtreeNeedleMap(slaveIdxFile)
  73. if err != nil {
  74. return fmt.Errorf("Load volume %d index file: %v", v.Id, err)
  75. }
  76. var delta []needle.NeedleValue
  77. if err := masterMap.Visit(func(needleValue needle.NeedleValue) error {
  78. if needleValue.Key == NeedleIdEmpty {
  79. return nil
  80. }
  81. if _, ok := slaveMap.Get(needleValue.Key); ok {
  82. return nil // skip intersection
  83. }
  84. delta = append(delta, needleValue)
  85. return nil
  86. }); err != nil {
  87. return fmt.Errorf("Add master entry: %v", err)
  88. }
  89. if err := slaveMap.m.Visit(func(needleValue needle.NeedleValue) error {
  90. if needleValue.Key == NeedleIdEmpty {
  91. return nil
  92. }
  93. if _, ok := masterMap.Get(needleValue.Key); ok {
  94. return nil // skip intersection
  95. }
  96. needleValue.Size = 0
  97. delta = append(delta, needleValue)
  98. return nil
  99. }); err != nil {
  100. return fmt.Errorf("Remove local entry: %v", err)
  101. }
  102. // simulate to same ordering of remote .dat file needle entries
  103. sort.Sort(ByOffset(delta))
  104. // make up the delta
  105. fetchCount := 0
  106. for _, needleValue := range delta {
  107. if needleValue.Size == 0 {
  108. // remove file entry from local
  109. v.removeNeedle(needleValue.Key)
  110. continue
  111. }
  112. // add master file entry to local data file
  113. if err := v.fetchNeedle(volumeServer, needleValue, compactRevision); err != nil {
  114. glog.V(0).Infof("Fetch needle %v from %s: %v", needleValue, volumeServer, err)
  115. return err
  116. }
  117. fetchCount++
  118. }
  119. glog.V(1).Infof("Fetched %d needles from %s", fetchCount, volumeServer)
  120. return nil
  121. }
  122. func fetchVolumeFileEntries(volumeServer string, vid VolumeId) (m *needle.CompactMap, lastOffset uint64, compactRevision uint16, err error) {
  123. m = needle.NewCompactMap()
  124. syncStatus, err := operation.GetVolumeSyncStatus(volumeServer, uint32(vid))
  125. if err != nil {
  126. return m, 0, 0, err
  127. }
  128. total := 0
  129. err = operation.GetVolumeIdxEntries(volumeServer, uint32(vid), func(key NeedleId, offset Offset, size uint32) {
  130. // println("remote key", key, "offset", offset*NeedlePaddingSize, "size", size)
  131. if offset > 0 && size != TombstoneFileSize {
  132. m.Set(NeedleId(key), offset, size)
  133. } else {
  134. m.Delete(NeedleId(key))
  135. }
  136. total++
  137. })
  138. glog.V(2).Infof("server %s volume %d, entries %d, last offset %d, revision %d", volumeServer, vid, total, syncStatus.TailOffset, syncStatus.CompactRevision)
  139. return m, syncStatus.TailOffset, uint16(syncStatus.CompactRevision), err
  140. }
  141. func (v *Volume) GetVolumeSyncStatus() *volume_server_pb.VolumeSyncStatusResponse {
  142. var syncStatus = &volume_server_pb.VolumeSyncStatusResponse{}
  143. if stat, err := v.dataFile.Stat(); err == nil {
  144. syncStatus.TailOffset = uint64(stat.Size())
  145. }
  146. syncStatus.Collection = v.Collection
  147. syncStatus.IdxFileSize = v.nm.IndexFileSize()
  148. syncStatus.CompactRevision = uint32(v.SuperBlock.CompactRevision)
  149. syncStatus.Ttl = v.SuperBlock.Ttl.String()
  150. syncStatus.Replication = v.SuperBlock.ReplicaPlacement.String()
  151. return syncStatus
  152. }
  153. func (v *Volume) IndexFileContent() ([]byte, error) {
  154. return v.nm.IndexFileContent()
  155. }
  156. // removeNeedle removes one needle by needle key
  157. func (v *Volume) removeNeedle(key NeedleId) {
  158. n := new(Needle)
  159. n.Id = key
  160. v.deleteNeedle(n)
  161. }
  162. // fetchNeedle fetches a remote volume needle by vid, id, offset
  163. // The compact revision is checked first in case the remote volume
  164. // is compacted and the offset is invalid any more.
  165. func (v *Volume) fetchNeedle(volumeServer string, needleValue needle.NeedleValue, compactRevision uint16) error {
  166. return operation.WithVolumeServerClient(volumeServer, func(client volume_server_pb.VolumeServerClient) error {
  167. stream, err := client.VolumeSyncData(context.Background(), &volume_server_pb.VolumeSyncDataRequest{
  168. VolumdId: uint32(v.Id),
  169. Revision: uint32(compactRevision),
  170. Offset: uint32(needleValue.Offset),
  171. Size: uint32(needleValue.Size),
  172. NeedleId: needleValue.Key.String(),
  173. })
  174. if err != nil {
  175. return err
  176. }
  177. var fileContent []byte
  178. for {
  179. resp, err := stream.Recv()
  180. if err == io.EOF {
  181. break
  182. }
  183. if err != nil {
  184. return fmt.Errorf("read needle %v: %v", needleValue.Key.String(), err)
  185. }
  186. fileContent = append(fileContent, resp.FileContent...)
  187. }
  188. offset, err := v.AppendBlob(fileContent)
  189. if err != nil {
  190. return fmt.Errorf("Appending volume %d error: %v", v.Id, err)
  191. }
  192. // println("add key", needleValue.Key, "offset", offset, "size", needleValue.Size)
  193. v.nm.Put(needleValue.Key, Offset(offset/NeedlePaddingSize), needleValue.Size)
  194. return nil
  195. })
  196. }