You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

238 lines
9.0 KiB

3 years ago
3 years ago
3 years ago
  1. package command
  2. import (
  3. "context"
  4. "fmt"
  5. "os"
  6. "strings"
  7. "time"
  8. "github.com/seaweedfs/seaweedfs/weed/filer"
  9. "github.com/seaweedfs/seaweedfs/weed/glog"
  10. "github.com/seaweedfs/seaweedfs/weed/pb"
  11. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  12. "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
  13. "github.com/seaweedfs/seaweedfs/weed/remote_storage"
  14. "github.com/seaweedfs/seaweedfs/weed/replication/source"
  15. "github.com/seaweedfs/seaweedfs/weed/util"
  16. "google.golang.org/grpc"
  17. "google.golang.org/protobuf/proto"
  18. )
  19. func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string) error {
  20. // read filer remote storage mount mappings
  21. _, _, remoteStorageMountLocation, remoteStorage, detectErr := filer.DetectMountInfo(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir)
  22. if detectErr != nil {
  23. return fmt.Errorf("read mount info: %v", detectErr)
  24. }
  25. eachEntryFunc, err := makeEventProcessor(remoteStorage, mountedDir, remoteStorageMountLocation, filerSource)
  26. if err != nil {
  27. return err
  28. }
  29. processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
  30. lastTime := time.Unix(0, lastTsNs)
  31. glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
  32. return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, lastTsNs)
  33. })
  34. lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)
  35. option.clientEpoch++
  36. return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId, option.clientEpoch,
  37. mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
  38. }
  39. func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
  40. client, err := remote_storage.GetRemoteStorage(remoteStorage)
  41. if err != nil {
  42. return nil, err
  43. }
  44. handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
  45. message := resp.EventNotification
  46. if message.NewEntry == nil {
  47. return nil
  48. }
  49. if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
  50. mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
  51. if readErr != nil {
  52. return fmt.Errorf("unmarshal mappings: %v", readErr)
  53. }
  54. if remoteLoc, found := mappings.Mappings[mountedDir]; found {
  55. if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
  56. glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
  57. }
  58. } else {
  59. glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
  60. os.Exit(0)
  61. }
  62. }
  63. if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
  64. conf := &remote_pb.RemoteConf{}
  65. if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
  66. return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
  67. }
  68. remoteStorage = conf
  69. if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
  70. client = newClient
  71. } else {
  72. return err
  73. }
  74. }
  75. return nil
  76. }
  77. eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
  78. message := resp.EventNotification
  79. if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
  80. return handleEtcRemoteChanges(resp)
  81. }
  82. if filer_pb.IsEmpty(resp) {
  83. return nil
  84. }
  85. if filer_pb.IsCreate(resp) {
  86. if !filer.HasData(message.NewEntry) {
  87. return nil
  88. }
  89. glog.V(2).Infof("create: %+v", resp)
  90. if !shouldSendToRemote(message.NewEntry) {
  91. glog.V(2).Infof("skipping creating: %+v", resp)
  92. return nil
  93. }
  94. dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
  95. if message.NewEntry.IsDirectory {
  96. glog.V(0).Infof("mkdir %s", remote_storage.FormatLocation(dest))
  97. return client.WriteDirectory(dest, message.NewEntry)
  98. }
  99. glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
  100. remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
  101. if writeErr != nil {
  102. return writeErr
  103. }
  104. return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
  105. }
  106. if filer_pb.IsDelete(resp) {
  107. glog.V(2).Infof("delete: %+v", resp)
  108. dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
  109. if message.OldEntry.IsDirectory {
  110. glog.V(0).Infof("rmdir %s", remote_storage.FormatLocation(dest))
  111. return client.RemoveDirectory(dest)
  112. }
  113. glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
  114. return client.DeleteFile(dest)
  115. }
  116. if message.OldEntry != nil && message.NewEntry != nil {
  117. oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
  118. dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
  119. if !shouldSendToRemote(message.NewEntry) {
  120. glog.V(2).Infof("skipping updating: %+v", resp)
  121. return nil
  122. }
  123. if message.NewEntry.IsDirectory {
  124. return client.WriteDirectory(dest, message.NewEntry)
  125. }
  126. if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
  127. if filer.IsSameData(message.OldEntry, message.NewEntry) {
  128. glog.V(2).Infof("update meta: %+v", resp)
  129. return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry)
  130. }
  131. }
  132. glog.V(2).Infof("update: %+v", resp)
  133. glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
  134. if err := client.DeleteFile(oldDest); err != nil {
  135. return err
  136. }
  137. remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
  138. if writeErr != nil {
  139. return writeErr
  140. }
  141. return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
  142. }
  143. return nil
  144. }
  145. return eachEntryFunc, nil
  146. }
  147. func retriedWriteFile(client remote_storage.RemoteStorageClient, filerSource *source.FilerSource, newEntry *filer_pb.Entry, dest *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
  148. var writeErr error
  149. err = util.Retry("writeFile", func() error {
  150. reader := filer.NewFileReader(filerSource, newEntry)
  151. glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
  152. remoteEntry, writeErr = client.WriteFile(dest, newEntry, reader)
  153. if writeErr != nil {
  154. return writeErr
  155. }
  156. return nil
  157. })
  158. if err != nil {
  159. glog.Errorf("write to %s: %v", dest, err)
  160. }
  161. return
  162. }
  163. func collectLastSyncOffset(filerClient filer_pb.FilerClient, grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, mountedDir string, timeAgo time.Duration) time.Time {
  164. // 1. specified by timeAgo
  165. // 2. last offset timestamp for this directory
  166. // 3. directory creation time
  167. var lastOffsetTs time.Time
  168. if timeAgo == 0 {
  169. mountedDirEntry, err := filer_pb.GetEntry(filerClient, util.FullPath(mountedDir))
  170. if err != nil {
  171. glog.V(0).Infof("get mounted directory %s: %v", mountedDir, err)
  172. return time.Now()
  173. }
  174. lastOffsetTsNs, err := remote_storage.GetSyncOffset(grpcDialOption, filerAddress, mountedDir)
  175. if mountedDirEntry != nil {
  176. if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 {
  177. lastOffsetTs = time.Unix(0, lastOffsetTsNs)
  178. glog.V(0).Infof("resume from %v", lastOffsetTs)
  179. } else {
  180. lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0)
  181. }
  182. } else {
  183. lastOffsetTs = time.Now()
  184. }
  185. } else {
  186. lastOffsetTs = time.Now().Add(-timeAgo)
  187. }
  188. return lastOffsetTs
  189. }
  190. func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation {
  191. source := string(sourcePath[len(mountDir):])
  192. dest := util.FullPath(remoteMountLocation.Path).Child(source)
  193. return &remote_pb.RemoteStorageLocation{
  194. Name: remoteMountLocation.Name,
  195. Bucket: remoteMountLocation.Bucket,
  196. Path: string(dest),
  197. }
  198. }
  199. func shouldSendToRemote(entry *filer_pb.Entry) bool {
  200. if entry.RemoteEntry == nil {
  201. return true
  202. }
  203. if entry.RemoteEntry.RemoteMtime < entry.Attributes.Mtime {
  204. return true
  205. }
  206. return false
  207. }
  208. func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
  209. remoteEntry.LastLocalSyncTsNs = time.Now().UnixNano()
  210. entry.RemoteEntry = remoteEntry
  211. return filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
  212. _, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
  213. Directory: dir,
  214. Entry: entry,
  215. })
  216. return err
  217. })
  218. }