You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

93 lines
3.0 KiB

3 years ago
3 years ago
3 years ago
  1. package command
  2. import (
  3. "fmt"
  4. "github.com/chrislusf/seaweedfs/weed/glog"
  5. "github.com/chrislusf/seaweedfs/weed/pb"
  6. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  7. "github.com/chrislusf/seaweedfs/weed/replication/source"
  8. "github.com/chrislusf/seaweedfs/weed/security"
  9. "github.com/chrislusf/seaweedfs/weed/util"
  10. "google.golang.org/grpc"
  11. "time"
  12. )
  13. type RemoteSyncOptions struct {
  14. filerAddress *string
  15. grpcDialOption grpc.DialOption
  16. readChunkFromFiler *bool
  17. timeAgo *time.Duration
  18. dir *string
  19. }
  20. var _ = filer_pb.FilerClient(&RemoteSyncOptions{})
  21. func (option *RemoteSyncOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
  22. return pb.WithFilerClient(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
  23. return fn(client)
  24. })
  25. }
  26. func (option *RemoteSyncOptions) AdjustedUrl(location *filer_pb.Location) string {
  27. return location.Url
  28. }
  29. var (
  30. remoteSyncOptions RemoteSyncOptions
  31. )
  32. func init() {
  33. cmdFilerRemoteSynchronize.Run = runFilerRemoteSynchronize // break init cycle
  34. remoteSyncOptions.filerAddress = cmdFilerRemoteSynchronize.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster")
  35. remoteSyncOptions.dir = cmdFilerRemoteSynchronize.Flag.String("dir", "", "a mounted directory on filer")
  36. remoteSyncOptions.readChunkFromFiler = cmdFilerRemoteSynchronize.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers")
  37. remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
  38. }
  39. var cmdFilerRemoteSynchronize = &Command{
  40. UsageLine: "filer.remote.sync",
  41. Short: "resumable continuously write back updates to remote storage",
  42. Long: `resumable continuously write back updates to remote storage
  43. filer.remote.sync listens on filer update events.
  44. If any mounted remote file is updated, it will fetch the updated content,
  45. and write to the remote storage.
  46. weed filer.remote.sync -dir=/mount/s3_on_cloud
  47. `,
  48. }
  49. func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
  50. util.LoadConfiguration("security", false)
  51. grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
  52. remoteSyncOptions.grpcDialOption = grpcDialOption
  53. dir := *remoteSyncOptions.dir
  54. filerAddress := pb.ServerAddress(*remoteSyncOptions.filerAddress)
  55. filerSource := &source.FilerSource{}
  56. filerSource.DoInitialize(
  57. filerAddress.ToHttpAddress(),
  58. filerAddress.ToGrpcAddress(),
  59. "/", // does not matter
  60. *remoteSyncOptions.readChunkFromFiler,
  61. )
  62. if dir != "" {
  63. fmt.Printf("synchronize %s to remote storage...\n", dir)
  64. util.RetryForever("filer.remote.sync "+dir, func() error {
  65. return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir)
  66. }, func(err error) bool {
  67. if err != nil {
  68. glog.Errorf("synchronize %s: %v", dir, err)
  69. }
  70. return true
  71. })
  72. return true
  73. }
  74. return true
  75. }