You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

233 lines
7.2 KiB

3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
  1. package shell
  2. import (
  3. "context"
  4. "flag"
  5. "fmt"
  6. "github.com/chrislusf/seaweedfs/weed/filer"
  7. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  8. "github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
  9. "github.com/chrislusf/seaweedfs/weed/remote_storage"
  10. "github.com/chrislusf/seaweedfs/weed/util"
  11. "github.com/golang/protobuf/jsonpb"
  12. "github.com/golang/protobuf/proto"
  13. "io"
  14. "os"
  15. "strings"
  16. "time"
  17. )
  18. func init() {
  19. Commands = append(Commands, &commandRemoteMount{})
  20. }
  21. type commandRemoteMount struct {
  22. }
  23. func (c *commandRemoteMount) Name() string {
  24. return "remote.mount"
  25. }
  26. func (c *commandRemoteMount) Help() string {
  27. return `mount remote storage and pull its metadata
  28. # assume a remote storage is configured to name "cloud1"
  29. remote.configure -name=cloud1 -type=s3 -access_key=xxx -secret_key=yyy
  30. # mount and pull one bucket
  31. remote.mount -dir=/xxx -remote=cloud1/bucket
  32. # mount and pull one directory in the bucket
  33. remote.mount -dir=/xxx -remote=cloud1/bucket/dir1
  34. # after mount, start a separate process to write updates to remote storage
  35. weed filer.remote.sync -filer=<filerHost>:<filerPort> -dir=/xxx
  36. `
  37. }
  38. func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
  39. remoteMountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
  40. dir := remoteMountCommand.String("dir", "", "a directory in filer")
  41. nonEmpty := remoteMountCommand.Bool("nonempty", false, "allows the mounting over a non-empty directory")
  42. remote := remoteMountCommand.String("remote", "", "a directory in remote storage, ex. <storageName>/<bucket>/path/to/dir")
  43. if err = remoteMountCommand.Parse(args); err != nil {
  44. return nil
  45. }
  46. if *dir == "" {
  47. _, err = listExistingRemoteStorageMounts(commandEnv, writer)
  48. return err
  49. }
  50. // find configuration for remote storage
  51. remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote_storage.ParseLocationName(*remote))
  52. if err != nil {
  53. return fmt.Errorf("find configuration for %s: %v", *remote, err)
  54. }
  55. remoteStorageLocation, err := remote_storage.ParseRemoteLocation(remoteConf.Type, *remote)
  56. if err != nil {
  57. return err
  58. }
  59. // sync metadata from remote
  60. if err = c.syncMetadata(commandEnv, writer, *dir, *nonEmpty, remoteConf, remoteStorageLocation); err != nil {
  61. return fmt.Errorf("pull metadata: %v", err)
  62. }
  63. // store a mount configuration in filer
  64. if err = c.saveMountMapping(commandEnv, writer, *dir, remoteStorageLocation); err != nil {
  65. return fmt.Errorf("save mount mapping: %v", err)
  66. }
  67. return nil
  68. }
  69. func listExistingRemoteStorageMounts(commandEnv *CommandEnv, writer io.Writer) (mappings *remote_pb.RemoteStorageMapping, err error) {
  70. // read current mapping
  71. mappings, err = filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress)
  72. if err != nil {
  73. return mappings, err
  74. }
  75. jsonPrintln(writer, mappings)
  76. return
  77. }
  78. func jsonPrintln(writer io.Writer, message proto.Message) error {
  79. if message == nil {
  80. return nil
  81. }
  82. m := jsonpb.Marshaler{
  83. EmitDefaults: false,
  84. Indent: " ",
  85. }
  86. err := m.Marshal(writer, message)
  87. fmt.Fprintln(writer)
  88. return err
  89. }
  90. func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandEnv, writer io.Writer, remote *remote_pb.RemoteStorageLocation) (conf *remote_pb.RemoteConf, err error) {
  91. return filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote.Name)
  92. }
  93. func (c *commandRemoteMount) syncMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *remote_pb.RemoteConf, remote *remote_pb.RemoteStorageLocation) error {
  94. // find existing directory, and ensure the directory is empty
  95. err := commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  96. parent, name := util.FullPath(dir).DirAndName()
  97. _, lookupErr := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
  98. Directory: parent,
  99. Name: name,
  100. })
  101. if lookupErr != nil {
  102. if !strings.Contains(lookupErr.Error(), filer_pb.ErrNotFound.Error()) {
  103. _, createErr := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
  104. Directory: parent,
  105. Entry: &filer_pb.Entry{
  106. Name: name,
  107. IsDirectory: true,
  108. Attributes: &filer_pb.FuseAttributes{
  109. Mtime: time.Now().Unix(),
  110. Crtime: time.Now().Unix(),
  111. FileMode: uint32(0644 | os.ModeDir),
  112. },
  113. },
  114. })
  115. return createErr
  116. }
  117. }
  118. mountToDirIsEmpty := true
  119. listErr := filer_pb.SeaweedList(client, dir, "", func(entry *filer_pb.Entry, isLast bool) error {
  120. mountToDirIsEmpty = false
  121. return nil
  122. }, "", false, 1)
  123. if listErr != nil {
  124. return fmt.Errorf("list %s: %v", dir, listErr)
  125. }
  126. if !mountToDirIsEmpty {
  127. if !nonEmpty {
  128. return fmt.Errorf("dir %s is not empty", dir)
  129. }
  130. }
  131. return nil
  132. })
  133. if err != nil {
  134. return err
  135. }
  136. // pull metadata from remote
  137. if err = pullMetadata(commandEnv, writer, util.FullPath(dir), remote, util.FullPath(dir), remoteConf); err != nil {
  138. return fmt.Errorf("cache metadata: %v", err)
  139. }
  140. return nil
  141. }
  142. func (c *commandRemoteMount) saveMountMapping(commandEnv *CommandEnv, writer io.Writer, dir string, remoteStorageLocation *remote_pb.RemoteStorageLocation) (err error) {
  143. // read current mapping
  144. var oldContent, newContent []byte
  145. err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  146. oldContent, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE)
  147. return err
  148. })
  149. if err != nil {
  150. if err != filer_pb.ErrNotFound {
  151. return fmt.Errorf("read existing mapping: %v", err)
  152. }
  153. }
  154. // add new mapping
  155. newContent, err = filer.AddRemoteStorageMapping(oldContent, dir, remoteStorageLocation)
  156. if err != nil {
  157. return fmt.Errorf("add mapping %s~%s: %v", dir, remoteStorageLocation, err)
  158. }
  159. // save back
  160. err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  161. return filer.SaveInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE, newContent)
  162. })
  163. if err != nil {
  164. return fmt.Errorf("save mapping: %v", err)
  165. }
  166. return nil
  167. }
  168. // if an entry has synchronized metadata but has not synchronized content
  169. // entry.Attributes.FileSize == entry.RemoteEntry.RemoteSize
  170. // entry.Attributes.Mtime == entry.RemoteEntry.RemoteMtime
  171. // entry.RemoteEntry.LastLocalSyncTsNs == 0
  172. // if an entry has synchronized metadata but has synchronized content before
  173. // entry.Attributes.FileSize == entry.RemoteEntry.RemoteSize
  174. // entry.Attributes.Mtime == entry.RemoteEntry.RemoteMtime
  175. // entry.RemoteEntry.LastLocalSyncTsNs > 0
  176. // if an entry has synchronized metadata but has new updates
  177. // entry.Attributes.Mtime * 1,000,000,000 > entry.RemoteEntry.LastLocalSyncTsNs
  178. func doSaveRemoteEntry(client filer_pb.SeaweedFilerClient, localDir string, existingEntry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
  179. existingEntry.RemoteEntry = remoteEntry
  180. existingEntry.Attributes.FileSize = uint64(remoteEntry.RemoteSize)
  181. existingEntry.Attributes.Mtime = remoteEntry.RemoteMtime
  182. _, updateErr := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
  183. Directory: localDir,
  184. Entry: existingEntry,
  185. })
  186. if updateErr != nil {
  187. return updateErr
  188. }
  189. return nil
  190. }