You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

121 lines
3.6 KiB

4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
  1. package remote_storage
  2. import (
  3. "fmt"
  4. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  5. "github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
  6. "io"
  7. "strings"
  8. "sync"
  9. )
  10. func ParseLocationName(remote string) (locationName string) {
  11. if strings.HasSuffix(string(remote), "/") {
  12. remote = remote[:len(remote)-1]
  13. }
  14. parts := strings.SplitN(string(remote), "/", 2)
  15. if len(parts) >= 1 {
  16. return parts[0]
  17. }
  18. return
  19. }
  20. func parseBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
  21. loc = &remote_pb.RemoteStorageLocation{}
  22. if strings.HasSuffix(string(remote), "/") {
  23. remote = remote[:len(remote)-1]
  24. }
  25. parts := strings.SplitN(string(remote), "/", 3)
  26. if len(parts) >= 1 {
  27. loc.Name = parts[0]
  28. }
  29. if len(parts) >= 2 {
  30. loc.Bucket = parts[1]
  31. }
  32. loc.Path = string(remote[len(loc.Name)+1+len(loc.Bucket):])
  33. if loc.Path == "" {
  34. loc.Path = "/"
  35. }
  36. return
  37. }
  38. func parseNoBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
  39. loc = &remote_pb.RemoteStorageLocation{}
  40. if strings.HasSuffix(string(remote), "/") {
  41. remote = remote[:len(remote)-1]
  42. }
  43. parts := strings.SplitN(string(remote), "/", 2)
  44. if len(parts) >= 1 {
  45. loc.Name = parts[0]
  46. }
  47. loc.Path = string(remote[len(loc.Name):])
  48. if loc.Path == "" {
  49. loc.Path = "/"
  50. }
  51. return
  52. }
  53. func FormatLocation(loc *remote_pb.RemoteStorageLocation) string {
  54. return fmt.Sprintf("%s/%s%s", loc.Name, loc.Bucket, loc.Path)
  55. }
  56. type VisitFunc func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error
  57. type RemoteStorageClient interface {
  58. Traverse(loc *remote_pb.RemoteStorageLocation, visitFn VisitFunc) error
  59. ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error)
  60. WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error)
  61. WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error)
  62. UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error)
  63. DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error)
  64. }
  65. type RemoteStorageClientMaker interface {
  66. Make(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error)
  67. HasBucket() bool
  68. }
  69. var (
  70. RemoteStorageClientMakers = make(map[string]RemoteStorageClientMaker)
  71. remoteStorageClients = make(map[string]RemoteStorageClient)
  72. remoteStorageClientsLock sync.Mutex
  73. )
  74. func ParseRemoteLocation(remoteConfType string, remote string) (remoteStorageLocation *remote_pb.RemoteStorageLocation, err error) {
  75. maker, found := RemoteStorageClientMakers[remoteConfType]
  76. if !found {
  77. return nil, fmt.Errorf("remote storage type %s not found", remoteConfType)
  78. }
  79. if !maker.HasBucket() {
  80. return parseNoBucketLocation(remote), nil
  81. }
  82. return parseBucketLocation(remote), nil
  83. }
  84. func makeRemoteStorageClient(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error) {
  85. maker, found := RemoteStorageClientMakers[remoteConf.Type]
  86. if !found {
  87. return nil, fmt.Errorf("remote storage type %s not found", remoteConf.Type)
  88. }
  89. return maker.Make(remoteConf)
  90. }
  91. func GetRemoteStorage(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error) {
  92. remoteStorageClientsLock.Lock()
  93. defer remoteStorageClientsLock.Unlock()
  94. existingRemoteStorageClient, found := remoteStorageClients[remoteConf.Name]
  95. if found {
  96. return existingRemoteStorageClient, nil
  97. }
  98. newRemoteStorageClient, err := makeRemoteStorageClient(remoteConf)
  99. if err != nil {
  100. return nil, fmt.Errorf("make remote storage client %s: %v", remoteConf.Name, err)
  101. }
  102. remoteStorageClients[remoteConf.Name] = newRemoteStorageClient
  103. return newRemoteStorageClient, nil
  104. }