You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

135 lines
3.9 KiB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. package backend
  2. import (
  3. "io"
  4. "os"
  5. "strings"
  6. "time"
  7. "github.com/chrislusf/seaweedfs/weed/glog"
  8. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  9. "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
  10. "github.com/spf13/viper"
  11. )
  12. type BackendStorageFile interface {
  13. io.ReaderAt
  14. io.WriterAt
  15. Truncate(off int64) error
  16. io.Closer
  17. GetStat() (datSize int64, modTime time.Time, err error)
  18. Name() string
  19. }
  20. type BackendStorage interface {
  21. ToProperties() map[string]string
  22. NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) BackendStorageFile
  23. CopyFile(f *os.File, attributes map[string]string, fn func(progressed int64, percentage float32) error) (key string, size int64, err error)
  24. DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error) (size int64, err error)
  25. DeleteFile(key string) (err error)
  26. }
  27. type StringProperties interface {
  28. GetString(key string) string
  29. }
  30. type StorageType string
  31. type BackendStorageFactory interface {
  32. StorageType() StorageType
  33. BuildStorage(configuration StringProperties, configPrefix string, id string) (BackendStorage, error)
  34. }
  35. var (
  36. BackendStorageFactories = make(map[StorageType]BackendStorageFactory)
  37. BackendStorages = make(map[string]BackendStorage)
  38. )
  39. // used by master to load remote storage configurations
  40. func LoadConfiguration(config *viper.Viper) {
  41. StorageBackendPrefix := "storage.backend"
  42. for backendTypeName := range config.GetStringMap(StorageBackendPrefix) {
  43. backendStorageFactory, found := BackendStorageFactories[StorageType(backendTypeName)]
  44. if !found {
  45. glog.Fatalf("backend storage type %s not found", backendTypeName)
  46. }
  47. for backendStorageId := range config.GetStringMap(StorageBackendPrefix + "." + backendTypeName) {
  48. if !config.GetBool(StorageBackendPrefix + "." + backendTypeName + "." + backendStorageId + ".enabled") {
  49. continue
  50. }
  51. backendStorage, buildErr := backendStorageFactory.BuildStorage(config,
  52. StorageBackendPrefix+"."+backendTypeName+"."+backendStorageId+".", backendStorageId)
  53. if buildErr != nil {
  54. glog.Fatalf("fail to create backend storage %s.%s", backendTypeName, backendStorageId)
  55. }
  56. BackendStorages[backendTypeName+"."+backendStorageId] = backendStorage
  57. if backendStorageId == "default" {
  58. BackendStorages[backendTypeName] = backendStorage
  59. }
  60. }
  61. }
  62. }
  63. // used by volume server to receive remote storage configurations from master
  64. func LoadFromPbStorageBackends(storageBackends []*master_pb.StorageBackend) {
  65. for _, storageBackend := range storageBackends {
  66. backendStorageFactory, found := BackendStorageFactories[StorageType(storageBackend.Type)]
  67. if !found {
  68. glog.Warningf("storage type %s not found", storageBackend.Type)
  69. continue
  70. }
  71. backendStorage, buildErr := backendStorageFactory.BuildStorage(newProperties(storageBackend.Properties), "", storageBackend.Id)
  72. if buildErr != nil {
  73. glog.Fatalf("fail to create backend storage %s.%s", storageBackend.Type, storageBackend.Id)
  74. }
  75. BackendStorages[storageBackend.Type+"."+storageBackend.Id] = backendStorage
  76. if storageBackend.Id == "default" {
  77. BackendStorages[storageBackend.Type] = backendStorage
  78. }
  79. }
  80. }
  81. type Properties struct {
  82. m map[string]string
  83. }
  84. func newProperties(m map[string]string) *Properties {
  85. return &Properties{m: m}
  86. }
  87. func (p *Properties) GetString(key string) string {
  88. if v, found := p.m[key]; found {
  89. return v
  90. }
  91. return ""
  92. }
  93. func ToPbStorageBackends() (backends []*master_pb.StorageBackend) {
  94. for sName, s := range BackendStorages {
  95. sType, sId := BackendNameToTypeId(sName)
  96. if sType == "" {
  97. continue
  98. }
  99. backends = append(backends, &master_pb.StorageBackend{
  100. Type: sType,
  101. Id: sId,
  102. Properties: s.ToProperties(),
  103. })
  104. }
  105. return
  106. }
  107. func BackendNameToTypeId(backendName string) (backendType, backendId string) {
  108. parts := strings.Split(backendName, ".")
  109. if len(parts) == 1 {
  110. return backendName, "default"
  111. }
  112. if len(parts) != 2 {
  113. return
  114. }
  115. backendType, backendId = parts[0], parts[1]
  116. return
  117. }