You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

214 lines
5.8 KiB

5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
  1. package mongodb
  2. import (
  3. "context"
  4. "fmt"
  5. "github.com/chrislusf/seaweedfs/weed/filer2"
  6. "github.com/chrislusf/seaweedfs/weed/glog"
  7. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  8. "github.com/chrislusf/seaweedfs/weed/util"
  9. "go.mongodb.org/mongo-driver/bson"
  10. "go.mongodb.org/mongo-driver/mongo"
  11. "go.mongodb.org/mongo-driver/mongo/options"
  12. "go.mongodb.org/mongo-driver/x/bsonx"
  13. "time"
  14. )
  15. func init() {
  16. filer2.Stores = append(filer2.Stores, &MongodbStore{})
  17. }
  18. type MongodbStore struct {
  19. connect *mongo.Client
  20. database string
  21. collectionName string
  22. }
  23. type Model struct {
  24. Directory string `bson:"directory"`
  25. Name string `bson:"name"`
  26. Meta []byte `bson:"meta"`
  27. }
  28. func (store *MongodbStore) GetName() string {
  29. return "mongodb"
  30. }
  31. func (store *MongodbStore) Initialize(configuration util.Configuration, prefix string) (err error) {
  32. store.database = configuration.GetString(prefix + "database")
  33. store.collectionName = "filemeta"
  34. poolSize := configuration.GetInt(prefix + "option_pool_size")
  35. return store.connection(configuration.GetString(prefix+"uri"), uint64(poolSize))
  36. }
  37. func (store *MongodbStore) connection(uri string, poolSize uint64) (err error) {
  38. ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
  39. opts := options.Client().ApplyURI(uri)
  40. if poolSize > 0 {
  41. opts.SetMaxPoolSize(poolSize)
  42. }
  43. client, err := mongo.Connect(ctx, opts)
  44. if err != nil {
  45. return err
  46. }
  47. c := client.Database(store.database).Collection(store.collectionName)
  48. err = store.indexUnique(c)
  49. store.connect = client
  50. return err
  51. }
  52. func (store *MongodbStore) createIndex(c *mongo.Collection, index mongo.IndexModel, opts *options.CreateIndexesOptions) error {
  53. _, err := c.Indexes().CreateOne(context.Background(), index, opts)
  54. return err
  55. }
  56. func (store *MongodbStore) indexUnique(c *mongo.Collection) error {
  57. opts := options.CreateIndexes().SetMaxTime(10 * time.Second)
  58. unique := new(bool)
  59. *unique = true
  60. index := mongo.IndexModel{
  61. Keys: bsonx.Doc{{Key: "directory", Value: bsonx.Int32(1)}, {Key: "name", Value: bsonx.Int32(1)}},
  62. Options: &options.IndexOptions{
  63. Unique: unique,
  64. },
  65. }
  66. return store.createIndex(c, index, opts)
  67. }
  68. func (store *MongodbStore) BeginTransaction(ctx context.Context) (context.Context, error) {
  69. return ctx, nil
  70. }
  71. func (store *MongodbStore) CommitTransaction(ctx context.Context) error {
  72. return nil
  73. }
  74. func (store *MongodbStore) RollbackTransaction(ctx context.Context) error {
  75. return nil
  76. }
  77. func (store *MongodbStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
  78. dir, name := entry.FullPath.DirAndName()
  79. meta, err := entry.EncodeAttributesAndChunks()
  80. if err != nil {
  81. return fmt.Errorf("encode %s: %s", entry.FullPath, err)
  82. }
  83. c := store.connect.Database(store.database).Collection(store.collectionName)
  84. _, err = c.InsertOne(ctx, Model{
  85. Directory: dir,
  86. Name: name,
  87. Meta: meta,
  88. })
  89. return nil
  90. }
  91. func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
  92. return store.InsertEntry(ctx, entry)
  93. }
  94. func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer2.Entry, err error) {
  95. dir, name := fullpath.DirAndName()
  96. var data Model
  97. var where = bson.M{"directory": dir, "name": name}
  98. err = store.connect.Database(store.database).Collection(store.collectionName).FindOne(ctx, where).Decode(&data)
  99. if err != mongo.ErrNoDocuments && err != nil {
  100. return nil, filer_pb.ErrNotFound
  101. }
  102. if len(data.Meta) == 0 {
  103. return nil, filer_pb.ErrNotFound
  104. }
  105. entry = &filer2.Entry{
  106. FullPath: fullpath,
  107. }
  108. err = entry.DecodeAttributesAndChunks(data.Meta)
  109. if err != nil {
  110. return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
  111. }
  112. return entry, nil
  113. }
  114. func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
  115. dir, name := fullpath.DirAndName()
  116. where := bson.M{"directory": dir, "name": name}
  117. _, err := store.connect.Database(store.database).Collection(store.collectionName).DeleteOne(ctx, where)
  118. if err != nil {
  119. return fmt.Errorf("delete %s : %v", fullpath, err)
  120. }
  121. return nil
  122. }
  123. func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
  124. where := bson.M{"directory": fullpath}
  125. _, err := store.connect.Database(store.database).Collection(store.collectionName).DeleteMany(ctx, where)
  126. if err != nil {
  127. return fmt.Errorf("delete %s : %v", fullpath, err)
  128. }
  129. return nil
  130. }
  131. func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) {
  132. return nil, filer2.ErrUnsupportedListDirectoryPrefixed
  133. }
  134. func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
  135. var where = bson.M{"directory": string(fullpath), "name": bson.M{"$gt": startFileName}}
  136. if inclusive {
  137. where["name"] = bson.M{
  138. "$gte": startFileName,
  139. }
  140. }
  141. optLimit := int64(limit)
  142. opts := &options.FindOptions{Limit: &optLimit, Sort: bson.M{"name": 1}}
  143. cur, err := store.connect.Database(store.database).Collection(store.collectionName).Find(ctx, where, opts)
  144. for cur.Next(ctx) {
  145. var data Model
  146. err := cur.Decode(&data)
  147. if err != nil && err != mongo.ErrNoDocuments {
  148. return nil, err
  149. }
  150. entry := &filer2.Entry{
  151. FullPath: util.NewFullPath(string(fullpath), data.Name),
  152. }
  153. if decodeErr := entry.DecodeAttributesAndChunks(data.Meta); decodeErr != nil {
  154. err = decodeErr
  155. glog.V(0).Infof("list %s : %v", entry.FullPath, err)
  156. break
  157. }
  158. entries = append(entries, entry)
  159. }
  160. if err := cur.Close(ctx); err != nil {
  161. glog.V(0).Infof("list iterator close: %v", err)
  162. }
  163. return entries, err
  164. }
  165. func (store *MongodbStore) Shutdown() {
  166. ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
  167. store.connect.Disconnect(ctx)
  168. }