You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

204 lines
5.7 KiB

4 years ago
  1. package redis2
  2. import (
  3. "context"
  4. "fmt"
  5. "time"
  6. "github.com/go-redis/redis/v8"
  7. "github.com/chrislusf/seaweedfs/weed/filer"
  8. "github.com/chrislusf/seaweedfs/weed/glog"
  9. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  10. "github.com/chrislusf/seaweedfs/weed/util"
  11. )
  12. const (
  13. DIR_LIST_MARKER = "\x00"
  14. )
  15. type UniversalRedis2Store struct {
  16. Client redis.UniversalClient
  17. superLargeDirectoryHash map[string]bool
  18. }
  19. func (store *UniversalRedis2Store) isSuperLargeDirectory(dir string) (isSuperLargeDirectory bool) {
  20. _, isSuperLargeDirectory = store.superLargeDirectoryHash[dir]
  21. return
  22. }
  23. func (store *UniversalRedis2Store) loadSuperLargeDirectories(superLargeDirectories []string) {
  24. // set directory hash
  25. store.superLargeDirectoryHash = make(map[string]bool)
  26. for _, dir := range superLargeDirectories {
  27. store.superLargeDirectoryHash[dir] = true
  28. }
  29. }
  30. func (store *UniversalRedis2Store) BeginTransaction(ctx context.Context) (context.Context, error) {
  31. return ctx, nil
  32. }
  33. func (store *UniversalRedis2Store) CommitTransaction(ctx context.Context) error {
  34. return nil
  35. }
  36. func (store *UniversalRedis2Store) RollbackTransaction(ctx context.Context) error {
  37. return nil
  38. }
  39. func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
  40. value, err := entry.EncodeAttributesAndChunks()
  41. if err != nil {
  42. return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
  43. }
  44. if len(entry.Chunks) > 50 {
  45. value = util.MaybeGzipData(value)
  46. }
  47. if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
  48. return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
  49. }
  50. dir, name := entry.FullPath.DirAndName()
  51. if store.isSuperLargeDirectory(dir) {
  52. return nil
  53. }
  54. if name != "" {
  55. if err = store.Client.ZAddNX(ctx, genDirectoryListKey(dir), &redis.Z{Score: 0, Member: name}).Err(); err != nil {
  56. return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err)
  57. }
  58. }
  59. return nil
  60. }
  61. func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
  62. return store.InsertEntry(ctx, entry)
  63. }
  64. func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
  65. data, err := store.Client.Get(ctx, string(fullpath)).Result()
  66. if err == redis.Nil {
  67. return nil, filer_pb.ErrNotFound
  68. }
  69. if err != nil {
  70. return nil, fmt.Errorf("get %s : %v", fullpath, err)
  71. }
  72. entry = &filer.Entry{
  73. FullPath: fullpath,
  74. }
  75. err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data)))
  76. if err != nil {
  77. return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
  78. }
  79. return entry, nil
  80. }
  81. func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
  82. _, err = store.Client.Del(ctx, genDirectoryListKey(string(fullpath))).Result()
  83. if err != nil {
  84. return fmt.Errorf("delete dir list %s : %v", fullpath, err)
  85. }
  86. _, err = store.Client.Del(ctx, string(fullpath)).Result()
  87. if err != nil {
  88. return fmt.Errorf("delete %s : %v", fullpath, err)
  89. }
  90. dir, name := fullpath.DirAndName()
  91. if store.isSuperLargeDirectory(dir) {
  92. return nil
  93. }
  94. if name != "" {
  95. _, err = store.Client.ZRem(ctx, genDirectoryListKey(dir), name).Result()
  96. if err != nil {
  97. return fmt.Errorf("DeleteEntry %s in parent dir: %v", fullpath, err)
  98. }
  99. }
  100. return nil
  101. }
  102. func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath, limit int64) (err error) {
  103. if store.isSuperLargeDirectory(string(fullpath)) {
  104. return nil
  105. }
  106. members, err := store.Client.ZRange(ctx, genDirectoryListKey(string(fullpath)), 0, -1).Result()
  107. if err != nil {
  108. return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err)
  109. }
  110. for _, fileName := range members {
  111. path := util.NewFullPath(string(fullpath), fileName)
  112. _, err = store.Client.Del(ctx, string(path)).Result()
  113. if err != nil {
  114. return fmt.Errorf("DeleteFolderChildren %s in parent dir: %v", fullpath, err)
  115. }
  116. }
  117. return nil
  118. }
  119. func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
  120. return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
  121. }
  122. func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
  123. dirListKey := genDirectoryListKey(string(dirPath))
  124. start := int64(0)
  125. if startFileName != "" {
  126. start, _ = store.Client.ZRank(ctx, dirListKey, startFileName).Result()
  127. if !includeStartFile {
  128. start++
  129. }
  130. }
  131. members, err := store.Client.ZRange(ctx, dirListKey, start, start+int64(limit)-1).Result()
  132. if err != nil {
  133. return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
  134. }
  135. // fetch entry meta
  136. for _, fileName := range members {
  137. path := util.NewFullPath(string(dirPath), fileName)
  138. entry, err := store.FindEntry(ctx, path)
  139. lastFileName = fileName
  140. if err != nil {
  141. glog.V(0).Infof("list %s : %v", path, err)
  142. if err == filer_pb.ErrNotFound {
  143. continue
  144. }
  145. } else {
  146. if entry.TtlSec > 0 {
  147. if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
  148. store.Client.Del(ctx, string(path)).Result()
  149. store.Client.ZRem(ctx, dirListKey, fileName).Result()
  150. continue
  151. }
  152. }
  153. if !eachEntryFunc(entry) {
  154. break
  155. }
  156. }
  157. }
  158. return lastFileName, err
  159. }
  160. func genDirectoryListKey(dir string) (dirList string) {
  161. return dir + DIR_LIST_MARKER
  162. }
  163. func (store *UniversalRedis2Store) Shutdown() {
  164. store.Client.Close()
  165. }