From f5d4952d7306ba013bb9c054b221d795a3e110d6 Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 12 Oct 2022 23:50:09 -0700 Subject: [PATCH] filer: redis store reduce from 2 redis operations to 1 for updates. --- weed/filer/redis/universal_redis_store.go | 27 ++++++++++++------- weed/filer/redis2/universal_redis_store.go | 31 +++++++++++++--------- weed/filer/redis3/universal_redis_store.go | 29 ++++++++++++-------- 3 files changed, 54 insertions(+), 33 deletions(-) diff --git a/weed/filer/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go index f5dc513c4..8e1fa326b 100644 --- a/weed/filer/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -35,6 +35,22 @@ func (store *UniversalRedisStore) RollbackTransaction(ctx context.Context) error func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + if err = store.doInsertEntry(ctx, entry); err != nil { + return err + } + + dir, name := entry.FullPath.DirAndName() + if name != "" { + _, err = store.Client.SAdd(ctx, genDirectoryListKey(dir), name).Result() + if err != nil { + return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) + } + } + + return nil +} + +func (store *UniversalRedisStore) doInsertEntry(ctx context.Context, entry *filer.Entry) error { value, err := entry.EncodeAttributesAndChunks() if err != nil { return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) @@ -49,21 +65,12 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer. if err != nil { return fmt.Errorf("persisting %s : %v", entry.FullPath, err) } - - dir, name := entry.FullPath.DirAndName() - if name != "" { - _, err = store.Client.SAdd(ctx, genDirectoryListKey(dir), name).Result() - if err != nil { - return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) - } - } - return nil } func (store *UniversalRedisStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { - return store.InsertEntry(ctx, entry) + return store.doInsertEntry(ctx, entry) } func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { diff --git a/weed/filer/redis2/universal_redis_store.go b/weed/filer/redis2/universal_redis_store.go index 8b23472b9..0c79c5255 100644 --- a/weed/filer/redis2/universal_redis_store.go +++ b/weed/filer/redis2/universal_redis_store.go @@ -47,17 +47,8 @@ func (store *UniversalRedis2Store) RollbackTransaction(ctx context.Context) erro func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { - value, err := entry.EncodeAttributesAndChunks() - if err != nil { - return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) - } - - if len(entry.Chunks) > filer.CountEntryChunksForGzip { - value = util.MaybeGzipData(value) - } - - if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil { - return fmt.Errorf("persisting %s : %v", entry.FullPath, err) + if err = store.doInsertEntry(ctx, entry); err != nil { + return err } dir, name := entry.FullPath.DirAndName() @@ -74,9 +65,25 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer return nil } +func (store *UniversalRedis2Store) doInsertEntry(ctx context.Context, entry *filer.Entry) error { + value, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) + } + + if len(entry.Chunks) > filer.CountEntryChunksForGzip { + value = util.MaybeGzipData(value) + } + + if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil { + return fmt.Errorf("persisting %s : %v", entry.FullPath, err) + } + return nil +} + func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { - return store.InsertEntry(ctx, entry) + return store.doInsertEntry(ctx, entry) } func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { diff --git a/weed/filer/redis3/universal_redis_store.go b/weed/filer/redis3/universal_redis_store.go index b076f78e7..88d4ed1e3 100644 --- a/weed/filer/redis3/universal_redis_store.go +++ b/weed/filer/redis3/universal_redis_store.go @@ -3,11 +3,11 @@ package redis3 import ( "context" "fmt" - "github.com/go-redsync/redsync/v4" "time" "github.com/go-redis/redis/v8" + redsync "github.com/go-redsync/redsync/v4" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -35,6 +35,22 @@ func (store *UniversalRedis3Store) RollbackTransaction(ctx context.Context) erro func (store *UniversalRedis3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + if err = store.doInsertEntry(ctx, entry); err != nil { + return err + } + + dir, name := entry.FullPath.DirAndName() + + if name != "" { + if err = insertChild(ctx, store, genDirectoryListKey(dir), name); err != nil { + return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) + } + } + + return nil +} + +func (store *UniversalRedis3Store) doInsertEntry(ctx context.Context, entry *filer.Entry) error { value, err := entry.EncodeAttributesAndChunks() if err != nil { return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) @@ -47,21 +63,12 @@ func (store *UniversalRedis3Store) InsertEntry(ctx context.Context, entry *filer if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil { return fmt.Errorf("persisting %s : %v", entry.FullPath, err) } - - dir, name := entry.FullPath.DirAndName() - - if name != "" { - if err = insertChild(ctx, store, genDirectoryListKey(dir), name); err != nil { - return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) - } - } - return nil } func (store *UniversalRedis3Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { - return store.InsertEntry(ctx, entry) + return store.doInsertEntry(ctx, entry) } func (store *UniversalRedis3Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {