From c6caa6e4cc65f3a72b454b8440e75cbbf2b2ea9c Mon Sep 17 00:00:00 2001 From: root Date: Sun, 20 Nov 2022 20:28:47 -0800 Subject: [PATCH 01/11] wip --- go.mod | 1 + go.sum | 7 ++ .../shell/command_fs_meta_snapshots_create.go | 86 +++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 weed/shell/command_fs_meta_snapshots_create.go diff --git a/go.mod b/go.mod index 1fe1c4dcb..fe85fa815 100644 --- a/go.mod +++ b/go.mod @@ -148,6 +148,7 @@ require ( github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17 github.com/hashicorp/raft v1.3.11 github.com/hashicorp/raft-boltdb/v2 v2.2.2 + github.com/otiai10/copy v1.9.0 github.com/rabbitmq/amqp091-go v1.5.0 github.com/tikv/client-go/v2 v2.0.2 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 diff --git a/go.sum b/go.sum index c863e62f2..902abfa61 100644 --- a/go.sum +++ b/go.sum @@ -1360,6 +1360,13 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4= +github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.4.0 h1:umwcf7gbpEwf7WFzqmWwSv0CzbeMsae2u9ZvpP8j2q4= +github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= diff --git a/weed/shell/command_fs_meta_snapshots_create.go b/weed/shell/command_fs_meta_snapshots_create.go new file mode 100644 index 000000000..1a58dbf81 --- /dev/null +++ b/weed/shell/command_fs_meta_snapshots_create.go @@ -0,0 +1,86 @@ +package shell + +import ( + "flag" + "fmt" + cp "github.com/otiai10/copy" + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/syndtr/goleveldb/leveldb" + "io" +) + +func init() { + Commands = append(Commands, &commandFsMetaSnapshotsCreate{}) +} + +type commandFsMetaSnapshotsCreate struct { +} + +func (c *commandFsMetaSnapshotsCreate) Name() string { + return "fs.meta.snapshots.create" +} + +func (c *commandFsMetaSnapshotsCreate) Help() string { + return `create snapshots of meta data from given time range. + + fs.meta.snapshots.create -s yyyy-mm-dd -e yyyy-mm-dd -o + + //These snapshot maybe later used to backup the system to certain timestamp. + +` +} + +func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + db, err := leveldb.OpenFile("/root/go/repos/seaweedfs/weed/snapshots.db", nil) + if err != nil { + return err + } + defer db.Close() + fsMetaSnapshotsCreateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + if err = fsMetaSnapshotsCreateCommand.Parse(args); err != nil { + return err + } + + path := filer.SystemLogDir + count := 0 + err = filer_pb.TraverseBfs(commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) { + if !entry.IsDirectory { + fmt.Println("%+v", entry) + err = db.Put([]byte(entry.GetName()), entry.Content, nil) + count++ + if count%2 == 0 { + snapshotName := fmt.Sprintf("/root/go/repos/seaweedfs/weed/snapshots-%d.db", count/2) + e := cp.Copy("/root/go/repos/seaweedfs/weed/snapshots.db", snapshotName) + println(snapshotName) + if e != nil { + println("failed to generate " + snapshotName + e.Error()) + } + } + } + }) + verifying, err := leveldb.OpenFile("/root/go/repos/seaweedfs/weed/snapshots-1.db", nil) + if err != nil { + return err + } + iter := verifying.NewIterator(nil, nil) + for iter.Next() { + key := iter.Key() + fmt.Println("1 ---->") + fmt.Println(string(key)) + } + verifying.Close() + verifying, err = leveldb.OpenFile("/root/go/repos/seaweedfs/weed/snapshots-4.db", nil) + if err != nil { + return err + } + iter = verifying.NewIterator(nil, nil) + for iter.Next() { + key := iter.Key() + fmt.Println("4 ---->") + fmt.Println(string(key)) + } + verifying.Close() + return err +} From 3dd482d3c24753ca7eb94c7344bbedf2fe727c6c Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Dec 2022 23:03:00 -0800 Subject: [PATCH 02/11] ADHOC: take snapshots of metadata --- weed/filer/leveldb/leveldb_store.go | 5 + .../shell/command_fs_meta_snapshots_create.go | 223 +++++++++++++++--- 2 files changed, 190 insertions(+), 38 deletions(-) diff --git a/weed/filer/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go index 747d1104d..08dc33f02 100644 --- a/weed/filer/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -43,6 +43,11 @@ func (store *LevelDBStore) Initialize(configuration weed_util.Configuration, pre return store.initialize(dir) } +// For adhoc usage +func (store *LevelDBStore) CustomInitialize(dir string) (err error) { + return store.initialize(dir) +} + func (store *LevelDBStore) initialize(dir string) (err error) { glog.Infof("filer store dir: %s", dir) os.MkdirAll(dir, 0755) diff --git a/weed/shell/command_fs_meta_snapshots_create.go b/weed/shell/command_fs_meta_snapshots_create.go index 1a58dbf81..adc07e804 100644 --- a/weed/shell/command_fs_meta_snapshots_create.go +++ b/weed/shell/command_fs_meta_snapshots_create.go @@ -1,16 +1,25 @@ package shell import ( + "context" "flag" "fmt" - cp "github.com/otiai10/copy" "github.com/seaweedfs/seaweedfs/weed/filer" + filer_leveldb "github.com/seaweedfs/seaweedfs/weed/filer/leveldb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" - "github.com/syndtr/goleveldb/leveldb" + "github.com/seaweedfs/seaweedfs/weed/util/mem" + "google.golang.org/protobuf/proto" "io" + "os" + "path/filepath" + "time" ) +const LevelDbPath = "/tmp/snapshots.db" +const SnapshotsParentPath = "/seaweedfs_meta_snapshots/" +const DateFormat = "2006-01-02" + func init() { Commands = append(Commands, &commandFsMetaSnapshotsCreate{}) } @@ -25,62 +34,200 @@ func (c *commandFsMetaSnapshotsCreate) Name() string { func (c *commandFsMetaSnapshotsCreate) Help() string { return `create snapshots of meta data from given time range. - fs.meta.snapshots.create -s yyyy-mm-dd -e yyyy-mm-dd -o + fs.meta.snapshots.create -s=2022-08-09 -e=2022-10-12 -f 7 create snapshot starting from 2022-08-09 ending at 2022-10-12 every seven days. //These snapshot maybe later used to backup the system to certain timestamp. - ` } -func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - db, err := leveldb.OpenFile("/root/go/repos/seaweedfs/weed/snapshots.db", nil) +func processMetaDataEvents(store *filer_leveldb.LevelDBStore, data []byte, snapshotCnt int, snapshotCheckPoints []time.Time, homeDir string) (err error) { + var event filer_pb.SubscribeMetadataResponse + err = proto.Unmarshal(data, &event) if err != nil { return err } - defer db.Close() - fsMetaSnapshotsCreateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) - if err = fsMetaSnapshotsCreateCommand.Parse(args); err != nil { + fmt.Printf("%+v\n", event) + eventTime := event.TsNs + println(time.Unix(0, eventTime).Format(DateFormat)) + for snapshotCnt < len(snapshotCheckPoints) && time.Unix(0, eventTime).After(snapshotCheckPoints[snapshotCnt]) { + snapshotPath := homeDir + SnapshotsParentPath + snapshotCheckPoints[snapshotCnt].Format(DateFormat) + err = CreateIfNotExists(snapshotPath, 0755) + if err != nil { + return err + } + println("generating snapshots of metadata at: " + snapshotPath) + err = generateSnapshots(homeDir+LevelDbPath, snapshotPath) + if err != nil { + return err + } + snapshotCnt++ + } + if snapshotCnt == len(snapshotCheckPoints) { + return nil + } + ctx := context.Background() + if filer_pb.IsEmpty(&event) { + return nil + } else if filer_pb.IsCreate(&event) { + println("+", util.FullPath(event.EventNotification.NewParentPath).Child(event.EventNotification.NewEntry.Name)) + entry := filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry) + return store.InsertEntry(ctx, entry) + } else if filer_pb.IsDelete(&event) { + println("-", util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)) + return store.DeleteEntry(ctx, util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)) + } else if filer_pb.IsUpdate(&event) { + println("~", util.FullPath(event.EventNotification.NewParentPath).Child(event.EventNotification.NewEntry.Name)) + entry := filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry) + return store.UpdateEntry(ctx, entry) + } else { + // renaming + println("-", util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)) + if err := store.DeleteEntry(ctx, util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)); err != nil { + return err + } + println("+", util.FullPath(event.EventNotification.NewParentPath).Child(event.EventNotification.NewEntry.Name)) + return store.InsertEntry(ctx, filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry)) + } + return nil +} + +func generateSnapshots(scrDir, dest string) error { + entries, err := os.ReadDir(scrDir) + if err != nil { return err } + for _, entry := range entries { + sourcePath := filepath.Join(scrDir, entry.Name()) + destPath := filepath.Join(dest, entry.Name()) - path := filer.SystemLogDir - count := 0 - err = filer_pb.TraverseBfs(commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) { - if !entry.IsDirectory { - fmt.Println("%+v", entry) - err = db.Put([]byte(entry.GetName()), entry.Content, nil) - count++ - if count%2 == 0 { - snapshotName := fmt.Sprintf("/root/go/repos/seaweedfs/weed/snapshots-%d.db", count/2) - e := cp.Copy("/root/go/repos/seaweedfs/weed/snapshots.db", snapshotName) - println(snapshotName) - if e != nil { - println("failed to generate " + snapshotName + e.Error()) - } + fileInfo, err := os.Stat(sourcePath) + if err != nil { + return err + } + + switch fileInfo.Mode() & os.ModeType { + case os.ModeDir: + if err := CreateIfNotExists(destPath, 0755); err != nil { + return err + } + if err := generateSnapshots(sourcePath, destPath); err != nil { + return err + } + default: + if err := Copy(sourcePath, destPath); err != nil { + return err } } - }) - verifying, err := leveldb.OpenFile("/root/go/repos/seaweedfs/weed/snapshots-1.db", nil) + } + return nil +} + +func Copy(srcFile, dstFile string) error { + out, err := os.Create(dstFile) + if err != nil { + return err + } + + defer out.Close() + + in, err := os.Open(srcFile) + defer in.Close() + if err != nil { + return err + } + + _, err = io.Copy(out, in) + if err != nil { + return err + } + + return nil +} + +func Exists(filePath string) bool { + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return false + } + + return true +} + +func CreateIfNotExists(dir string, perm os.FileMode) error { + if Exists(dir) { + return nil + } + + if err := os.MkdirAll(dir, perm); err != nil { + return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error()) + } + + return nil +} + +func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + fsMetaSnapshotsCreateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + start := fsMetaSnapshotsCreateCommand.String("s", "", "start date of metadata compaction in yyyy-mm-dd format") + end := fsMetaSnapshotsCreateCommand.String("e", "", "end date of metadata compaction in yyyy-mm-dd format") + frequency := fsMetaSnapshotsCreateCommand.Int("f", 7, "the frequency of generating the metadata snapshots in days") + if err = fsMetaSnapshotsCreateCommand.Parse(args); err != nil { + return err + } + startDate, err := time.Parse(DateFormat, *start) + if err != nil { + return err + } + endDate, err := time.Parse(DateFormat, *end) if err != nil { return err } - iter := verifying.NewIterator(nil, nil) - for iter.Next() { - key := iter.Key() - fmt.Println("1 ---->") - fmt.Println(string(key)) + var snapshotCheckPoints []time.Time + for startDate.Before(endDate) || startDate.Equal(endDate) { + snapshotCheckPoints = append(snapshotCheckPoints, startDate) + startDate = startDate.AddDate(0, 0, *frequency) + } + snapshotCnt := 0 + homeDirname, err := os.UserHomeDir() + if err != nil { + return err } - verifying.Close() - verifying, err = leveldb.OpenFile("/root/go/repos/seaweedfs/weed/snapshots-4.db", nil) + levelDbPath := homeDirname + LevelDbPath + err = os.RemoveAll(levelDbPath) if err != nil { return err } - iter = verifying.NewIterator(nil, nil) - for iter.Next() { - key := iter.Key() - fmt.Println("4 ---->") - fmt.Println(string(key)) + store := &filer_leveldb.LevelDBStore{} + store.CustomInitialize(levelDbPath) + path := filer.SystemLogDir + var processEntry func(entry *filer_pb.Entry, isLast bool) error + processEntry = func(entry *filer_pb.Entry, isLast bool) error { + if entry.IsDirectory { + return filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(path+"/"+entry.Name), "", processEntry) + } + totalSize := filer.FileSize(entry) + buf := mem.Allocate(int(totalSize)) + if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + return filer.ReadAll(buf, commandEnv.MasterClient, entry.GetChunks()) + }); err != nil && err != filer_pb.ErrNotFound { + return err + } + idx := uint32(0) + for idx < uint32(totalSize) { + logEntrySize := util.BytesToUint32(buf[idx : idx+4]) + var logEntry filer_pb.LogEntry + err = proto.Unmarshal(buf[idx+4:idx+4+logEntrySize], &logEntry) + if err != nil { + return err + } + idx = idx + 4 + logEntrySize + err = processMetaDataEvents(store, logEntry.Data, snapshotCnt, snapshotCheckPoints, homeDirname) + if err != nil { + return err + } + } + + return nil } - verifying.Close() + + err = filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(path), "", processEntry) return err } From faa3a6298c1a400532b915c610269be3d10dba93 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Dec 2022 23:06:49 -0800 Subject: [PATCH 03/11] nit --- go.mod | 41 +++++++++++++------------ go.sum | 94 ++++++++++++++++++++++++++-------------------------------- 2 files changed, 62 insertions(+), 73 deletions(-) diff --git a/go.mod b/go.mod index fd4f2477d..1fe1c4dcb 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ go 1.19 require ( cloud.google.com/go v0.105.0 // indirect - cloud.google.com/go/pubsub v1.27.1 - cloud.google.com/go/storage v1.28.1 + cloud.google.com/go/pubsub v1.26.0 + cloud.google.com/go/storage v1.28.0 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Shopify/sarama v1.37.2 - github.com/aws/aws-sdk-go v1.44.146 + github.com/aws/aws-sdk-go v1.44.136 github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -32,7 +32,7 @@ require ( github.com/go-errors/errors v1.1.1 // indirect github.com/go-redis/redis/v8 v8.11.5 github.com/go-redsync/redsync/v4 v4.6.0 - github.com/go-sql-driver/mysql v1.7.0 + github.com/go-sql-driver/mysql v1.6.0 github.com/go-zookeeper/zk v1.0.2 // indirect github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d github.com/golang-jwt/jwt v3.2.2+incompatible @@ -58,7 +58,7 @@ require ( github.com/karlseguin/ccache/v2 v2.0.8 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.15.11 // indirect - github.com/klauspost/reedsolomon v1.11.3 + github.com/klauspost/reedsolomon v1.11.1 github.com/kurin/blazer v0.5.3 github.com/lib/pq v1.10.7 github.com/linxGnu/grocksdb v1.7.10 @@ -77,8 +77,8 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/posener/complete v1.2.3 github.com/pquerna/cachecontrol v0.1.0 - github.com/prometheus/client_golang v1.14.0 - github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/client_golang v1.13.0 + github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect @@ -93,7 +93,7 @@ require ( github.com/stretchr/testify v1.8.1 github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 - github.com/tidwall/gjson v1.14.4 + github.com/tidwall/gjson v1.14.3 github.com/tidwall/match v1.1.1 github.com/tidwall/pretty v1.2.0 // indirect github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 @@ -106,7 +106,7 @@ require ( github.com/xdg-go/scram v1.1.1 // indirect github.com/xdg-go/stringprep v1.0.3 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - go.etcd.io/etcd/client/v3 v3.5.6 + go.etcd.io/etcd/client/v3 v3.5.5 go.mongodb.org/mongo-driver v1.11.0 go.opencensus.io v0.24.0 // indirect gocloud.dev v0.27.0 @@ -117,42 +117,41 @@ require ( golang.org/x/image v0.0.0-20200119044424-58c23975cae1 golang.org/x/net v0.2.0 golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect - golang.org/x/sys v0.3.0 + golang.org/x/sys v0.2.0 golang.org/x/text v0.4.0 // indirect golang.org/x/tools v0.3.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.103.0 google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c // indirect - google.golang.org/grpc v1.51.0 + google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect + google.golang.org/grpc v1.50.1 google.golang.org/protobuf v1.28.1 gopkg.in/inf.v0 v0.9.1 // indirect modernc.org/b v1.0.0 // indirect modernc.org/cc/v3 v3.40.0 // indirect modernc.org/ccgo/v3 v3.16.13 // indirect - modernc.org/libc v1.21.5 // indirect + modernc.org/libc v1.21.4 // indirect modernc.org/mathutil v1.5.0 // indirect modernc.org/memory v1.4.0 // indirect modernc.org/opt v0.1.3 // indirect - modernc.org/sqlite v1.20.0 + modernc.org/sqlite v1.19.4 modernc.org/strutil v1.1.3 modernc.org/token v1.0.1 // indirect ) require ( - github.com/Jille/raft-grpc-transport v1.3.0 + github.com/Jille/raft-grpc-transport v1.2.0 github.com/arangodb/go-driver v1.4.0 github.com/armon/go-metrics v0.4.1 github.com/fluent/fluent-logger-golang v1.9.0 - github.com/google/flatbuffers v22.11.23+incompatible + github.com/google/flatbuffers v22.10.26+incompatible github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17 github.com/hashicorp/raft v1.3.11 github.com/hashicorp/raft-boltdb/v2 v2.2.2 - github.com/otiai10/copy v1.9.0 github.com/rabbitmq/amqp091-go v1.5.0 github.com/tikv/client-go/v2 v2.0.2 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 - github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1 + github.com/ydb-platform/ydb-go-sdk/v3 v3.39.0 golang.org/x/sync v0.1.0 google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 ) @@ -160,7 +159,7 @@ require ( require ( cloud.google.com/go/compute v1.12.1 // indirect cloud.google.com/go/compute/metadata v0.2.1 // indirect - cloud.google.com/go/iam v0.7.0 // indirect + cloud.google.com/go/iam v0.6.0 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/aws/aws-sdk-go-v2 v1.16.8 // indirect github.com/aws/aws-sdk-go-v2/config v1.15.15 // indirect @@ -223,8 +222,8 @@ require ( github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect go.etcd.io/bbolt v1.3.6 // indirect - go.etcd.io/etcd/api/v3 v3.5.6 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.6 // indirect + go.etcd.io/etcd/api/v3 v3.5.5 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.22.0 // indirect diff --git a/go.sum b/go.sum index 1d2d97c34..c863e62f2 100644 --- a/go.sum +++ b/go.sum @@ -61,11 +61,11 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/iam v0.7.0 h1:k4MuwOsS7zGJJ+QfZ5vBK8SgHBAvYN/23BWsiihJ1vs= -cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.6.0 h1:nsqQC88kT5Iwlm4MeNGTpfMWddp6NB/UOLFTH6m1QfQ= +cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= -cloud.google.com/go/kms v1.6.0 h1:OWRZzrPmOZUzurjI2FBGtgY2mB1WaJkqhw6oIwSj0Yg= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= +cloud.google.com/go/kms v1.5.0 h1:uc58n3b/n/F2yDMJzHMbXORkJSh3fzO4/+jju6eR7Zg= +cloud.google.com/go/longrunning v0.1.1 h1:y50CXG4j0+qvEukslYFBCrzaXX0qpFbBzc3PchSu/LE= cloud.google.com/go/monitoring v1.1.0/go.mod h1:L81pzz7HKn14QCMaCs6NTQkdBnE87TElyanS95vIcl4= cloud.google.com/go/monitoring v1.5.0/go.mod h1:/o9y8NYX5j91JjD/JvGLYbi86kL11OjyJXq2XziLJu4= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -73,8 +73,8 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.24.0/go.mod h1:rWv09Te1SsRpRGPiWOMDKraMQTJyJps4MkUCoMGUgqw= -cloud.google.com/go/pubsub v1.27.1 h1:q+J/Nfr6Qx4RQeu3rJcnN48SNC0qzlYzSeqkPq93VHs= -cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsub v1.26.0 h1:Y/HcMxVXgkUV2pYeLMUkclMg0ue6U0jVyI5xEARQ4zA= +cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= cloud.google.com/go/secretmanager v1.5.0/go.mod h1:5C9kM+RwSpkURNovKySkNvGQLUaOgyoR5W0RUx2SyHQ= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= @@ -85,8 +85,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.24.0/go.mod h1:3xrJEFMXBsQLgxwThyjuD3aYlroL0TMRec1ypGUQ0KE= -cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= -cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.28.0 h1:DLrIZ6xkeZX6K70fU/boWx5INJumt6f+nwwWSHXzzGY= +cloud.google.com/go/storage v1.28.0/go.mod h1:qlgZML35PXA3zoEnIkiPLY4/TOkUleufRlu6qmcf7sI= cloud.google.com/go/trace v1.0.0/go.mod h1:4iErSByzxkyHWzzlAj63/Gmjz0NH1ASqhJguHpGcr6A= cloud.google.com/go/trace v1.2.0/go.mod h1:Wc8y/uYyOhPy12KEnXG9XGrvfMz5F5SrYecQlbW1rwM= code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= @@ -157,8 +157,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/GoogleCloudPlatform/cloudsql-proxy v1.31.2/go.mod h1:qR6jVnZTKDCW3j+fC9mOEPHm++1nKDMkqbbkD6KNsfo= -github.com/Jille/raft-grpc-transport v1.3.0 h1:s6a7F4k23n7IlH5O6fWW5xz2sxbz6RIB4CzWlsvmbpY= -github.com/Jille/raft-grpc-transport v1.3.0/go.mod h1:afVUd8LQKUUo3V/ToLBH3mbSyvivRlMYCDK0eJRGTfQ= +github.com/Jille/raft-grpc-transport v1.2.0 h1:W/YSPz8IsirEyomjKmDog5Xk71o9+l4KhyMEX2TsgSs= +github.com/Jille/raft-grpc-transport v1.2.0/go.mod h1:GQGUXJfjlzwA390Ox1AyVYpjCLhtGd6yqY9Sb5hpQfc= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= @@ -239,8 +239,8 @@ github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4 github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.45/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.68/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.146 h1:7YdGgPxDPRJu/yYffzZp/H7yHzQ6AqmuNFZPYraaN8I= -github.com/aws/aws-sdk-go v1.44.146/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.136 h1:J1KJJssa8pjU8jETYUxwRS37KTcxjACfKd9GK8t+5ZU= +github.com/aws/aws-sdk-go v1.44.136/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.16.8 h1:gOe9UPR98XSf7oEJCcojYg+N2/jCRm4DdeIsP85pIyQ= github.com/aws/aws-sdk-go-v2 v1.16.8/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= @@ -682,9 +682,8 @@ github.com/go-redsync/redsync/v4 v4.6.0 h1:CXpvsHB3XzktCleBu2Vo9Df0/qInrTG3jgzhv github.com/go-redsync/redsync/v4 v4.6.0/go.mod h1:IxV3sygNwjOERTXrj3XvNMSb1tgNgic8GvM8alwnWcM= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= -github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -798,8 +797,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/flatbuffers v22.11.23+incompatible h1:334TygA7iuxt0hoamawsM36xoui01YiouEZnr0qeFMI= -github.com/google/flatbuffers v22.11.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v22.10.26+incompatible h1:z1QiaMyPu1x3Z6xf2u1dsLj1ZxicdGSeaLpCuIsQNZM= +github.com/google/flatbuffers v22.10.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -855,7 +854,6 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20220318212150-b2ab0324ddda/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20220608213341-c488b8fa1db3/go.mod h1:gSuNB+gJaOiQKLEZ+q+PK9Mq3SOzhRcw2GsGS/FhYDk= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -982,7 +980,7 @@ github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/nomad/api v0.0.0-20220629141207-c2428e1673ec/go.mod h1:jP79oXjopTyH6E8LF0CEMq67STgrlmBRIyijA0tuR5o= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= -github.com/hashicorp/raft v1.3.7/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft v1.3.1/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.3.11 h1:p3v6gf6l3S797NnK5av3HcczOC1T5CLoaRvg0g9ys4A= github.com/hashicorp/raft v1.3.11/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4= github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea h1:RxcPJuutPRM8PUOyiweMmkuNO+RJyfy2jds2gfvgNmU= @@ -1114,8 +1112,8 @@ github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/reedsolomon v1.11.3 h1:rX9UNNvDhJ0Bq45y6uBy/eYehcjyz5faokTuZmu1Q9U= -github.com/klauspost/reedsolomon v1.11.3/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747hZ4oYp2Ml60Lb/k= +github.com/klauspost/reedsolomon v1.11.1 h1:0gCWQXOB8pVe1Y5SGozDA5t2qoVxX3prsV+qHgI/Fik= +github.com/klauspost/reedsolomon v1.11.1/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747hZ4oYp2Ml60Lb/k= github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1362,13 +1360,6 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4= -github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI= -github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= -github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= -github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= -github.com/otiai10/mint v1.4.0 h1:umwcf7gbpEwf7WFzqmWwSv0CzbeMsae2u9ZvpP8j2q4= -github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= @@ -1437,17 +1428,16 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= +github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= @@ -1619,8 +1609,8 @@ github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= +github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -1693,8 +1683,8 @@ github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba/go.mo github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 h1:EYSI1kulnHb0H0zt3yOw4cRj4ABMSMGwNe43D+fX7e4= github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2/go.mod h1:Xfjce+VMU9yJVr1lj60yK2fFPWjB4jr/4cp3K7cjzi4= github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88= -github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1 h1:6a/yMcXT+ZpybYsGvEpFSsOdQlbqtArycUqY7U1Yd1w= -github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1/go.mod h1:hJqWSE2NZ2o2c9geHtRJee+xwiHgEfQX9koBZPLTfHY= +github.com/ydb-platform/ydb-go-sdk/v3 v3.39.0 h1:fGRZa9mGacXBG83BhgcP6i4BPVXHRQfxFoqqleEpHCU= +github.com/ydb-platform/ydb-go-sdk/v3 v3.39.0/go.mod h1:hJqWSE2NZ2o2c9geHtRJee+xwiHgEfQX9koBZPLTfHY= github.com/ydb-platform/ydb-go-yc v0.8.3 h1:92UUUMsfvtMl6mho8eQ9lbkiPrF3a9CT+RrVRAKNRwo= github.com/ydb-platform/ydb-go-yc v0.8.3/go.mod h1:zUolAFGzJ5XG8uwiseTLr9Lapm7L7hdVdZgLSuv9FXE= github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg= @@ -1720,15 +1710,15 @@ go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/api/v3 v3.5.6 h1:Cy2qx3npLcYqTKqGJzMypnMv2tiRyifZJ17BlWIWA7A= -go.etcd.io/etcd/api/v3 v3.5.6/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= +go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= +go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.6 h1:TXQWYceBKqLp4sa87rcPs11SXxUA/mHwH975v+BDvLU= -go.etcd.io/etcd/client/pkg/v3 v3.5.6/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= +go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= +go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/client/v3 v3.5.6 h1:coLs69PWCXE9G4FKquzNaSHrRyMCAXwF+IX1tAPVO8E= -go.etcd.io/etcd/client/v3 v3.5.6/go.mod h1:f6GRinRMCsFVv9Ht42EyY7nfsVGwrNO0WEoS2pRKzQk= +go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= +go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= @@ -2196,8 +2186,8 @@ golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2228,7 +2218,7 @@ golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2492,8 +2482,8 @@ google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljW google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220802133213-ce4fa296bf78/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= -google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c h1:S34D59DS2GWOEwWNt4fYmTcFrtlOgukG2k9WsomZ7tg= -google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c h1:QgY/XxIAIeccR+Ca/rDdKubLIU9rcJ3xfy1DC/Wd2Oo= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -2536,8 +2526,8 @@ google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b h1:NuxyvVZoDfHZwYW9LD4GJiF5/nhiSyP4/InTrvw9Ibk= google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM= @@ -2686,8 +2676,8 @@ modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= -modernc.org/libc v1.21.5 h1:xBkU9fnHV+hvZuPSRszN0AXDG4M7nwPLwTWwkYcvLCI= -modernc.org/libc v1.21.5/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= +modernc.org/libc v1.21.4 h1:CzTlumWeIbPV5/HVIMzYHNPCRP8uiU/CWiN2gtd/Qu8= +modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= @@ -2695,8 +2685,8 @@ modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk= modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.20.0 h1:80zmD3BGkm8BZ5fUi/4lwJQHiO3GXgIUvZRXpoIfROY= -modernc.org/sqlite v1.20.0/go.mod h1:EsYz8rfOvLCiYTy5ZFsOYzoCcRMu98YYkwAcCw5YIYw= +modernc.org/sqlite v1.19.4 h1:nlPIDqumn6/mSvs7T5C8MNYEuN73sISzPdKtMdURpUI= +modernc.org/sqlite v1.19.4/go.mod h1:x/yZNb3h5+I3zGQSlwIv4REL5eJhiRkUH5MReogAeIc= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= From 9acf3e71e9cad2493597d55b38970ccfa9281131 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Dec 2022 23:08:15 -0800 Subject: [PATCH 04/11] nit again; --- go.mod | 40 +++++++++++++-------------- go.sum | 87 ++++++++++++++++++++++++++++++---------------------------- 2 files changed, 65 insertions(+), 62 deletions(-) diff --git a/go.mod b/go.mod index 1fe1c4dcb..cca84e893 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ go 1.19 require ( cloud.google.com/go v0.105.0 // indirect - cloud.google.com/go/pubsub v1.26.0 - cloud.google.com/go/storage v1.28.0 + cloud.google.com/go/pubsub v1.27.1 + cloud.google.com/go/storage v1.28.1 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Shopify/sarama v1.37.2 - github.com/aws/aws-sdk-go v1.44.136 + github.com/aws/aws-sdk-go v1.44.146 github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -32,7 +32,7 @@ require ( github.com/go-errors/errors v1.1.1 // indirect github.com/go-redis/redis/v8 v8.11.5 github.com/go-redsync/redsync/v4 v4.6.0 - github.com/go-sql-driver/mysql v1.6.0 + github.com/go-sql-driver/mysql v1.7.0 github.com/go-zookeeper/zk v1.0.2 // indirect github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d github.com/golang-jwt/jwt v3.2.2+incompatible @@ -58,7 +58,7 @@ require ( github.com/karlseguin/ccache/v2 v2.0.8 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.15.11 // indirect - github.com/klauspost/reedsolomon v1.11.1 + github.com/klauspost/reedsolomon v1.11.3 github.com/kurin/blazer v0.5.3 github.com/lib/pq v1.10.7 github.com/linxGnu/grocksdb v1.7.10 @@ -77,8 +77,8 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/posener/complete v1.2.3 github.com/pquerna/cachecontrol v0.1.0 - github.com/prometheus/client_golang v1.13.0 - github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect @@ -93,7 +93,7 @@ require ( github.com/stretchr/testify v1.8.1 github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 - github.com/tidwall/gjson v1.14.3 + github.com/tidwall/gjson v1.14.4 github.com/tidwall/match v1.1.1 github.com/tidwall/pretty v1.2.0 // indirect github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 @@ -106,7 +106,7 @@ require ( github.com/xdg-go/scram v1.1.1 // indirect github.com/xdg-go/stringprep v1.0.3 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - go.etcd.io/etcd/client/v3 v3.5.5 + go.etcd.io/etcd/client/v3 v3.5.6 go.mongodb.org/mongo-driver v1.11.0 go.opencensus.io v0.24.0 // indirect gocloud.dev v0.27.0 @@ -117,41 +117,41 @@ require ( golang.org/x/image v0.0.0-20200119044424-58c23975cae1 golang.org/x/net v0.2.0 golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect - golang.org/x/sys v0.2.0 + golang.org/x/sys v0.3.0 golang.org/x/text v0.4.0 // indirect golang.org/x/tools v0.3.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.103.0 google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect - google.golang.org/grpc v1.50.1 + google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c // indirect + google.golang.org/grpc v1.51.0 google.golang.org/protobuf v1.28.1 gopkg.in/inf.v0 v0.9.1 // indirect modernc.org/b v1.0.0 // indirect modernc.org/cc/v3 v3.40.0 // indirect modernc.org/ccgo/v3 v3.16.13 // indirect - modernc.org/libc v1.21.4 // indirect + modernc.org/libc v1.21.5 // indirect modernc.org/mathutil v1.5.0 // indirect modernc.org/memory v1.4.0 // indirect modernc.org/opt v0.1.3 // indirect - modernc.org/sqlite v1.19.4 + modernc.org/sqlite v1.20.0 modernc.org/strutil v1.1.3 modernc.org/token v1.0.1 // indirect ) require ( - github.com/Jille/raft-grpc-transport v1.2.0 + github.com/Jille/raft-grpc-transport v1.3.0 github.com/arangodb/go-driver v1.4.0 github.com/armon/go-metrics v0.4.1 github.com/fluent/fluent-logger-golang v1.9.0 - github.com/google/flatbuffers v22.10.26+incompatible + github.com/google/flatbuffers v22.11.23+incompatible github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17 github.com/hashicorp/raft v1.3.11 github.com/hashicorp/raft-boltdb/v2 v2.2.2 github.com/rabbitmq/amqp091-go v1.5.0 github.com/tikv/client-go/v2 v2.0.2 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 - github.com/ydb-platform/ydb-go-sdk/v3 v3.39.0 + github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1 golang.org/x/sync v0.1.0 google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 ) @@ -159,7 +159,7 @@ require ( require ( cloud.google.com/go/compute v1.12.1 // indirect cloud.google.com/go/compute/metadata v0.2.1 // indirect - cloud.google.com/go/iam v0.6.0 // indirect + cloud.google.com/go/iam v0.7.0 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/aws/aws-sdk-go-v2 v1.16.8 // indirect github.com/aws/aws-sdk-go-v2/config v1.15.15 // indirect @@ -222,8 +222,8 @@ require ( github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect go.etcd.io/bbolt v1.3.6 // indirect - go.etcd.io/etcd/api/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect + go.etcd.io/etcd/api/v3 v3.5.6 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.22.0 // indirect diff --git a/go.sum b/go.sum index c863e62f2..d27e85aff 100644 --- a/go.sum +++ b/go.sum @@ -61,11 +61,11 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/iam v0.6.0 h1:nsqQC88kT5Iwlm4MeNGTpfMWddp6NB/UOLFTH6m1QfQ= -cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0 h1:k4MuwOsS7zGJJ+QfZ5vBK8SgHBAvYN/23BWsiihJ1vs= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= -cloud.google.com/go/kms v1.5.0 h1:uc58n3b/n/F2yDMJzHMbXORkJSh3fzO4/+jju6eR7Zg= -cloud.google.com/go/longrunning v0.1.1 h1:y50CXG4j0+qvEukslYFBCrzaXX0qpFbBzc3PchSu/LE= +cloud.google.com/go/kms v1.6.0 h1:OWRZzrPmOZUzurjI2FBGtgY2mB1WaJkqhw6oIwSj0Yg= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/monitoring v1.1.0/go.mod h1:L81pzz7HKn14QCMaCs6NTQkdBnE87TElyanS95vIcl4= cloud.google.com/go/monitoring v1.5.0/go.mod h1:/o9y8NYX5j91JjD/JvGLYbi86kL11OjyJXq2XziLJu4= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -73,8 +73,8 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.24.0/go.mod h1:rWv09Te1SsRpRGPiWOMDKraMQTJyJps4MkUCoMGUgqw= -cloud.google.com/go/pubsub v1.26.0 h1:Y/HcMxVXgkUV2pYeLMUkclMg0ue6U0jVyI5xEARQ4zA= -cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= +cloud.google.com/go/pubsub v1.27.1 h1:q+J/Nfr6Qx4RQeu3rJcnN48SNC0qzlYzSeqkPq93VHs= +cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= cloud.google.com/go/secretmanager v1.5.0/go.mod h1:5C9kM+RwSpkURNovKySkNvGQLUaOgyoR5W0RUx2SyHQ= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= @@ -85,8 +85,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.24.0/go.mod h1:3xrJEFMXBsQLgxwThyjuD3aYlroL0TMRec1ypGUQ0KE= -cloud.google.com/go/storage v1.28.0 h1:DLrIZ6xkeZX6K70fU/boWx5INJumt6f+nwwWSHXzzGY= -cloud.google.com/go/storage v1.28.0/go.mod h1:qlgZML35PXA3zoEnIkiPLY4/TOkUleufRlu6qmcf7sI= +cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= cloud.google.com/go/trace v1.0.0/go.mod h1:4iErSByzxkyHWzzlAj63/Gmjz0NH1ASqhJguHpGcr6A= cloud.google.com/go/trace v1.2.0/go.mod h1:Wc8y/uYyOhPy12KEnXG9XGrvfMz5F5SrYecQlbW1rwM= code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= @@ -157,8 +157,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/GoogleCloudPlatform/cloudsql-proxy v1.31.2/go.mod h1:qR6jVnZTKDCW3j+fC9mOEPHm++1nKDMkqbbkD6KNsfo= -github.com/Jille/raft-grpc-transport v1.2.0 h1:W/YSPz8IsirEyomjKmDog5Xk71o9+l4KhyMEX2TsgSs= -github.com/Jille/raft-grpc-transport v1.2.0/go.mod h1:GQGUXJfjlzwA390Ox1AyVYpjCLhtGd6yqY9Sb5hpQfc= +github.com/Jille/raft-grpc-transport v1.3.0 h1:s6a7F4k23n7IlH5O6fWW5xz2sxbz6RIB4CzWlsvmbpY= +github.com/Jille/raft-grpc-transport v1.3.0/go.mod h1:afVUd8LQKUUo3V/ToLBH3mbSyvivRlMYCDK0eJRGTfQ= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= @@ -239,8 +239,8 @@ github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4 github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.45/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.68/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.136 h1:J1KJJssa8pjU8jETYUxwRS37KTcxjACfKd9GK8t+5ZU= -github.com/aws/aws-sdk-go v1.44.136/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.146 h1:7YdGgPxDPRJu/yYffzZp/H7yHzQ6AqmuNFZPYraaN8I= +github.com/aws/aws-sdk-go v1.44.146/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.16.8 h1:gOe9UPR98XSf7oEJCcojYg+N2/jCRm4DdeIsP85pIyQ= github.com/aws/aws-sdk-go-v2 v1.16.8/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= @@ -682,8 +682,9 @@ github.com/go-redsync/redsync/v4 v4.6.0 h1:CXpvsHB3XzktCleBu2Vo9Df0/qInrTG3jgzhv github.com/go-redsync/redsync/v4 v4.6.0/go.mod h1:IxV3sygNwjOERTXrj3XvNMSb1tgNgic8GvM8alwnWcM= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -797,8 +798,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/flatbuffers v22.10.26+incompatible h1:z1QiaMyPu1x3Z6xf2u1dsLj1ZxicdGSeaLpCuIsQNZM= -github.com/google/flatbuffers v22.10.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v22.11.23+incompatible h1:334TygA7iuxt0hoamawsM36xoui01YiouEZnr0qeFMI= +github.com/google/flatbuffers v22.11.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -854,6 +855,7 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20220318212150-b2ab0324ddda/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20220608213341-c488b8fa1db3/go.mod h1:gSuNB+gJaOiQKLEZ+q+PK9Mq3SOzhRcw2GsGS/FhYDk= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -980,7 +982,7 @@ github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/nomad/api v0.0.0-20220629141207-c2428e1673ec/go.mod h1:jP79oXjopTyH6E8LF0CEMq67STgrlmBRIyijA0tuR5o= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= -github.com/hashicorp/raft v1.3.1/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft v1.3.7/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.3.11 h1:p3v6gf6l3S797NnK5av3HcczOC1T5CLoaRvg0g9ys4A= github.com/hashicorp/raft v1.3.11/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4= github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea h1:RxcPJuutPRM8PUOyiweMmkuNO+RJyfy2jds2gfvgNmU= @@ -1112,8 +1114,8 @@ github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/reedsolomon v1.11.1 h1:0gCWQXOB8pVe1Y5SGozDA5t2qoVxX3prsV+qHgI/Fik= -github.com/klauspost/reedsolomon v1.11.1/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747hZ4oYp2Ml60Lb/k= +github.com/klauspost/reedsolomon v1.11.3 h1:rX9UNNvDhJ0Bq45y6uBy/eYehcjyz5faokTuZmu1Q9U= +github.com/klauspost/reedsolomon v1.11.3/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747hZ4oYp2Ml60Lb/k= github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1428,16 +1430,17 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= @@ -1609,8 +1612,8 @@ github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= -github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= -github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -1683,8 +1686,8 @@ github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba/go.mo github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 h1:EYSI1kulnHb0H0zt3yOw4cRj4ABMSMGwNe43D+fX7e4= github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2/go.mod h1:Xfjce+VMU9yJVr1lj60yK2fFPWjB4jr/4cp3K7cjzi4= github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88= -github.com/ydb-platform/ydb-go-sdk/v3 v3.39.0 h1:fGRZa9mGacXBG83BhgcP6i4BPVXHRQfxFoqqleEpHCU= -github.com/ydb-platform/ydb-go-sdk/v3 v3.39.0/go.mod h1:hJqWSE2NZ2o2c9geHtRJee+xwiHgEfQX9koBZPLTfHY= +github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1 h1:6a/yMcXT+ZpybYsGvEpFSsOdQlbqtArycUqY7U1Yd1w= +github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1/go.mod h1:hJqWSE2NZ2o2c9geHtRJee+xwiHgEfQX9koBZPLTfHY= github.com/ydb-platform/ydb-go-yc v0.8.3 h1:92UUUMsfvtMl6mho8eQ9lbkiPrF3a9CT+RrVRAKNRwo= github.com/ydb-platform/ydb-go-yc v0.8.3/go.mod h1:zUolAFGzJ5XG8uwiseTLr9Lapm7L7hdVdZgLSuv9FXE= github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg= @@ -1710,15 +1713,15 @@ go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= -go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= +go.etcd.io/etcd/api/v3 v3.5.6 h1:Cy2qx3npLcYqTKqGJzMypnMv2tiRyifZJ17BlWIWA7A= +go.etcd.io/etcd/api/v3 v3.5.6/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= -go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= +go.etcd.io/etcd/client/pkg/v3 v3.5.6 h1:TXQWYceBKqLp4sa87rcPs11SXxUA/mHwH975v+BDvLU= +go.etcd.io/etcd/client/pkg/v3 v3.5.6/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= -go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= +go.etcd.io/etcd/client/v3 v3.5.6 h1:coLs69PWCXE9G4FKquzNaSHrRyMCAXwF+IX1tAPVO8E= +go.etcd.io/etcd/client/v3 v3.5.6/go.mod h1:f6GRinRMCsFVv9Ht42EyY7nfsVGwrNO0WEoS2pRKzQk= go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= @@ -2186,8 +2189,8 @@ golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2218,7 +2221,7 @@ golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= +golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2482,8 +2485,8 @@ google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljW google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220802133213-ce4fa296bf78/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= -google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c h1:QgY/XxIAIeccR+Ca/rDdKubLIU9rcJ3xfy1DC/Wd2Oo= -google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c h1:S34D59DS2GWOEwWNt4fYmTcFrtlOgukG2k9WsomZ7tg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -2526,8 +2529,8 @@ google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= -google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b h1:NuxyvVZoDfHZwYW9LD4GJiF5/nhiSyP4/InTrvw9Ibk= google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM= @@ -2676,8 +2679,8 @@ modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= -modernc.org/libc v1.21.4 h1:CzTlumWeIbPV5/HVIMzYHNPCRP8uiU/CWiN2gtd/Qu8= -modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= +modernc.org/libc v1.21.5 h1:xBkU9fnHV+hvZuPSRszN0AXDG4M7nwPLwTWwkYcvLCI= +modernc.org/libc v1.21.5/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= @@ -2685,8 +2688,8 @@ modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk= modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.19.4 h1:nlPIDqumn6/mSvs7T5C8MNYEuN73sISzPdKtMdURpUI= -modernc.org/sqlite v1.19.4/go.mod h1:x/yZNb3h5+I3zGQSlwIv4REL5eJhiRkUH5MReogAeIc= +modernc.org/sqlite v1.20.0 h1:80zmD3BGkm8BZ5fUi/4lwJQHiO3GXgIUvZRXpoIfROY= +modernc.org/sqlite v1.20.0/go.mod h1:EsYz8rfOvLCiYTy5ZFsOYzoCcRMu98YYkwAcCw5YIYw= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= From d5dfc7a4ae82ecbd437f0bc805154bb96505be6c Mon Sep 17 00:00:00 2001 From: root Date: Thu, 15 Dec 2022 14:21:05 -0800 Subject: [PATCH 05/11] comments --- .../shell/command_fs_meta_snapshots_create.go | 53 +++++++------------ 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/weed/shell/command_fs_meta_snapshots_create.go b/weed/shell/command_fs_meta_snapshots_create.go index adc07e804..ebe12b536 100644 --- a/weed/shell/command_fs_meta_snapshots_create.go +++ b/weed/shell/command_fs_meta_snapshots_create.go @@ -17,7 +17,6 @@ import ( ) const LevelDbPath = "/tmp/snapshots.db" -const SnapshotsParentPath = "/seaweedfs_meta_snapshots/" const DateFormat = "2006-01-02" func init() { @@ -34,28 +33,26 @@ func (c *commandFsMetaSnapshotsCreate) Name() string { func (c *commandFsMetaSnapshotsCreate) Help() string { return `create snapshots of meta data from given time range. - fs.meta.snapshots.create -s=2022-08-09 -e=2022-10-12 -f 7 create snapshot starting from 2022-08-09 ending at 2022-10-12 every seven days. - - //These snapshot maybe later used to backup the system to certain timestamp. + fs.meta.snapshots.create -snapshot-interval=7 -snapshot-cnt=3 -path=/your/path + // fs.meta.snapshots.create will generate desired number of snapshots with desired duration interval from yesterday the generated files will be saved from input path. + // These snapshot maybe later used to backup the system to certain timestamp. ` } -func processMetaDataEvents(store *filer_leveldb.LevelDBStore, data []byte, snapshotCnt int, snapshotCheckPoints []time.Time, homeDir string) (err error) { +func processMetaDataEvents(store *filer_leveldb.LevelDBStore, data []byte, snapshotCheckPoints []time.Time, homeDir string, snapshotPath string) (err error) { var event filer_pb.SubscribeMetadataResponse err = proto.Unmarshal(data, &event) if err != nil { return err } - fmt.Printf("%+v\n", event) eventTime := event.TsNs - println(time.Unix(0, eventTime).Format(DateFormat)) - for snapshotCnt < len(snapshotCheckPoints) && time.Unix(0, eventTime).After(snapshotCheckPoints[snapshotCnt]) { - snapshotPath := homeDir + SnapshotsParentPath + snapshotCheckPoints[snapshotCnt].Format(DateFormat) + snapshotCnt := len(snapshotCheckPoints) - 1 + for snapshotCnt >= 0 && time.Unix(0, eventTime).After(snapshotCheckPoints[snapshotCnt]) { + snapshotPath := homeDir + snapshotPath + snapshotCheckPoints[snapshotCnt].Format(DateFormat) err = CreateIfNotExists(snapshotPath, 0755) if err != nil { return err } - println("generating snapshots of metadata at: " + snapshotPath) err = generateSnapshots(homeDir+LevelDbPath, snapshotPath) if err != nil { return err @@ -69,23 +66,17 @@ func processMetaDataEvents(store *filer_leveldb.LevelDBStore, data []byte, snaps if filer_pb.IsEmpty(&event) { return nil } else if filer_pb.IsCreate(&event) { - println("+", util.FullPath(event.EventNotification.NewParentPath).Child(event.EventNotification.NewEntry.Name)) entry := filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry) return store.InsertEntry(ctx, entry) } else if filer_pb.IsDelete(&event) { - println("-", util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)) return store.DeleteEntry(ctx, util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)) } else if filer_pb.IsUpdate(&event) { - println("~", util.FullPath(event.EventNotification.NewParentPath).Child(event.EventNotification.NewEntry.Name)) entry := filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry) return store.UpdateEntry(ctx, entry) } else { - // renaming - println("-", util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)) if err := store.DeleteEntry(ctx, util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)); err != nil { return err } - println("+", util.FullPath(event.EventNotification.NewParentPath).Child(event.EventNotification.NewEntry.Name)) return store.InsertEntry(ctx, filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry)) } return nil @@ -166,26 +157,18 @@ func CreateIfNotExists(dir string, perm os.FileMode) error { func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { fsMetaSnapshotsCreateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) - start := fsMetaSnapshotsCreateCommand.String("s", "", "start date of metadata compaction in yyyy-mm-dd format") - end := fsMetaSnapshotsCreateCommand.String("e", "", "end date of metadata compaction in yyyy-mm-dd format") - frequency := fsMetaSnapshotsCreateCommand.Int("f", 7, "the frequency of generating the metadata snapshots in days") + snapshotPath := fsMetaSnapshotsCreateCommand.String("path", "", "the path to store generated snapshot files") + snapshotCnt := fsMetaSnapshotsCreateCommand.Int("snapshot-cnt", 3, "number of snapshots generated") + snapshotInterval := fsMetaSnapshotsCreateCommand.Int("snapshot-interval", 7, "the duration interval between each generated snapshot") if err = fsMetaSnapshotsCreateCommand.Parse(args); err != nil { return err } - startDate, err := time.Parse(DateFormat, *start) - if err != nil { - return err - } - endDate, err := time.Parse(DateFormat, *end) - if err != nil { - return err - } + snapshotdate := time.Now().AddDate(0, 0, -1) var snapshotCheckPoints []time.Time - for startDate.Before(endDate) || startDate.Equal(endDate) { - snapshotCheckPoints = append(snapshotCheckPoints, startDate) - startDate = startDate.AddDate(0, 0, *frequency) + for i := 0; i < *snapshotCnt; i++ { + snapshotCheckPoints = append(snapshotCheckPoints, snapshotdate) + snapshotdate = snapshotdate.AddDate(0, 0, -1**snapshotInterval) } - snapshotCnt := 0 homeDirname, err := os.UserHomeDir() if err != nil { return err @@ -197,11 +180,11 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, } store := &filer_leveldb.LevelDBStore{} store.CustomInitialize(levelDbPath) - path := filer.SystemLogDir + changeLogPath := filer.SystemLogDir var processEntry func(entry *filer_pb.Entry, isLast bool) error processEntry = func(entry *filer_pb.Entry, isLast bool) error { if entry.IsDirectory { - return filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(path+"/"+entry.Name), "", processEntry) + return filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(changeLogPath+"/"+entry.Name), "", processEntry) } totalSize := filer.FileSize(entry) buf := mem.Allocate(int(totalSize)) @@ -219,7 +202,7 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, return err } idx = idx + 4 + logEntrySize - err = processMetaDataEvents(store, logEntry.Data, snapshotCnt, snapshotCheckPoints, homeDirname) + err = processMetaDataEvents(store, logEntry.Data, snapshotCheckPoints, homeDirname, *snapshotPath) if err != nil { return err } @@ -228,6 +211,6 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, return nil } - err = filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(path), "", processEntry) + err = filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(changeLogPath), "", processEntry) return err } From b4477fae9c6cb6b9d165576f51956541183620f5 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 15 Dec 2022 14:27:01 -0800 Subject: [PATCH 06/11] nit --- weed/filer/leveldb/leveldb_store.go | 5 ---- .../shell/command_fs_meta_snapshots_create.go | 29 ++++++++++++++++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/weed/filer/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go index 08dc33f02..747d1104d 100644 --- a/weed/filer/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -43,11 +43,6 @@ func (store *LevelDBStore) Initialize(configuration weed_util.Configuration, pre return store.initialize(dir) } -// For adhoc usage -func (store *LevelDBStore) CustomInitialize(dir string) (err error) { - return store.initialize(dir) -} - func (store *LevelDBStore) initialize(dir string) (err error) { glog.Infof("filer store dir: %s", dir) os.MkdirAll(dir, 0755) diff --git a/weed/shell/command_fs_meta_snapshots_create.go b/weed/shell/command_fs_meta_snapshots_create.go index ebe12b536..bccff2230 100644 --- a/weed/shell/command_fs_meta_snapshots_create.go +++ b/weed/shell/command_fs_meta_snapshots_create.go @@ -30,6 +30,30 @@ func (c *commandFsMetaSnapshotsCreate) Name() string { return "fs.meta.snapshots.create" } +type SnapshotConfig struct { + dir string +} + +func (c SnapshotConfig) GetString(key string) string { + return c.dir +} + +func (c SnapshotConfig) GetBool(key string) bool { + panic("implement me") +} + +func (c SnapshotConfig) GetInt(key string) int { + panic("implement me") +} + +func (c SnapshotConfig) GetStringSlice(key string) []string { + panic("implement me") +} + +func (c SnapshotConfig) SetDefault(key string, value interface{}) { + panic("implement me") +} + func (c *commandFsMetaSnapshotsCreate) Help() string { return `create snapshots of meta data from given time range. @@ -179,7 +203,10 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, return err } store := &filer_leveldb.LevelDBStore{} - store.CustomInitialize(levelDbPath) + config := SnapshotConfig{ + dir: levelDbPath, + } + store.Initialize(config, "") changeLogPath := filer.SystemLogDir var processEntry func(entry *filer_pb.Entry, isLast bool) error processEntry = func(entry *filer_pb.Entry, isLast bool) error { From 058ece86d245e1606a26214915afef7c54af4cdf Mon Sep 17 00:00:00 2001 From: root Date: Thu, 15 Dec 2022 14:36:53 -0800 Subject: [PATCH 07/11] more nits --- weed/shell/command_fs_meta_snapshots_create.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/weed/shell/command_fs_meta_snapshots_create.go b/weed/shell/command_fs_meta_snapshots_create.go index bccff2230..9cf1c3a75 100644 --- a/weed/shell/command_fs_meta_snapshots_create.go +++ b/weed/shell/command_fs_meta_snapshots_create.go @@ -187,11 +187,16 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, if err = fsMetaSnapshotsCreateCommand.Parse(args); err != nil { return err } - snapshotdate := time.Now().AddDate(0, 0, -1) + yesterday := time.Now().Add(-time.Hour * 24).Format(DateFormat) + // ensure snapshot start at yesterday 00:00 + snapshotDate, err := time.Parse(DateFormat, yesterday) + if err != nil { + return err + } var snapshotCheckPoints []time.Time for i := 0; i < *snapshotCnt; i++ { - snapshotCheckPoints = append(snapshotCheckPoints, snapshotdate) - snapshotdate = snapshotdate.AddDate(0, 0, -1**snapshotInterval) + snapshotCheckPoints = append(snapshotCheckPoints, snapshotDate) + snapshotDate = snapshotDate.AddDate(0, 0, -1**snapshotInterval) } homeDirname, err := os.UserHomeDir() if err != nil { From 73840fb076eac4c63dae30494dcd4cc9967bb8fc Mon Sep 17 00:00:00 2001 From: root Date: Thu, 15 Dec 2022 22:39:30 -0800 Subject: [PATCH 08/11] cover edge cases --- .../shell/command_fs_meta_snapshots_create.go | 52 ++++++++++++------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/weed/shell/command_fs_meta_snapshots_create.go b/weed/shell/command_fs_meta_snapshots_create.go index 9cf1c3a75..e3f95f3a6 100644 --- a/weed/shell/command_fs_meta_snapshots_create.go +++ b/weed/shell/command_fs_meta_snapshots_create.go @@ -63,47 +63,46 @@ func (c *commandFsMetaSnapshotsCreate) Help() string { ` } -func processMetaDataEvents(store *filer_leveldb.LevelDBStore, data []byte, snapshotCheckPoints []time.Time, homeDir string, snapshotPath string) (err error) { +func processMetaDataEvents(store *filer_leveldb.LevelDBStore, data []byte, unfinshiedSnapshotCnt int, snapshotCheckPoints []time.Time, homeDir string, snapshotPath string) (todoCnt int, err error) { var event filer_pb.SubscribeMetadataResponse err = proto.Unmarshal(data, &event) if err != nil { - return err + return unfinshiedSnapshotCnt, err } eventTime := event.TsNs - snapshotCnt := len(snapshotCheckPoints) - 1 - for snapshotCnt >= 0 && time.Unix(0, eventTime).After(snapshotCheckPoints[snapshotCnt]) { - snapshotPath := homeDir + snapshotPath + snapshotCheckPoints[snapshotCnt].Format(DateFormat) + for unfinshiedSnapshotCnt >= 0 && time.Unix(0, eventTime).After(snapshotCheckPoints[unfinshiedSnapshotCnt]) { + snapshotPath := filepath.Join(homeDir, snapshotPath, snapshotCheckPoints[unfinshiedSnapshotCnt].Format(DateFormat)) err = CreateIfNotExists(snapshotPath, 0755) if err != nil { - return err + return unfinshiedSnapshotCnt, err } err = generateSnapshots(homeDir+LevelDbPath, snapshotPath) if err != nil { - return err + return unfinshiedSnapshotCnt, err } - snapshotCnt++ + unfinshiedSnapshotCnt-- } - if snapshotCnt == len(snapshotCheckPoints) { - return nil + if unfinshiedSnapshotCnt < 0 { + return unfinshiedSnapshotCnt, nil } ctx := context.Background() if filer_pb.IsEmpty(&event) { - return nil + return unfinshiedSnapshotCnt, nil } else if filer_pb.IsCreate(&event) { entry := filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry) - return store.InsertEntry(ctx, entry) + return unfinshiedSnapshotCnt, store.InsertEntry(ctx, entry) } else if filer_pb.IsDelete(&event) { - return store.DeleteEntry(ctx, util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)) + return unfinshiedSnapshotCnt, store.DeleteEntry(ctx, util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)) } else if filer_pb.IsUpdate(&event) { entry := filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry) - return store.UpdateEntry(ctx, entry) + return unfinshiedSnapshotCnt, store.UpdateEntry(ctx, entry) } else { if err := store.DeleteEntry(ctx, util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)); err != nil { - return err + return unfinshiedSnapshotCnt, err } - return store.InsertEntry(ctx, filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry)) + return unfinshiedSnapshotCnt, store.InsertEntry(ctx, filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry)) } - return nil + return unfinshiedSnapshotCnt, nil } func generateSnapshots(scrDir, dest string) error { @@ -198,6 +197,7 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, snapshotCheckPoints = append(snapshotCheckPoints, snapshotDate) snapshotDate = snapshotDate.AddDate(0, 0, -1**snapshotInterval) } + homeDirname, err := os.UserHomeDir() if err != nil { return err @@ -212,6 +212,7 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, dir: levelDbPath, } store.Initialize(config, "") + unfinishedSnapshotCnt := len(snapshotCheckPoints) - 1 changeLogPath := filer.SystemLogDir var processEntry func(entry *filer_pb.Entry, isLast bool) error processEntry = func(entry *filer_pb.Entry, isLast bool) error { @@ -234,10 +235,25 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, return err } idx = idx + 4 + logEntrySize - err = processMetaDataEvents(store, logEntry.Data, snapshotCheckPoints, homeDirname, *snapshotPath) + unfinishedSnapshotCnt, err = processMetaDataEvents(store, logEntry.Data, unfinishedSnapshotCnt, snapshotCheckPoints, homeDirname, *snapshotPath) + if err != nil { + return err + } + } + // edge case + // there might be unfinished snapshot left over in the duration gaps. + // process meta event only triggers snapshots when there are event after the snapshot time + for unfinishedSnapshotCnt >= 0 { + generatePath := filepath.Join(homeDirname, *snapshotPath, snapshotCheckPoints[unfinishedSnapshotCnt].Format(DateFormat)) + err = CreateIfNotExists(generatePath, 0755) + if err != nil { + return err + } + err = generateSnapshots(levelDbPath, generatePath) if err != nil { return err } + unfinishedSnapshotCnt-- } return nil From 97ea20b5b539dca29fbc08139beed331c01c92af Mon Sep 17 00:00:00 2001 From: root Date: Sun, 18 Dec 2022 20:44:12 -0800 Subject: [PATCH 09/11] add compute snapshot requirements --- .../shell/command_fs_meta_snapshots_create.go | 185 +++++++++++++----- 1 file changed, 135 insertions(+), 50 deletions(-) diff --git a/weed/shell/command_fs_meta_snapshots_create.go b/weed/shell/command_fs_meta_snapshots_create.go index e3f95f3a6..800c87287 100644 --- a/weed/shell/command_fs_meta_snapshots_create.go +++ b/weed/shell/command_fs_meta_snapshots_create.go @@ -2,6 +2,7 @@ package shell import ( "context" + "errors" "flag" "fmt" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -13,11 +14,14 @@ import ( "io" "os" "path/filepath" + "sort" + "strings" "time" ) -const LevelDbPath = "/tmp/snapshots.db" +const LevelDbPath = "tmp/snapshots.db" const DateFormat = "2006-01-02" +const SnapshotDirPostFix = "-snapshot" func init() { Commands = append(Commands, &commandFsMetaSnapshotsCreate{}) @@ -37,7 +41,6 @@ type SnapshotConfig struct { func (c SnapshotConfig) GetString(key string) string { return c.dir } - func (c SnapshotConfig) GetBool(key string) bool { panic("implement me") } @@ -57,9 +60,10 @@ func (c SnapshotConfig) SetDefault(key string, value interface{}) { func (c *commandFsMetaSnapshotsCreate) Help() string { return `create snapshots of meta data from given time range. - fs.meta.snapshots.create -snapshot-interval=7 -snapshot-cnt=3 -path=/your/path + fs.meta.snapshots.create -interval-days=7 -count=3 -path=/your/path // fs.meta.snapshots.create will generate desired number of snapshots with desired duration interval from yesterday the generated files will be saved from input path. // These snapshot maybe later used to backup the system to certain timestamp. + // path input is relative to home directory. ` } @@ -71,8 +75,8 @@ func processMetaDataEvents(store *filer_leveldb.LevelDBStore, data []byte, unfin } eventTime := event.TsNs for unfinshiedSnapshotCnt >= 0 && time.Unix(0, eventTime).After(snapshotCheckPoints[unfinshiedSnapshotCnt]) { - snapshotPath := filepath.Join(homeDir, snapshotPath, snapshotCheckPoints[unfinshiedSnapshotCnt].Format(DateFormat)) - err = CreateIfNotExists(snapshotPath, 0755) + snapshotPath := filepath.Join(homeDir, snapshotPath, snapshotCheckPoints[unfinshiedSnapshotCnt].Format(DateFormat)+SnapshotDirPostFix) + err = createIfNotExists(snapshotPath, 0755) if err != nil { return unfinshiedSnapshotCnt, err } @@ -121,14 +125,14 @@ func generateSnapshots(scrDir, dest string) error { switch fileInfo.Mode() & os.ModeType { case os.ModeDir: - if err := CreateIfNotExists(destPath, 0755); err != nil { + if err := createIfNotExists(destPath, 0755); err != nil { return err } if err := generateSnapshots(sourcePath, destPath); err != nil { return err } default: - if err := Copy(sourcePath, destPath); err != nil { + if err := copy(sourcePath, destPath); err != nil { return err } } @@ -136,7 +140,7 @@ func generateSnapshots(scrDir, dest string) error { return nil } -func Copy(srcFile, dstFile string) error { +func copy(srcFile, dstFile string) error { out, err := os.Create(dstFile) if err != nil { return err @@ -158,16 +162,15 @@ func Copy(srcFile, dstFile string) error { return nil } -func Exists(filePath string) bool { +func exists(filePath string) bool { if _, err := os.Stat(filePath); os.IsNotExist(err) { return false } - return true } -func CreateIfNotExists(dir string, perm os.FileMode) error { - if Exists(dir) { +func createIfNotExists(dir string, perm os.FileMode) error { + if exists(dir) { return nil } @@ -178,41 +181,114 @@ func CreateIfNotExists(dir string, perm os.FileMode) error { return nil } -func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - fsMetaSnapshotsCreateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) - snapshotPath := fsMetaSnapshotsCreateCommand.String("path", "", "the path to store generated snapshot files") - snapshotCnt := fsMetaSnapshotsCreateCommand.Int("snapshot-cnt", 3, "number of snapshots generated") - snapshotInterval := fsMetaSnapshotsCreateCommand.Int("snapshot-interval", 7, "the duration interval between each generated snapshot") - if err = fsMetaSnapshotsCreateCommand.Parse(args); err != nil { - return err +func computeRequirementsFromDirectory(previousSnapshots []os.DirEntry, homeDirectory string, snapshotPath string, count int, durationDays int) (snapshotsToRemove []string, snapshotsToGenerate []time.Time, err error) { + lastSnapshotDate, err := time.Parse(DateFormat, previousSnapshots[len(previousSnapshots)-1].Name()[:len(DateFormat)]) + if err != nil { + return snapshotsToRemove, snapshotsToGenerate, err } + yesterday := time.Now().Add(-time.Hour * 24) + yesterdayStr := yesterday.Format(DateFormat) + // ensure snapshot start at yesterday 00:00 + yesterday, err = time.Parse(DateFormat, yesterdayStr) + if err != nil { + return snapshotsToRemove, snapshotsToGenerate, err + } + gapDays := int(yesterday.Sub(lastSnapshotDate).Hours() / 24) + // gap too small no snapshot will be generated + if gapDays < durationDays { + return snapshotsToRemove, snapshotsToGenerate, errors.New(fmt.Sprintf("last snapshot was generated at %v no need to generate new snapshots", lastSnapshotDate.Format(DateFormat))) + } else if gapDays > durationDays*count { + // gap too large generate from yesterday + // and remove all previous snapshots + _, snapshotsToGenerate, err = computeRequirementsFromEmpty(homeDirectory, count, durationDays) + for _, file := range previousSnapshots { + snapshotsToRemove = append(snapshotsToRemove, filepath.Join(homeDirectory, snapshotPath, file.Name())) + } + return + } + snapshotDate := lastSnapshotDate.AddDate(0, 0, 1*durationDays) + for snapshotDate.Before(yesterday) || snapshotDate.Equal(yesterday) { + snapshotsToGenerate = append(snapshotsToGenerate, snapshotDate) + snapshotDate = lastSnapshotDate.AddDate(0, 0, 1*durationDays) + } + totalCount := len(previousSnapshots) + len(snapshotsToGenerate) + toRemoveIdx := 0 + for toRemoveIdx < len(previousSnapshots) && totalCount-toRemoveIdx > count { + snapshotsToRemove = append(snapshotsToRemove, filepath.Join(homeDirectory, snapshotPath, previousSnapshots[toRemoveIdx].Name())) + toRemoveIdx += 1 + } + return +} + +func computeRequirementsFromEmpty(homeDirectory string, count int, durationDays int) (snapshotsToRemove []string, snapshotsToGenerate []time.Time, err error) { yesterday := time.Now().Add(-time.Hour * 24).Format(DateFormat) // ensure snapshot start at yesterday 00:00 snapshotDate, err := time.Parse(DateFormat, yesterday) if err != nil { - return err + return snapshotsToRemove, snapshotsToGenerate, err } - var snapshotCheckPoints []time.Time - for i := 0; i < *snapshotCnt; i++ { - snapshotCheckPoints = append(snapshotCheckPoints, snapshotDate) - snapshotDate = snapshotDate.AddDate(0, 0, -1**snapshotInterval) + for i := 0; i < count; i++ { + snapshotsToGenerate = append(snapshotsToGenerate, snapshotDate) + snapshotDate = snapshotDate.AddDate(0, 0, -1*durationDays) } + return snapshotsToRemove, snapshotsToGenerate, nil +} - homeDirname, err := os.UserHomeDir() - if err != nil { - return err +// compute number of snapshot need to be generated and number of snapshots to remove from give directory. +func computeRequirements(homeDirectory string, snapshotPath string, count int, durationDays int) (snapshotsToRemove []string, snapshotsToGenerate []time.Time, err error) { + snapshotDirectory := filepath.Join(homeDirectory, snapshotPath) + files, _ := os.ReadDir(snapshotDirectory) + if len(files) == 0 { + return computeRequirementsFromEmpty(homeDirectory, count, durationDays) } - levelDbPath := homeDirname + LevelDbPath + // sort files by name + sort.Slice(files, func(i, j int) bool { + return files[i].Name() < files[j].Name() + }) + // filter for snapshots file only + var prevSnapshotFiles []os.DirEntry + for _, file := range files { + if strings.HasSuffix(file.Name(), SnapshotDirPostFix) { + prevSnapshotFiles = append(prevSnapshotFiles, file) + } + } + return computeRequirementsFromDirectory(prevSnapshotFiles, homeDirectory, snapshotPath, count, durationDays) +} + +func setupLevelDb(levelDbPath string) (store *filer_leveldb.LevelDBStore, err error) { err = os.RemoveAll(levelDbPath) if err != nil { - return err + return &filer_leveldb.LevelDBStore{}, err } - store := &filer_leveldb.LevelDBStore{} config := SnapshotConfig{ dir: levelDbPath, } store.Initialize(config, "") - unfinishedSnapshotCnt := len(snapshotCheckPoints) - 1 + return +} + +func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, _writer io.Writer) (err error) { + fsMetaSnapshotsCreateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + snapshotPath := fsMetaSnapshotsCreateCommand.String("path", "", "the path to store generated snapshot files") + count := fsMetaSnapshotsCreateCommand.Int("count", 3, "number of snapshots generated") + intervalDays := fsMetaSnapshotsCreateCommand.Int("interval-days", 7, "the duration interval between each generated snapshot") + if err = fsMetaSnapshotsCreateCommand.Parse(args); err != nil { + return err + } + homeDirname, err := os.UserHomeDir() + if err != nil { + return err + } + snapshotsToRemove, snapshotsToGenerate, err := computeRequirements(homeDirname, *snapshotPath, *count, *intervalDays) + if err != nil { + return err + } + levelDbPath := filepath.Join(homeDirname, LevelDbPath) + store, err := setupLevelDb(levelDbPath) + if err != nil { + return err + } + unfinishedSnapshotCnt := len(snapshotsToGenerate) - 1 changeLogPath := filer.SystemLogDir var processEntry func(entry *filer_pb.Entry, isLast bool) error processEntry = func(entry *filer_pb.Entry, isLast bool) error { @@ -235,30 +311,39 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, return err } idx = idx + 4 + logEntrySize - unfinishedSnapshotCnt, err = processMetaDataEvents(store, logEntry.Data, unfinishedSnapshotCnt, snapshotCheckPoints, homeDirname, *snapshotPath) + unfinishedSnapshotCnt, err = processMetaDataEvents(store, logEntry.Data, unfinishedSnapshotCnt, snapshotsToGenerate, homeDirname, *snapshotPath) if err != nil { return err } } - // edge case - // there might be unfinished snapshot left over in the duration gaps. - // process meta event only triggers snapshots when there are event after the snapshot time - for unfinishedSnapshotCnt >= 0 { - generatePath := filepath.Join(homeDirname, *snapshotPath, snapshotCheckPoints[unfinishedSnapshotCnt].Format(DateFormat)) - err = CreateIfNotExists(generatePath, 0755) - if err != nil { - return err - } - err = generateSnapshots(levelDbPath, generatePath) - if err != nil { - return err - } - unfinishedSnapshotCnt-- - } - - return nil + return err } err = filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(changeLogPath), "", processEntry) - return err + if err != nil { + return err + } + // edge case + // there might be unfinished snapshot left over in the duration gaps. + // process meta event only triggers snapshots when there are event after the snapshot time + for unfinishedSnapshotCnt >= 0 { + generatePath := filepath.Join(homeDirname, *snapshotPath, snapshotsToGenerate[unfinishedSnapshotCnt].Format(DateFormat)) + err = createIfNotExists(generatePath, 0755) + if err != nil { + return err + } + err = generateSnapshots(levelDbPath, generatePath) + if err != nil { + return err + } + unfinishedSnapshotCnt-- + } + // remove previous snapshot if needed. + for _, snapshot := range snapshotsToRemove { + err = os.RemoveAll(snapshot) + if err != nil { + return err + } + } + return nil } From b7394a48f07960cff9a5547f702213d4596afbc5 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 27 Dec 2022 16:12:28 -0800 Subject: [PATCH 10/11] fix --- .../shell/command_fs_meta_snapshots_create.go | 193 +++++++----------- ...mand_fs_meta_snapshots_create_scheduler.go | 86 ++++++++ ...fs_meta_snapshots_create_scheduler_test.go | 64 ++++++ 3 files changed, 221 insertions(+), 122 deletions(-) create mode 100644 weed/shell/command_fs_meta_snapshots_create_scheduler.go create mode 100644 weed/shell/command_fs_meta_snapshots_create_scheduler_test.go diff --git a/weed/shell/command_fs_meta_snapshots_create.go b/weed/shell/command_fs_meta_snapshots_create.go index 800c87287..af915f9cc 100644 --- a/weed/shell/command_fs_meta_snapshots_create.go +++ b/weed/shell/command_fs_meta_snapshots_create.go @@ -2,7 +2,6 @@ package shell import ( "context" - "errors" "flag" "fmt" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -15,7 +14,6 @@ import ( "os" "path/filepath" "sort" - "strings" "time" ) @@ -67,46 +65,71 @@ func (c *commandFsMetaSnapshotsCreate) Help() string { ` } -func processMetaDataEvents(store *filer_leveldb.LevelDBStore, data []byte, unfinshiedSnapshotCnt int, snapshotCheckPoints []time.Time, homeDir string, snapshotPath string) (todoCnt int, err error) { +func processMetaDataEvents(store *filer_leveldb.LevelDBStore, data []byte, count int, snapshotCheckPoints []time.Time, homeDir string, snapshotPath string) (SnapshotCount int, err error) { var event filer_pb.SubscribeMetadataResponse err = proto.Unmarshal(data, &event) if err != nil { - return unfinshiedSnapshotCnt, err + return count, err } eventTime := event.TsNs - for unfinshiedSnapshotCnt >= 0 && time.Unix(0, eventTime).After(snapshotCheckPoints[unfinshiedSnapshotCnt]) { - snapshotPath := filepath.Join(homeDir, snapshotPath, snapshotCheckPoints[unfinshiedSnapshotCnt].Format(DateFormat)+SnapshotDirPostFix) + for count < len(snapshotCheckPoints) && time.Unix(0, eventTime).After(snapshotCheckPoints[count].Add(-time.Microsecond)) { + snapshotPath := filepath.Join(homeDir, snapshotPath, snapshotCheckPoints[count].Format(DateFormat)+SnapshotDirPostFix) err = createIfNotExists(snapshotPath, 0755) if err != nil { - return unfinshiedSnapshotCnt, err + return count, err } err = generateSnapshots(homeDir+LevelDbPath, snapshotPath) if err != nil { - return unfinshiedSnapshotCnt, err + return count, err } - unfinshiedSnapshotCnt-- + count++ } - if unfinshiedSnapshotCnt < 0 { - return unfinshiedSnapshotCnt, nil + if count < 0 { + return count, nil } ctx := context.Background() if filer_pb.IsEmpty(&event) { - return unfinshiedSnapshotCnt, nil + return count, nil } else if filer_pb.IsCreate(&event) { entry := filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry) - return unfinshiedSnapshotCnt, store.InsertEntry(ctx, entry) + return count, store.InsertEntry(ctx, entry) } else if filer_pb.IsDelete(&event) { - return unfinshiedSnapshotCnt, store.DeleteEntry(ctx, util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)) + return count, store.DeleteEntry(ctx, util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)) } else if filer_pb.IsUpdate(&event) { entry := filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry) - return unfinshiedSnapshotCnt, store.UpdateEntry(ctx, entry) + return count, store.UpdateEntry(ctx, entry) } else { if err := store.DeleteEntry(ctx, util.FullPath(event.Directory).Child(event.EventNotification.OldEntry.Name)); err != nil { - return unfinshiedSnapshotCnt, err + return count, err } - return unfinshiedSnapshotCnt, store.InsertEntry(ctx, filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry)) + return count, store.InsertEntry(ctx, filer.FromPbEntry(event.EventNotification.NewParentPath, event.EventNotification.NewEntry)) } - return unfinshiedSnapshotCnt, nil + return count, nil +} + +func processEntryLog(entry *filer_pb.Entry, commandEnv *CommandEnv, snapshotCount int, store *filer_leveldb.LevelDBStore, snapshotsToGenerate []time.Time, homeDirname string, snapshotPath string) (count int, err error) { + totalSize := filer.FileSize(entry) + buf := mem.Allocate(int(totalSize)) + if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + return filer.ReadAll(buf, commandEnv.MasterClient, entry.GetChunks()) + }); err != nil && err != filer_pb.ErrNotFound { + return snapshotCount, err + } + idx := uint32(0) + for idx < uint32(totalSize) { + logEntrySize := util.BytesToUint32(buf[idx : idx+4]) + var logEntry filer_pb.LogEntry + err = proto.Unmarshal(buf[idx+4:idx+4+logEntrySize], &logEntry) + if err != nil { + return snapshotCount, err + } + idx = idx + 4 + logEntrySize + snapshotCount, err = processMetaDataEvents(store, logEntry.Data, snapshotCount, snapshotsToGenerate, homeDirname, snapshotPath) + if err != nil { + return snapshotCount, err + } + } + return snapshotCount, err } func generateSnapshots(scrDir, dest string) error { @@ -181,85 +204,18 @@ func createIfNotExists(dir string, perm os.FileMode) error { return nil } -func computeRequirementsFromDirectory(previousSnapshots []os.DirEntry, homeDirectory string, snapshotPath string, count int, durationDays int) (snapshotsToRemove []string, snapshotsToGenerate []time.Time, err error) { - lastSnapshotDate, err := time.Parse(DateFormat, previousSnapshots[len(previousSnapshots)-1].Name()[:len(DateFormat)]) - if err != nil { - return snapshotsToRemove, snapshotsToGenerate, err - } - yesterday := time.Now().Add(-time.Hour * 24) - yesterdayStr := yesterday.Format(DateFormat) - // ensure snapshot start at yesterday 00:00 - yesterday, err = time.Parse(DateFormat, yesterdayStr) - if err != nil { - return snapshotsToRemove, snapshotsToGenerate, err - } - gapDays := int(yesterday.Sub(lastSnapshotDate).Hours() / 24) - // gap too small no snapshot will be generated - if gapDays < durationDays { - return snapshotsToRemove, snapshotsToGenerate, errors.New(fmt.Sprintf("last snapshot was generated at %v no need to generate new snapshots", lastSnapshotDate.Format(DateFormat))) - } else if gapDays > durationDays*count { - // gap too large generate from yesterday - // and remove all previous snapshots - _, snapshotsToGenerate, err = computeRequirementsFromEmpty(homeDirectory, count, durationDays) - for _, file := range previousSnapshots { - snapshotsToRemove = append(snapshotsToRemove, filepath.Join(homeDirectory, snapshotPath, file.Name())) - } - return - } - snapshotDate := lastSnapshotDate.AddDate(0, 0, 1*durationDays) - for snapshotDate.Before(yesterday) || snapshotDate.Equal(yesterday) { - snapshotsToGenerate = append(snapshotsToGenerate, snapshotDate) - snapshotDate = lastSnapshotDate.AddDate(0, 0, 1*durationDays) - } - totalCount := len(previousSnapshots) + len(snapshotsToGenerate) - toRemoveIdx := 0 - for toRemoveIdx < len(previousSnapshots) && totalCount-toRemoveIdx > count { - snapshotsToRemove = append(snapshotsToRemove, filepath.Join(homeDirectory, snapshotPath, previousSnapshots[toRemoveIdx].Name())) - toRemoveIdx += 1 - } - return -} - -func computeRequirementsFromEmpty(homeDirectory string, count int, durationDays int) (snapshotsToRemove []string, snapshotsToGenerate []time.Time, err error) { - yesterday := time.Now().Add(-time.Hour * 24).Format(DateFormat) - // ensure snapshot start at yesterday 00:00 - snapshotDate, err := time.Parse(DateFormat, yesterday) - if err != nil { - return snapshotsToRemove, snapshotsToGenerate, err - } - for i := 0; i < count; i++ { - snapshotsToGenerate = append(snapshotsToGenerate, snapshotDate) - snapshotDate = snapshotDate.AddDate(0, 0, -1*durationDays) - } - return snapshotsToRemove, snapshotsToGenerate, nil -} - -// compute number of snapshot need to be generated and number of snapshots to remove from give directory. -func computeRequirements(homeDirectory string, snapshotPath string, count int, durationDays int) (snapshotsToRemove []string, snapshotsToGenerate []time.Time, err error) { - snapshotDirectory := filepath.Join(homeDirectory, snapshotPath) - files, _ := os.ReadDir(snapshotDirectory) - if len(files) == 0 { - return computeRequirementsFromEmpty(homeDirectory, count, durationDays) - } - // sort files by name - sort.Slice(files, func(i, j int) bool { - return files[i].Name() < files[j].Name() - }) - // filter for snapshots file only - var prevSnapshotFiles []os.DirEntry - for _, file := range files { - if strings.HasSuffix(file.Name(), SnapshotDirPostFix) { - prevSnapshotFiles = append(prevSnapshotFiles, file) - } - } - return computeRequirementsFromDirectory(prevSnapshotFiles, homeDirectory, snapshotPath, count, durationDays) -} - -func setupLevelDb(levelDbPath string) (store *filer_leveldb.LevelDBStore, err error) { +func setupLevelDb(levelDbPath string, levelDbBootstrapPath string) (store *filer_leveldb.LevelDBStore, err error) { err = os.RemoveAll(levelDbPath) if err != nil { return &filer_leveldb.LevelDBStore{}, err } + if len(levelDbBootstrapPath) != 0 { + // copy the latest snapshot as starting point + err = generateSnapshots(levelDbBootstrapPath, levelDbPath) + if err != nil { + return + } + } config := SnapshotConfig{ dir: levelDbPath, } @@ -279,43 +235,36 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, if err != nil { return err } - snapshotsToRemove, snapshotsToGenerate, err := computeRequirements(homeDirname, *snapshotPath, *count, *intervalDays) + snapshotsToRemove, snapshotsToGenerate, levelDbBootstrapPath, err := computeRequirements(homeDirname, *snapshotPath, *count, *intervalDays) if err != nil { return err } levelDbPath := filepath.Join(homeDirname, LevelDbPath) - store, err := setupLevelDb(levelDbPath) + store, err := setupLevelDb(levelDbPath, filepath.Join(homeDirname, *snapshotPath, levelDbBootstrapPath)) if err != nil { return err } - unfinishedSnapshotCnt := len(snapshotsToGenerate) - 1 + // sort to make sure we are processing ascending list of snapshots to generate + sort.Slice(snapshotsToGenerate, func(i, j int) bool { + return snapshotsToGenerate[i].Before(snapshotsToGenerate[j]) + }) changeLogPath := filer.SystemLogDir var processEntry func(entry *filer_pb.Entry, isLast bool) error + var levelDbBootstrapDate string + if levelDbBootstrapPath != "" { + levelDbBootstrapDate = levelDbBootstrapPath[:len(DateFormat)] + } + snapshotCount := 0 processEntry = func(entry *filer_pb.Entry, isLast bool) error { if entry.IsDirectory { - return filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(changeLogPath+"/"+entry.Name), "", processEntry) - } - totalSize := filer.FileSize(entry) - buf := mem.Allocate(int(totalSize)) - if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - return filer.ReadAll(buf, commandEnv.MasterClient, entry.GetChunks()) - }); err != nil && err != filer_pb.ErrNotFound { - return err - } - idx := uint32(0) - for idx < uint32(totalSize) { - logEntrySize := util.BytesToUint32(buf[idx : idx+4]) - var logEntry filer_pb.LogEntry - err = proto.Unmarshal(buf[idx+4:idx+4+logEntrySize], &logEntry) - if err != nil { - return err - } - idx = idx + 4 + logEntrySize - unfinishedSnapshotCnt, err = processMetaDataEvents(store, logEntry.Data, unfinishedSnapshotCnt, snapshotsToGenerate, homeDirname, *snapshotPath) - if err != nil { - return err + // skip logs prior to the latest previous snapshot + if entry.GetName() <= levelDbBootstrapDate { + println(entry.GetName()) + return nil } + return filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(changeLogPath+"/"+entry.Name), "", processEntry) } + snapshotCount, err = processEntryLog(entry, commandEnv, snapshotCount, store, snapshotsToGenerate, homeDirname, *snapshotPath) return err } @@ -325,9 +274,9 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, } // edge case // there might be unfinished snapshot left over in the duration gaps. - // process meta event only triggers snapshots when there are event after the snapshot time - for unfinishedSnapshotCnt >= 0 { - generatePath := filepath.Join(homeDirname, *snapshotPath, snapshotsToGenerate[unfinishedSnapshotCnt].Format(DateFormat)) + // process meta event only triggers snapshots when there are event after the snapshot time. + for snapshotCount < len(snapshotsToGenerate) { + generatePath := filepath.Join(homeDirname, *snapshotPath, snapshotsToGenerate[snapshotCount].Format(DateFormat)) err = createIfNotExists(generatePath, 0755) if err != nil { return err @@ -336,9 +285,9 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, if err != nil { return err } - unfinishedSnapshotCnt-- + snapshotCount++ } - // remove previous snapshot if needed. + // remove previous snapshots. for _, snapshot := range snapshotsToRemove { err = os.RemoveAll(snapshot) if err != nil { diff --git a/weed/shell/command_fs_meta_snapshots_create_scheduler.go b/weed/shell/command_fs_meta_snapshots_create_scheduler.go new file mode 100644 index 000000000..962445007 --- /dev/null +++ b/weed/shell/command_fs_meta_snapshots_create_scheduler.go @@ -0,0 +1,86 @@ +package shell + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +func computeRequirementsFromDirectory(previousSnapshots []string, homeDirectory string, snapshotPath string, count int, durationDays int, targetDate time.Time) (snapshotsToRemove []string, snapshotsToGenerate []time.Time, levelDbBootstrapPath string, err error) { + levelDbBootstrapPath = previousSnapshots[len(previousSnapshots)-1] + lastSnapshotDate, err := time.Parse(DateFormat, levelDbBootstrapPath[:len(DateFormat)]) + if err != nil { + return + } + if err != nil { + return + } + // the snapshots cover the last nanosecond of the current date + lastSnapshotDate = lastSnapshotDate.AddDate(0, 0, 1).Add(-1 * time.Nanosecond) + gapDuration := targetDate.Sub(lastSnapshotDate) + oneSnapshotInterval := 24 * time.Hour * time.Duration(durationDays) + totalSnapshotsInterval := 24 * time.Hour * time.Duration(durationDays*count) + // gap too small no snapshot will be generated + if gapDuration < oneSnapshotInterval { + return snapshotsToRemove, snapshotsToGenerate, levelDbBootstrapPath, errors.New(fmt.Sprintf("last snapshot was generated at %v no need to generate new snapshots", lastSnapshotDate.Format(DateFormat))) + } else if gapDuration > totalSnapshotsInterval { + // gap too large generate from targetDate + // and remove all previous snapshots + _, snapshotsToGenerate, _, err = computeRequirementsFromEmpty(count, durationDays, targetDate) + for _, file := range previousSnapshots { + snapshotsToRemove = append(snapshotsToRemove, filepath.Join(homeDirectory, snapshotPath, file)) + } + return + } + snapshotDate := lastSnapshotDate.AddDate(0, 0, durationDays) + for snapshotDate.Before(targetDate) || snapshotDate.Equal(targetDate) { + snapshotsToGenerate = append(snapshotsToGenerate, snapshotDate) + snapshotDate = snapshotDate.AddDate(0, 0, durationDays) + } + totalCount := len(previousSnapshots) + len(snapshotsToGenerate) + toRemoveIdx := 0 + for toRemoveIdx < len(previousSnapshots) && totalCount-toRemoveIdx > count { + snapshotsToRemove = append(snapshotsToRemove, filepath.Join(homeDirectory, snapshotPath, previousSnapshots[toRemoveIdx])) + toRemoveIdx += 1 + } + return +} + +func computeRequirementsFromEmpty(count int, durationDays int, targetDate time.Time) (snapshotsToRemove []string, snapshotsToGenerate []time.Time, levelDbBootstrapPath string, err error) { + snapshotDate := targetDate + for i := 0; i < count; i++ { + snapshotsToGenerate = append(snapshotsToGenerate, snapshotDate) + snapshotDate = snapshotDate.AddDate(0, 0, -1*durationDays) + } + return snapshotsToRemove, snapshotsToGenerate, "", nil +} + +// compute number of snapshot need to be generated and number of snapshots to remove from give directory. +func computeRequirements(homeDirectory string, snapshotPath string, count int, durationDays int) (snapshotsToRemove []string, snapshotsToGenerate []time.Time, levelDbBootstrapPath string, err error) { + snapshotDirectory := filepath.Join(homeDirectory, snapshotPath) + files, _ := os.ReadDir(snapshotDirectory) + // sort files by name + sort.Slice(files, func(i, j int) bool { + return files[i].Name() < files[j].Name() + }) + // filter for snapshots file name only + var prevSnapshotFiles []string + for _, file := range files { + if strings.HasSuffix(file.Name(), SnapshotDirPostFix) { + prevSnapshotFiles = append(prevSnapshotFiles, file.Name()) + } + } + curDate := time.Now() + curDateStr := curDate.Format(DateFormat) + // ensure snapshot start at today 00:00 - 1ns + today, err := time.Parse(DateFormat, curDateStr) + targetDate := today.Add(-1 * time.Nanosecond) + if len(prevSnapshotFiles) == 0 { + return computeRequirementsFromEmpty(count, durationDays, targetDate) + } + return computeRequirementsFromDirectory(prevSnapshotFiles, homeDirectory, snapshotPath, count, durationDays, targetDate) +} diff --git a/weed/shell/command_fs_meta_snapshots_create_scheduler_test.go b/weed/shell/command_fs_meta_snapshots_create_scheduler_test.go new file mode 100644 index 000000000..9a2159aa6 --- /dev/null +++ b/weed/shell/command_fs_meta_snapshots_create_scheduler_test.go @@ -0,0 +1,64 @@ +package shell + +import ( + "github.com/stretchr/testify/assert" + "path/filepath" + "reflect" + "testing" + "time" +) + +func TestComputeRequirementsFromDirectory(t *testing.T) { + homeDir := "home" + snapDir := "test" + // case1: we have previous snapshots, target date is relative close to the latest snapshot date, we will use previous snapshots to generate future snapshots and remove some previous snapshots. + var prevSnapshots []string + prevSnapshots = append(prevSnapshots, "2022-01-01") + prevSnapshots = append(prevSnapshots, "2022-02-01") + prevSnapshots = append(prevSnapshots, "2022-03-01") + targetDate, _ := time.Parse(DateFormat, "2022-04-01") + targetDate = targetDate.Add(-1 * time.Nanosecond) + snapshotsToRemove, snapshotsToGenerate, levelDbBootstrapPath, err := computeRequirementsFromDirectory(prevSnapshots, homeDir, snapDir, 3, 15, targetDate) + assert.Nil(t, err) + assert.Equal(t, levelDbBootstrapPath, "2022-03-01", "latest previous snapshot should be 2022-03-01") + expectedToRemoveSnapshots := []string{filepath.Join(homeDir, snapDir, "2022-01-01"), filepath.Join(homeDir, snapDir, "2022-02-01")} + assert.True(t, reflect.DeepEqual(snapshotsToRemove, expectedToRemoveSnapshots)) + expectedSnapshotsGenerationDate := []string{"2022-03-16", "2022-03-31"} + for i, date := range expectedSnapshotsGenerationDate { + assert.Equal(t, snapshotsToGenerate[i].Format(DateFormat), date) + } + + // case2: we have previous snapshots, target date is too close to the latest snapshot date, no change will happen. + targetDate, _ = time.Parse(DateFormat, "2022-03-02") + snapshotsToRemove, snapshotsToGenerate, levelDbBootstrapPath, err = computeRequirementsFromDirectory(prevSnapshots, "home", "test", 3, 15, targetDate) + assert.NotNil(t, err) + assert.Containsf(t, err.Error(), "no need to generate new snapshots", "expected error containing %q, got %s", "no need to generate new snapshots", err) + assert.Empty(t, snapshotsToRemove) + assert.Empty(t, snapshotsToGenerate) + assert.Equal(t, levelDbBootstrapPath, "2022-03-01", "latest previous snapshot should be 2022-03-01") + + // case3: we have previous snapshots, target date is too far out to the latest snapshot date, all previous snapshots will be removed, snapshots will be generated going backward starting on target date. + targetDate, _ = time.Parse(DateFormat, "2022-12-01") + targetDate = targetDate.Add(-1 * time.Nanosecond) + snapshotsToRemove, snapshotsToGenerate, levelDbBootstrapPath, err = computeRequirementsFromDirectory(prevSnapshots, "home", "test", 3, 15, targetDate) + assert.Nil(t, err) + expectedToRemoveSnapshots = []string{filepath.Join(homeDir, snapDir, "2022-01-01"), filepath.Join(homeDir, snapDir, "2022-02-01"), filepath.Join(homeDir, snapDir, "2022-03-01")} + assert.True(t, reflect.DeepEqual(snapshotsToRemove, expectedToRemoveSnapshots)) + // we still need to skip all logs prior to 2022-03-02 + assert.Equal(t, levelDbBootstrapPath, "2022-03-01", "latest previous snapshot should be 2022-03-01") + expectedSnapshotsGenerationDate = []string{"2022-11-30", "2022-11-15", "2022-10-31"} + for i, date := range expectedSnapshotsGenerationDate { + assert.Equal(t, snapshotsToGenerate[i].Format(DateFormat), date) + } + + // case4: the target date is exactly n snapshots away from previous snapshot date + targetDate, _ = time.Parse(DateFormat, "2022-03-04") + targetDate = targetDate.Add(-1 * time.Nanosecond) + snapshotsToRemove, snapshotsToGenerate, levelDbBootstrapPath, err = computeRequirementsFromDirectory(prevSnapshots, "home", "test", 3, 1, targetDate) + expectedToRemoveSnapshots = []string{filepath.Join(homeDir, snapDir, "2022-01-01"), filepath.Join(homeDir, snapDir, "2022-02-01")} + assert.True(t, reflect.DeepEqual(snapshotsToRemove, expectedToRemoveSnapshots)) + expectedSnapshotsGenerationDate = []string{"2022-03-02", "2022-03-03"} + for i, date := range expectedSnapshotsGenerationDate { + assert.Equal(t, snapshotsToGenerate[i].Format(DateFormat), date) + } +} From b51d94a792765a209151a6031d48878de7fef40f Mon Sep 17 00:00:00 2001 From: root Date: Tue, 27 Dec 2022 16:49:10 -0800 Subject: [PATCH 11/11] more fix --- weed/shell/command_fs_meta_snapshots_create.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/weed/shell/command_fs_meta_snapshots_create.go b/weed/shell/command_fs_meta_snapshots_create.go index af915f9cc..57eaef25a 100644 --- a/weed/shell/command_fs_meta_snapshots_create.go +++ b/weed/shell/command_fs_meta_snapshots_create.go @@ -17,7 +17,7 @@ import ( "time" ) -const LevelDbPath = "tmp/snapshots.db" +const LevelDbPath = "snapshots.db" const DateFormat = "2006-01-02" const SnapshotDirPostFix = "-snapshot" @@ -78,7 +78,7 @@ func processMetaDataEvents(store *filer_leveldb.LevelDBStore, data []byte, count if err != nil { return count, err } - err = generateSnapshots(homeDir+LevelDbPath, snapshotPath) + err = generateSnapshots(filepath.Join(homeDir, LevelDbPath), snapshotPath) if err != nil { return count, err } @@ -134,6 +134,9 @@ func processEntryLog(entry *filer_pb.Entry, commandEnv *CommandEnv, snapshotCoun func generateSnapshots(scrDir, dest string) error { entries, err := os.ReadDir(scrDir) + if err := createIfNotExists(dest, 0755); err != nil { + return err + } if err != nil { return err } @@ -205,9 +208,10 @@ func createIfNotExists(dir string, perm os.FileMode) error { } func setupLevelDb(levelDbPath string, levelDbBootstrapPath string) (store *filer_leveldb.LevelDBStore, err error) { + store = &filer_leveldb.LevelDBStore{} err = os.RemoveAll(levelDbPath) if err != nil { - return &filer_leveldb.LevelDBStore{}, err + return } if len(levelDbBootstrapPath) != 0 { // copy the latest snapshot as starting point @@ -219,6 +223,7 @@ func setupLevelDb(levelDbPath string, levelDbBootstrapPath string) (store *filer config := SnapshotConfig{ dir: levelDbPath, } + store.Initialize(config, "") return } @@ -258,8 +263,7 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, processEntry = func(entry *filer_pb.Entry, isLast bool) error { if entry.IsDirectory { // skip logs prior to the latest previous snapshot - if entry.GetName() <= levelDbBootstrapDate { - println(entry.GetName()) + if levelDbBootstrapDate != "" && entry.GetName() <= levelDbBootstrapDate { return nil } return filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(changeLogPath+"/"+entry.Name), "", processEntry) @@ -276,7 +280,7 @@ func (c *commandFsMetaSnapshotsCreate) Do(args []string, commandEnv *CommandEnv, // there might be unfinished snapshot left over in the duration gaps. // process meta event only triggers snapshots when there are event after the snapshot time. for snapshotCount < len(snapshotsToGenerate) { - generatePath := filepath.Join(homeDirname, *snapshotPath, snapshotsToGenerate[snapshotCount].Format(DateFormat)) + generatePath := filepath.Join(homeDirname, *snapshotPath, snapshotsToGenerate[snapshotCount].Format(DateFormat)+SnapshotDirPostFix) err = createIfNotExists(generatePath, 0755) if err != nil { return err