From 3a82f5ffad3122a3adeb9e652cee72420a2f6f97 Mon Sep 17 00:00:00 2001 From: chrislu Date: Fri, 12 Jul 2024 11:17:28 -0700 Subject: [PATCH] ensure metadata follow a specific folder fix https://github.com/seaweedfs/seaweedfs/issues/5774 --- .../load_test/load_test_meta_tail/load_test_meta_tail.go | 8 +++++++- weed/command/filer_backup.go | 8 +++++++- weed/command/filer_meta_backup.go | 7 ++++++- weed/command/filer_remote_gateway_buckets.go | 2 +- weed/command/filer_remote_sync_dir.go | 7 ++++++- weed/command/filer_sync.go | 7 ++++++- weed/mount/meta_cache/meta_cache_subscribe.go | 8 +++++++- 7 files changed, 40 insertions(+), 7 deletions(-) diff --git a/unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go b/unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go index eb08e5353..2eeb5d6f9 100644 --- a/unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go +++ b/unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go @@ -10,6 +10,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "strconv" + "strings" "time" ) @@ -79,12 +80,17 @@ func startGenerateMetadata() { func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) { + prefix := *dir + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + metadataFollowOption := &pb.MetadataFollowOption{ ClientName: "tail", ClientId: 0, ClientEpoch: 0, SelfSignature: 0, - PathPrefix: *dir, + PathPrefix: prefix, AdditionalPathPrefixes: nil, DirectoriesToWatch: nil, StartTsNs: 0, diff --git a/weed/command/filer_backup.go b/weed/command/filer_backup.go index 43b7c6256..a66c53f8e 100644 --- a/weed/command/filer_backup.go +++ b/weed/command/filer_backup.go @@ -9,6 +9,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" "regexp" + "strings" "time" ) @@ -148,12 +149,17 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti }() } + prefix := sourcePath + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + metadataFollowOption := &pb.MetadataFollowOption{ ClientName: "backup_" + dataSink.GetName(), ClientId: clientId, ClientEpoch: clientEpoch, SelfSignature: 0, - PathPrefix: sourcePath, + PathPrefix: prefix, AdditionalPathPrefixes: nil, DirectoriesToWatch: nil, StartTsNs: startFrom.UnixNano(), diff --git a/weed/command/filer_meta_backup.go b/weed/command/filer_meta_backup.go index c47ccbd0c..6ed8c4c64 100644 --- a/weed/command/filer_meta_backup.go +++ b/weed/command/filer_meta_backup.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/viper" "google.golang.org/grpc" "reflect" + "strings" "time" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -197,12 +198,16 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error { metaBackup.clientEpoch++ + prefix := *metaBackup.filerDirectory + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } metadataFollowOption := &pb.MetadataFollowOption{ ClientName: "meta_backup", ClientId: metaBackup.clientId, ClientEpoch: metaBackup.clientEpoch, SelfSignature: 0, - PathPrefix: *metaBackup.filerDirectory, + PathPrefix: prefix, AdditionalPathPrefixes: nil, DirectoriesToWatch: nil, StartTsNs: startTime.UnixNano(), diff --git a/weed/command/filer_remote_gateway_buckets.go b/weed/command/filer_remote_gateway_buckets.go index 8ca109282..f6fe9a99c 100644 --- a/weed/command/filer_remote_gateway_buckets.go +++ b/weed/command/filer_remote_gateway_buckets.go @@ -55,7 +55,7 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo ClientId: option.clientId, ClientEpoch: option.clientEpoch, SelfSignature: 0, - PathPrefix: option.bucketsDir, + PathPrefix: option.bucketsDir + "/", AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote}, DirectoriesToWatch: nil, StartTsNs: lastOffsetTs.UnixNano(), diff --git a/weed/command/filer_remote_sync_dir.go b/weed/command/filer_remote_sync_dir.go index 42dc9ff28..186523e45 100644 --- a/weed/command/filer_remote_sync_dir.go +++ b/weed/command/filer_remote_sync_dir.go @@ -64,12 +64,17 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour option.clientEpoch++ + prefix := mountedDir + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + metadataFollowOption := &pb.MetadataFollowOption{ ClientName: "filer.remote.sync", ClientId: option.clientId, ClientEpoch: option.clientEpoch, SelfSignature: 0, - PathPrefix: mountedDir, + PathPrefix: prefix, AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote}, DirectoriesToWatch: nil, StartTsNs: lastOffsetTs.UnixNano(), diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index 9ad76e31b..9d29b6fee 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -296,12 +296,17 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs) }) + prefix := sourcePath + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + metadataFollowOption := &pb.MetadataFollowOption{ ClientName: clientName, ClientId: clientId, ClientEpoch: clientEpoch, SelfSignature: targetFilerSignature, - PathPrefix: sourcePath, + PathPrefix: prefix, AdditionalPathPrefixes: nil, DirectoriesToWatch: nil, StartTsNs: sourceFilerOffsetTsNs, diff --git a/weed/mount/meta_cache/meta_cache_subscribe.go b/weed/mount/meta_cache/meta_cache_subscribe.go index 72da0ca64..d3bb27d08 100644 --- a/weed/mount/meta_cache/meta_cache_subscribe.go +++ b/weed/mount/meta_cache/meta_cache_subscribe.go @@ -7,6 +7,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" + "strings" ) func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64) error { @@ -57,12 +58,17 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil } + prefix := dir + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + metadataFollowOption := &pb.MetadataFollowOption{ ClientName: "mount", ClientId: selfSignature, ClientEpoch: 1, SelfSignature: selfSignature, - PathPrefix: dir, + PathPrefix: prefix, AdditionalPathPrefixes: nil, DirectoriesToWatch: nil, StartTsNs: lastTsNs,