Browse Source

ensure metadata follow a specific folder

fix https://github.com/seaweedfs/seaweedfs/issues/5774
pull/5783/head
chrislu 5 months ago
parent
commit
3a82f5ffad
  1. 8
      unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go
  2. 8
      weed/command/filer_backup.go
  3. 7
      weed/command/filer_meta_backup.go
  4. 2
      weed/command/filer_remote_gateway_buckets.go
  5. 7
      weed/command/filer_remote_sync_dir.go
  6. 7
      weed/command/filer_sync.go
  7. 8
      weed/mount/meta_cache/meta_cache_subscribe.go

8
unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go

@ -10,6 +10,7 @@ import (
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/insecure"
"strconv" "strconv"
"strings"
"time" "time"
) )
@ -79,12 +80,17 @@ func startGenerateMetadata() {
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) { func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
prefix := *dir
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{ metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "tail", ClientName: "tail",
ClientId: 0, ClientId: 0,
ClientEpoch: 0, ClientEpoch: 0,
SelfSignature: 0, SelfSignature: 0,
PathPrefix: *dir,
PathPrefix: prefix,
AdditionalPathPrefixes: nil, AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: 0, StartTsNs: 0,

8
weed/command/filer_backup.go

@ -9,6 +9,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc" "google.golang.org/grpc"
"regexp" "regexp"
"strings"
"time" "time"
) )
@ -148,12 +149,17 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
}() }()
} }
prefix := sourcePath
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{ metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "backup_" + dataSink.GetName(), ClientName: "backup_" + dataSink.GetName(),
ClientId: clientId, ClientId: clientId,
ClientEpoch: clientEpoch, ClientEpoch: clientEpoch,
SelfSignature: 0, SelfSignature: 0,
PathPrefix: sourcePath,
PathPrefix: prefix,
AdditionalPathPrefixes: nil, AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: startFrom.UnixNano(), StartTsNs: startFrom.UnixNano(),

7
weed/command/filer_meta_backup.go

@ -8,6 +8,7 @@ import (
"github.com/spf13/viper" "github.com/spf13/viper"
"google.golang.org/grpc" "google.golang.org/grpc"
"reflect" "reflect"
"strings"
"time" "time"
"github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
@ -197,12 +198,16 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
metaBackup.clientEpoch++ metaBackup.clientEpoch++
prefix := *metaBackup.filerDirectory
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{ metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "meta_backup", ClientName: "meta_backup",
ClientId: metaBackup.clientId, ClientId: metaBackup.clientId,
ClientEpoch: metaBackup.clientEpoch, ClientEpoch: metaBackup.clientEpoch,
SelfSignature: 0, SelfSignature: 0,
PathPrefix: *metaBackup.filerDirectory,
PathPrefix: prefix,
AdditionalPathPrefixes: nil, AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: startTime.UnixNano(), StartTsNs: startTime.UnixNano(),

2
weed/command/filer_remote_gateway_buckets.go

@ -55,7 +55,7 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo
ClientId: option.clientId, ClientId: option.clientId,
ClientEpoch: option.clientEpoch, ClientEpoch: option.clientEpoch,
SelfSignature: 0, SelfSignature: 0,
PathPrefix: option.bucketsDir,
PathPrefix: option.bucketsDir + "/",
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote}, AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: lastOffsetTs.UnixNano(), StartTsNs: lastOffsetTs.UnixNano(),

7
weed/command/filer_remote_sync_dir.go

@ -64,12 +64,17 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
option.clientEpoch++ option.clientEpoch++
prefix := mountedDir
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{ metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "filer.remote.sync", ClientName: "filer.remote.sync",
ClientId: option.clientId, ClientId: option.clientId,
ClientEpoch: option.clientEpoch, ClientEpoch: option.clientEpoch,
SelfSignature: 0, SelfSignature: 0,
PathPrefix: mountedDir,
PathPrefix: prefix,
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote}, AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: lastOffsetTs.UnixNano(), StartTsNs: lastOffsetTs.UnixNano(),

7
weed/command/filer_sync.go

@ -296,12 +296,17 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs) return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
}) })
prefix := sourcePath
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{ metadataFollowOption := &pb.MetadataFollowOption{
ClientName: clientName, ClientName: clientName,
ClientId: clientId, ClientId: clientId,
ClientEpoch: clientEpoch, ClientEpoch: clientEpoch,
SelfSignature: targetFilerSignature, SelfSignature: targetFilerSignature,
PathPrefix: sourcePath,
PathPrefix: prefix,
AdditionalPathPrefixes: nil, AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: sourceFilerOffsetTsNs, StartTsNs: sourceFilerOffsetTsNs,

8
weed/mount/meta_cache/meta_cache_subscribe.go

@ -7,6 +7,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"strings"
) )
func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64) error { func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64) error {
@ -57,12 +58,17 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
} }
prefix := dir
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{ metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "mount", ClientName: "mount",
ClientId: selfSignature, ClientId: selfSignature,
ClientEpoch: 1, ClientEpoch: 1,
SelfSignature: selfSignature, SelfSignature: selfSignature,
PathPrefix: dir,
PathPrefix: prefix,
AdditionalPathPrefixes: nil, AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: lastTsNs, StartTsNs: lastTsNs,

Loading…
Cancel
Save