filer.remote.gateway: add options to include or exclude new bucket names to mirror

weed/command/filer_remote_gateway.go

@@ -22,6 +22,8 @@ type RemoteGatewayOptions struct {
 	timeAgo                  *time.Duration
 	createBucketAt           *string
 	createBucketRandomSuffix *bool
+	include *string
+	exclude *string
 
 	mappings    *remote_pb.RemoteStorageMapping
 	remoteConfs map[string]*remote_pb.RemoteConf
@@ -50,6 +52,8 @@ func init() {
 	remoteGatewayOptions.createBucketRandomSuffix = cmdFilerRemoteGateway.Flag.Bool("createBucketWithRandomSuffix", true, "add randomized suffix to bucket name to avoid conflicts")
 	remoteGatewayOptions.readChunkFromFiler = cmdFilerRemoteGateway.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers")
 	remoteGatewayOptions.timeAgo = cmdFilerRemoteGateway.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
+	remoteGatewayOptions.include = cmdFilerRemoteGateway.Flag.String("include", "", "pattens of new bucket names, e.g., s3*")
+	remoteGatewayOptions.exclude = cmdFilerRemoteGateway.Flag.String("exclude", "", "pattens of new bucket names, e.g., local*")
 }
 
 var cmdFilerRemoteGateway = &Command{

weed/command/filer_remote_gateway_buckets.go

@@ -13,6 +13,7 @@ import (
 	"github.com/golang/protobuf/proto"
 	"math"
 	"math/rand"
+	"path/filepath"
 	"strings"
 	"time"
 )
@@ -75,6 +76,16 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 		}
 
 		bucketName := strings.ToLower(entry.Name)
+		if *option.include != "" {
+			if ok, _ := filepath.Match(*option.include, entry.Name); !ok {
+				return nil
+			}
+		}
+		if *option.exclude != "" {
+			if ok, _ := filepath.Match(*option.exclude, entry.Name); ok {
+				return nil
+			}
+		}
 		if *option.createBucketRandomSuffix {
 			// https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
 			if len(bucketName)+5 > 63 {