From 1c7e404abec01da8bb322ef4000d6acecf1a6be5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Aug 2021 12:23:16 -0700 Subject: [PATCH 1/4] remove buckets folder option the related code still works for old deployments --- weed/command/scaffold/filer.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 1de7d02aa..9e9258865 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -12,8 +12,6 @@ # with http DELETE, by default the filer would check whether a folder is empty. # recursive_delete will delete all sub folders and files, similar to "rm -Rf" recursive_delete = false -# directories under this folder will be automatically creating a separate bucket -buckets_folder = "/buckets" #################################################### # The following are filer store options From 89933c46d224008cf53cd16e1bf2d04188d7bb1a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Aug 2021 12:28:08 -0700 Subject: [PATCH 2/4] s3: skip hidden directories in /buckets folder --- weed/filer/filer_buckets.go | 4 ++++ weed/filer/filer_on_meta_event.go | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/weed/filer/filer_buckets.go b/weed/filer/filer_buckets.go index 43fb000c9..38a1abadb 100644 --- a/weed/filer/filer_buckets.go +++ b/weed/filer/filer_buckets.go @@ -3,6 +3,7 @@ package filer import ( "context" "math" + "strings" "sync" "github.com/chrislusf/seaweedfs/weed/glog" @@ -78,6 +79,9 @@ func (f *Filer) isBucket(entry *Entry) bool { if parent != f.DirBucketsPath { return false } + if strings.HasPrefix(dirName, ".") { + return false + } f.buckets.RLock() defer f.buckets.RUnlock() diff --git a/weed/filer/filer_on_meta_event.go b/weed/filer/filer_on_meta_event.go index c9f75a5ca..5717b2b09 100644 --- a/weed/filer/filer_on_meta_event.go +++ b/weed/filer/filer_on_meta_event.go @@ -24,10 +24,14 @@ func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) { } if f.DirBucketsPath == event.Directory { if message.OldEntry == nil && message.NewEntry != nil { - f.Store.OnBucketCreation(message.NewEntry.Name) + if message.NewEntry.IsDirectory { + f.Store.OnBucketCreation(message.NewEntry.Name) + } } if message.OldEntry != nil && message.NewEntry == nil { - f.Store.OnBucketDeletion(message.OldEntry.Name) + if message.OldEntry.IsDirectory { + f.Store.OnBucketDeletion(message.OldEntry.Name) + } } } } From 2ca1839d77956371d2af384a13f2cb8bd9981fcd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Aug 2021 15:33:45 -0700 Subject: [PATCH 3/4] shell: add `volume.deleteEmpty` command --- weed/command/scaffold/master.toml | 1 + weed/shell/command_volume_delete_empty.go | 69 +++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 weed/shell/command_volume_delete_empty.go diff --git a/weed/command/scaffold/master.toml b/weed/command/scaffold/master.toml index 60a7dbcfc..9acefe69e 100644 --- a/weed/command/scaffold/master.toml +++ b/weed/command/scaffold/master.toml @@ -11,6 +11,7 @@ scripts = """ ec.encode -fullPercent=95 -quietFor=1h ec.rebuild -force ec.balance -force + volume.deleteEmpty volume.balance -force volume.fix.replication unlock diff --git a/weed/shell/command_volume_delete_empty.go b/weed/shell/command_volume_delete_empty.go new file mode 100644 index 000000000..07c6d5b2c --- /dev/null +++ b/weed/shell/command_volume_delete_empty.go @@ -0,0 +1,69 @@ +package shell + +import ( + "flag" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "io" + "log" + "time" + + "github.com/chrislusf/seaweedfs/weed/storage/needle" +) + +func init() { + Commands = append(Commands, &commandVolumeDeleteEmpty{}) +} + +type commandVolumeDeleteEmpty struct { +} + +func (c *commandVolumeDeleteEmpty) Name() string { + return "volume.deleteEmpty" +} + +func (c *commandVolumeDeleteEmpty) Help() string { + return `delete empty volumes from all volume servers + + volume.deleteEmpty -quietFor=24h + + This command deletes all empty volumes from one volume server. + +` +} + +func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + if err = commandEnv.confirmIsLocked(); err != nil { + return + } + + volDeleteCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + quietPeriod := volDeleteCommand.Duration("quietFor", 24*time.Hour, "select empty volumes with no recent writes, avoid newly created ones") + if err = volDeleteCommand.Parse(args); err != nil { + return nil + } + + // collect topology information + topologyInfo, _, err := collectTopologyInfo(commandEnv) + if err != nil { + return err + } + + quietSeconds := int64(*quietPeriod / time.Second) + nowUnixSeconds := time.Now().Unix() + + eachDataNode(topologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) { + for _, diskInfo := range dn.DiskInfos { + for _, v := range diskInfo.VolumeInfos { + if v.Size <= 8 && v.ModifiedAtSecond + quietSeconds < nowUnixSeconds { + log.Printf("deleting empty volume %d from %s", v.Id, dn.Id) + if deleteErr := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id), dn.Id); deleteErr != nil { + err = deleteErr + } + } + } + } + }) + + return +} From 58bc3ecf4712dd8c94c3470324aa68492072c78b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Aug 2021 15:36:06 -0700 Subject: [PATCH 4/4] add default quietFor value --- weed/command/scaffold/master.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/scaffold/master.toml b/weed/command/scaffold/master.toml index 9acefe69e..cdd92016c 100644 --- a/weed/command/scaffold/master.toml +++ b/weed/command/scaffold/master.toml @@ -11,7 +11,7 @@ scripts = """ ec.encode -fullPercent=95 -quietFor=1h ec.rebuild -force ec.balance -force - volume.deleteEmpty + volume.deleteEmpty -quietFor=24h volume.balance -force volume.fix.replication unlock