Merge branch 'master' into add_remote_storage

weed/command/scaffold/filer.toml

@@ -12,8 +12,6 @@
 # with http DELETE, by default the filer would check whether a folder is empty.
 # recursive_delete will delete all sub folders and files, similar to "rm -Rf"
 recursive_delete = false
-# directories under this folder will be automatically creating a separate bucket
-buckets_folder = "/buckets"
 
 ####################################################
 # The following are filer store options

weed/command/scaffold/master.toml

@@ -11,6 +11,7 @@ scripts = """
   ec.encode -fullPercent=95 -quietFor=1h
   ec.rebuild -force
   ec.balance -force
+  volume.deleteEmpty -quietFor=24h
   volume.balance -force
   volume.fix.replication
   unlock

weed/filer/filer_buckets.go

@@ -3,6 +3,7 @@ package filer
 import (
 	"context"
 	"math"
+	"strings"
 	"sync"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -78,6 +79,9 @@ func (f *Filer) isBucket(entry *Entry) bool {
 	if parent != f.DirBucketsPath {
 		return false
 	}
+	if strings.HasPrefix(dirName, ".") {
+		return false
+	}
 
 	f.buckets.RLock()
 	defer f.buckets.RUnlock()

weed/filer/filer_on_meta_event.go

@@ -25,13 +25,17 @@ func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
 	}
 	if f.DirBucketsPath == event.Directory {
 		if message.OldEntry == nil && message.NewEntry != nil {
+			if message.NewEntry.IsDirectory {
 				f.Store.OnBucketCreation(message.NewEntry.Name)
 			}
+		}
 		if message.OldEntry != nil && message.NewEntry == nil {
+			if message.OldEntry.IsDirectory {
 				f.Store.OnBucketDeletion(message.OldEntry.Name)
 			}
 		}
 	}
+}
 
 func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataResponse) {
 	if DirectoryEtcSeaweedFS != event.Directory {

weed/shell/command_volume_delete_empty.go

@@ -0,0 +1,69 @@
+package shell
+
+import (
+	"flag"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"io"
+	"log"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+)
+
+func init() {
+	Commands = append(Commands, &commandVolumeDeleteEmpty{})
+}
+
+type commandVolumeDeleteEmpty struct {
+}
+
+func (c *commandVolumeDeleteEmpty) Name() string {
+	return "volume.deleteEmpty"
+}
+
+func (c *commandVolumeDeleteEmpty) Help() string {
+	return `delete empty volumes from all volume servers
+
+	volume.deleteEmpty -quietFor=24h
+
+	This command deletes all empty volumes from one volume server.
+
+`
+}
+
+func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if err = commandEnv.confirmIsLocked(); err != nil {
+		return
+	}
+
+	volDeleteCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	quietPeriod := volDeleteCommand.Duration("quietFor", 24*time.Hour, "select empty volumes with no recent writes, avoid newly created ones")
+	if err = volDeleteCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	// collect topology information
+	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	if err != nil {
+		return err
+	}
+
+	quietSeconds := int64(*quietPeriod / time.Second)
+	nowUnixSeconds := time.Now().Unix()
+
+	eachDataNode(topologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
+		for _, diskInfo := range dn.DiskInfos {
+			for _, v := range diskInfo.VolumeInfos {
+				if v.Size <= 8 && v.ModifiedAtSecond + quietSeconds < nowUnixSeconds {
+					log.Printf("deleting empty volume %d from %s", v.Id, dn.Id)
+					if deleteErr := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id), dn.Id); deleteErr != nil {
+						err = deleteErr
+					}
+				}
+			}
+		}
+	})
+
+	return
+}