use logical number of files and sizes for statistics and quota

weed/shell/command_collection_list.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
 	"io"
 )
 
@@ -23,10 +24,10 @@ func (c *commandCollectionList) Help() string {
 }
 
 type CollectionInfo struct {
-	FileCount        uint64
-	DeleteCount      uint64
-	DeletedByteCount uint64
-	Size             uint64
+	FileCount        float64
+	DeleteCount      float64
+	DeletedByteCount float64
+	Size             float64
 	VolumeCount      int
 }
 
@@ -52,7 +53,7 @@ func (c *commandCollectionList) Do(args []string, commandEnv *CommandEnv, writer
 		if !found {
 			continue
 		}
-		fmt.Fprintf(writer, "collection:\"%s\"\tvolumeCount:%d\tsize:%d\tfileCount:%d\tdeletedBytes:%d\tdeletion:%d\n", c, cif.VolumeCount, cif.Size, cif.FileCount, cif.DeletedByteCount, cif.DeleteCount)
+		fmt.Fprintf(writer, "collection:\"%s\"\tvolumeCount:%d\tsize:%.0f\tfileCount:%.0f\tdeletedBytes:%.0f\tdeletion:%.0f\n", c, cif.VolumeCount, cif.Size, cif.FileCount, cif.DeletedByteCount, cif.DeleteCount)
 	}
 
 	fmt.Fprintf(writer, "Total %d collections.\n", len(collections))
@@ -85,10 +86,12 @@ func addToCollection(collectionInfos map[string]*CollectionInfo, vif *master_pb.
 		cif = &CollectionInfo{}
 		collectionInfos[c] = cif
 	}
-	cif.Size += vif.Size
-	cif.DeleteCount += vif.DeleteCount
-	cif.FileCount += vif.FileCount
-	cif.DeletedByteCount += vif.DeletedByteCount
+	replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(vif.ReplicaPlacement))
+	copyCount := float64(replicaPlacement.GetCopyCount())
+	cif.Size += float64(vif.Size) / copyCount
+	cif.DeleteCount += float64(vif.DeleteCount) / copyCount
+	cif.FileCount += float64(vif.FileCount) / copyCount
+	cif.DeletedByteCount += float64(vif.DeletedByteCount) / copyCount
 	cif.VolumeCount++
 }
 

weed/shell/command_s3_bucket_list.go

@@ -58,12 +58,12 @@ func (c *commandS3BucketList) Do(args []string, commandEnv *CommandEnv, writer i
 			return nil
 		}
 		collection := entry.Name
-		var collectionSize, fileCount uint64
+		var collectionSize, fileCount float64
 		if collectionInfo, found := collectionInfos[collection]; found {
 			collectionSize = collectionInfo.Size
 			fileCount = collectionInfo.FileCount - collectionInfo.DeleteCount
 		}
-		fmt.Fprintf(writer, "  %s\tsize:%d\tfile:%d", entry.Name, collectionSize, fileCount)
+		fmt.Fprintf(writer, "  %s\tsize:%.0f\tfile:%.0f", entry.Name, collectionSize, fileCount)
 		if entry.Quota > 0 {
 			fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, float64(collectionSize)*100/float64(entry.Quota))
 		}

weed/shell/command_s3_bucket_quota_check.go

@@ -65,7 +65,7 @@ func (c *commandS3BucketQuotaEnforce) Do(args []string, commandEnv *CommandEnv,
 			return nil
 		}
 		collection := entry.Name
-		var collectionSize uint64
+		var collectionSize float64
 		if collectionInfo, found := collectionInfos[collection]; found {
 			collectionSize = collectionInfo.Size
 		}
@@ -95,7 +95,7 @@ func (c *commandS3BucketQuotaEnforce) Do(args []string, commandEnv *CommandEnv,
 
 }
 
-func (c *commandS3BucketQuotaEnforce) processEachBucket(fc *filer.FilerConf, filerBucketsPath string, entry *filer_pb.Entry, writer io.Writer, collectionSize uint64) (hasConfChanges bool) {
+func (c *commandS3BucketQuotaEnforce) processEachBucket(fc *filer.FilerConf, filerBucketsPath string, entry *filer_pb.Entry, writer io.Writer, collectionSize float64) (hasConfChanges bool) {
 
 	locPrefix := filerBucketsPath + "/" + entry.Name + "/"
 	locConf := fc.MatchStorageRule(locPrefix)
@@ -103,12 +103,12 @@ func (c *commandS3BucketQuotaEnforce) processEachBucket(fc *filer.FilerConf, fil
 
 	if entry.Quota > 0 {
 		if locConf.ReadOnly {
-			if collectionSize < uint64(entry.Quota) {
+			if collectionSize < float64(entry.Quota) {
 				locConf.ReadOnly = false
 				hasConfChanges = true
 			}
 		} else {
-			if collectionSize > uint64(entry.Quota) {
+			if collectionSize > float64(entry.Quota) {
 				locConf.ReadOnly = true
 				hasConfChanges = true
 			}
@@ -122,7 +122,7 @@ func (c *commandS3BucketQuotaEnforce) processEachBucket(fc *filer.FilerConf, fil
 
 	if hasConfChanges {
 		fmt.Fprintf(writer, "  %s\tsize:%d", entry.Name, collectionSize)
-		fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, float64(collectionSize)*100/float64(entry.Quota))
+		fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, collectionSize*100/float64(entry.Quota))
 		fmt.Fprintln(writer)
 		if locConf.ReadOnly {
 			fmt.Fprintf(writer, "    changing bucket %s to read only!\n", entry.Name)