From 187ef65e8fa744d14dce0c79f61221272deaad4b Mon Sep 17 00:00:00 2001 From: Lisandro Pin Date: Mon, 15 Dec 2025 20:18:45 +0100 Subject: [PATCH] Humanize output for `weed.server` by default (#7758) * Implement a `weed shell` command to return a status overview of the cluster. Detailed file information will be implemented in a follow-up MR. Note also that masters are currently not reporting back EC shard sizes correctly, via `master_pb.VolumeEcShardInformationMessage.shard_sizes`. F.ex: ``` > status cluster: id: topo status: LOCKED nodes: 10 topology: 1 DC(s)s, 1 disk(s) on 1 rack(s) volumes: total: 3 volumes on 1 collections max size: 31457280000 bytes regular: 2/80 volumes on 6 replicas, 6 writable (100.00%), 0 read-only (0.00%) EC: 1 EC volumes on 14 shards (14.00 shards/volume) storage: total: 186024424 bytes regular volumes: 186024424 bytes EC volumes: 0 bytes raw: 558073152 bytes on volume replicas, 0 bytes on EC shard files ``` * Humanize output for `weed.server` by default. Makes things more readable :) ``` > cluster.status cluster: id: topo status: LOCKED nodes: 10 topology: 1 DC, 10 disks on 1 rack volumes: total: 3 volumes, 1 collection max size: 32 GB regular: 2/80 volumes on 6 replicas, 6 writable (100%), 0 read-only (0%) EC: 1 EC volume on 14 shards (14 shards/volume) storage: total: 172 MB regular volumes: 172 MB EC volumes: 0 B raw: 516 MB on volume replicas, 0 B on EC shards ``` ``` > cluster.status --humanize=false cluster: id: topo status: LOCKED nodes: 10 topology: 1 DC(s), 10 disk(s) on 1 rack(s) volumes: total: 3 volume(s), 1 collection(s) max size: 31457280000 byte(s) regular: 2/80 volume(s) on 6 replica(s), 5 writable (83.33%), 1 read-only (16.67%) EC: 1 EC volume(s) on 14 shard(s) (14.00 shards/volume) storage: total: 172128072 byte(s) regular volumes: 172128072 byte(s) EC volumes: 0 byte(s) raw: 516384216 byte(s) on volume replicas, 0 byte(s) on EC shards ``` Also adds unit tests, and reshuffles test files handling for clarity. --- weed/shell/command_cluster_status.go | 135 +++++++++++++++------ weed/shell/command_cluster_status_test.go | 140 ++++++++++++++++++++++ weed/shell/command_ec_common_test.go | 82 ++++++------- weed/shell/command_volume_list_test.go | 16 +-- weed/shell/common_test.go | 20 ++++ weed/shell/volume.ecshards.txt | 4 +- weed/shell/volume.list.txt | 2 +- weed/shell/volume.list2.txt | 2 +- 8 files changed, 306 insertions(+), 95 deletions(-) create mode 100644 weed/shell/command_cluster_status_test.go create mode 100644 weed/shell/common_test.go diff --git a/weed/shell/command_cluster_status.go b/weed/shell/command_cluster_status.go index c943e10b8..01c45d5a9 100644 --- a/weed/shell/command_cluster_status.go +++ b/weed/shell/command_cluster_status.go @@ -5,6 +5,8 @@ import ( "fmt" "strings" + "github.com/dustin/go-humanize" + "github.com/dustin/go-humanize/english" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/needle" @@ -18,7 +20,8 @@ func init() { type commandClusterStatus struct{} type ClusterStatusPrinter struct { - writer io.Writer + writer io.Writer + humanize bool locked bool collections []string @@ -40,6 +43,7 @@ func (c *commandClusterStatus) HasTag(CommandTag) bool { func (c *commandClusterStatus) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { flags := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + humanize := flags.Bool("humanize", true, "human-readable output") if err = flags.Parse(args); err != nil { return err @@ -55,7 +59,8 @@ func (c *commandClusterStatus) Do(args []string, commandEnv *CommandEnv, writer } sp := &ClusterStatusPrinter{ - writer: writer, + writer: writer, + humanize: *humanize, locked: commandEnv.isLocked(), collections: collections, @@ -67,7 +72,66 @@ func (c *commandClusterStatus) Do(args []string, commandEnv *CommandEnv, writer return nil } -// TODO: humanize figures in output +func (sp *ClusterStatusPrinter) uint64(n uint64) string { + if !sp.humanize { + return fmt.Sprintf("%d", n) + } + return humanize.Comma(int64(n)) +} + +func (sp *ClusterStatusPrinter) int(n int) string { + return sp.uint64(uint64(n)) +} + +func (sp *ClusterStatusPrinter) plural(n int, str string) string { + if !sp.humanize { + return fmt.Sprintf("%s(s)", str) + } + return english.PluralWord(n, str, "") +} + +func (sp *ClusterStatusPrinter) bytes(b uint64) string { + if !sp.humanize { + return fmt.Sprintf("%d %s", b, sp.plural(int(b), "byte")) + } + return fmt.Sprintf("%s", humanize.Bytes(b)) +} + +func (sp *ClusterStatusPrinter) uint64Ratio(a, b uint64) string { + var p float64 + if b != 0 { + p = float64(a) / float64(b) + } + if !sp.humanize { + return fmt.Sprintf("%.02f", p) + } + return fmt.Sprintf("%s", humanize.FtoaWithDigits(p, 2)) +} + +func (sp *ClusterStatusPrinter) intRatio(a, b int) string { + return sp.uint64Ratio(uint64(a), uint64(b)) +} + +func (sp *ClusterStatusPrinter) uint64Pct(a, b uint64) string { + var p float64 + if b != 0 { + p = 100 * float64(a) / float64(b) + } + if !sp.humanize { + return fmt.Sprintf("%.02f%%", p) + } + return fmt.Sprintf("%s%%", humanize.FtoaWithDigits(p, 2)) +} + +func (sp *ClusterStatusPrinter) intPct(a, b int) string { + return sp.uint64Pct(uint64(a), uint64(b)) +} + +func (sp *ClusterStatusPrinter) write(format string, a ...any) { + fmt.Fprintf(sp.writer, strings.TrimRight(format, "\r\n "), a...) + fmt.Fprint(sp.writer, "\n") +} + // TODO: add option to collect detailed file stats func (sp *ClusterStatusPrinter) Print() { sp.write("") @@ -76,11 +140,6 @@ func (sp *ClusterStatusPrinter) Print() { sp.printStorageInfo() } -func (sp *ClusterStatusPrinter) write(format string, a ...any) { - fmt.Fprintf(sp.writer, strings.TrimRight(format, "\r\n "), a...) - fmt.Fprint(sp.writer, "\n") -} - func (sp *ClusterStatusPrinter) printClusterInfo() { dcs := len(sp.topology.DataCenterInfos) @@ -105,18 +164,22 @@ func (sp *ClusterStatusPrinter) printClusterInfo() { sp.write("cluster:") sp.write("\tid: %s", sp.topology.Id) sp.write("\tstatus: %s", status) - sp.write("\tnodes: %d", nodes) - sp.write("\ttopology: %d DC(s), %d disk(s) on %d rack(s)", dcs, disks, racks) + sp.write("\tnodes: %s", sp.int(nodes)) + sp.write("\ttopology: %s %s, %s %s on %s %s", + sp.int(dcs), sp.plural(dcs, "DC"), + sp.int(disks), sp.plural(disks, "disk"), + sp.int(racks), sp.plural(racks, "rack")) sp.write("") + } func (sp *ClusterStatusPrinter) printVolumeInfo() { collections := len(sp.collections) var maxVolumes uint64 - volumes := map[needle.VolumeId]bool{} - ecVolumes := map[needle.VolumeId]bool{} + volumeIds := map[needle.VolumeId]bool{} + ecVolumeIds := map[needle.VolumeId]bool{} - var replicas, roReplicas, rwReplicas, ecShards uint64 + var replicas, roReplicas, rwReplicas, ecShards int for _, dci := range sp.topology.DataCenterInfos { for _, ri := range dci.RackInfos { @@ -125,7 +188,7 @@ func (sp *ClusterStatusPrinter) printVolumeInfo() { maxVolumes += uint64(di.MaxVolumeCount) for _, vi := range di.VolumeInfos { vid := needle.VolumeId(vi.Id) - volumes[vid] = true + volumeIds[vid] = true replicas++ if vi.ReadOnly { roReplicas++ @@ -135,31 +198,34 @@ func (sp *ClusterStatusPrinter) printVolumeInfo() { } for _, eci := range di.EcShardInfos { vid := needle.VolumeId(eci.Id) - ecVolumes[vid] = true - ecShards += uint64(erasure_coding.ShardBits(eci.EcIndexBits).ShardIdCount()) + ecVolumeIds[vid] = true + ecShards += erasure_coding.ShardBits(eci.EcIndexBits).ShardIdCount() } } } } } - var roReplicasRatio, rwReplicasRatio, ecShardsPerVolume float64 - if replicas != 0 { - roReplicasRatio = float64(roReplicas) / float64(replicas) - rwReplicasRatio = float64(rwReplicas) / float64(replicas) - } - if len(ecVolumes) != 0 { - ecShardsPerVolume = float64(ecShards) / float64(len(ecVolumes)) - } - - totalVolumes := len(volumes) + len(ecVolumes) + volumes := len(volumeIds) + ecVolumes := len(ecVolumeIds) + totalVolumes := volumes + ecVolumes sp.write("volumes:") - sp.write("\ttotal: %d volumes on %d collections", totalVolumes, collections) - sp.write("\tmax size: %d bytes", sp.volumeSizeLimitMb*1024*1024) - sp.write("\tregular: %d/%d volumes on %d replicas, %d writable (%.02f%%), %d read-only (%.02f%%)", len(volumes), maxVolumes, replicas, rwReplicas, 100*rwReplicasRatio, roReplicas, 100*roReplicasRatio) - sp.write("\tEC: %d EC volumes on %d shards (%.02f shards/volume)", len(ecVolumes), ecShards, ecShardsPerVolume) + sp.write("\ttotal: %s %s, %s %s", + sp.int(totalVolumes), sp.plural(totalVolumes, "volume"), + sp.int(collections), sp.plural(collections, "collection")) + sp.write("\tmax size: %s", sp.bytes(sp.volumeSizeLimitMb*1024*1024)) + sp.write("\tregular: %s/%s %s on %s %s, %s writable (%s), %s read-only (%s)", + sp.int(volumes), sp.uint64(maxVolumes), sp.plural(volumes, "volume"), + sp.int(replicas), sp.plural(replicas, "replica"), + sp.int(rwReplicas), sp.intPct(rwReplicas, replicas), + sp.int(roReplicas), sp.intPct(roReplicas, replicas)) + sp.write("\tEC: %s EC %s on %s %s (%s shards/volume)", + sp.int(ecVolumes), sp.plural(ecVolumes, "volume"), + sp.int(ecShards), sp.plural(ecShards, "shard"), + sp.intRatio(ecShards, ecVolumes)) sp.write("") + } func (sp *ClusterStatusPrinter) printStorageInfo() { @@ -202,13 +268,12 @@ func (sp *ClusterStatusPrinter) printStorageInfo() { for _, s := range perEcVolumeSize { ecVolumeSize += s } - totalSize := volumeSize + ecVolumeSize sp.write("storage:") - sp.write("\ttotal: %d bytes", totalSize) - sp.write("\tregular volumes: %d bytes", volumeSize) - sp.write("\tEC volumes: %d bytes", ecVolumeSize) - sp.write("\traw: %d bytes on volume replicas, %d bytes on EC shard files", rawVolumeSize, rawEcVolumeSize) + sp.write("\ttotal: %s", sp.bytes(totalSize)) + sp.write("\tregular volumes: %s", sp.bytes(volumeSize)) + sp.write("\tEC volumes: %s", sp.bytes(ecVolumeSize)) + sp.write("\traw: %s on volume replicas, %s on EC shards", sp.bytes(rawVolumeSize), sp.bytes(rawEcVolumeSize)) sp.write("") } diff --git a/weed/shell/command_cluster_status_test.go b/weed/shell/command_cluster_status_test.go new file mode 100644 index 000000000..92de049c6 --- /dev/null +++ b/weed/shell/command_cluster_status_test.go @@ -0,0 +1,140 @@ +package shell + +import ( + "bytes" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +func TestPrintClusterInfo(t *testing.T) { + testCases := []struct { + topology *master_pb.TopologyInfo + humanize bool + want string + }{ + { + testTopology1, true, + `cluster: + id: test_topo_1 + status: unlocked + nodes: 5 + topology: 5 DCs, 5 disks on 6 racks + +`, + }, + { + testTopology1, false, + `cluster: + id: test_topo_1 + status: unlocked + nodes: 5 + topology: 5 DC(s), 5 disk(s) on 6 rack(s) + +`, + }, + } + + for _, tc := range testCases { + var buf bytes.Buffer + sp := &ClusterStatusPrinter{ + writer: &buf, + humanize: tc.humanize, + topology: tc.topology, + } + sp.printClusterInfo() + got := buf.String() + + if got != tc.want { + t.Errorf("for %v: got %v, want %v", tc.topology.Id, got, tc.want) + } + } +} + +func TestPrintVolumeInfo(t *testing.T) { + testCases := []struct { + topology *master_pb.TopologyInfo + humanize bool + want string + }{ + { + testTopology2, true, + `volumes: + total: 12,056 volumes, 0 collections + max size: 0 B + regular: 5,302/25,063 volumes on 15,900 replicas, 15,900 writable (100%), 0 read-only (0%) + EC: 6,754 EC volumes on 91,662 shards (13.57 shards/volume) + +`, + }, + { + testTopology2, false, + `volumes: + total: 12056 volume(s), 0 collection(s) + max size: 0 byte(s) + regular: 5302/25063 volume(s) on 15900 replica(s), 15900 writable (100.00%), 0 read-only (0.00%) + EC: 6754 EC volume(s) on 91662 shard(s) (13.57 shards/volume) + +`, + }, + } + + for _, tc := range testCases { + var buf bytes.Buffer + sp := &ClusterStatusPrinter{ + writer: &buf, + humanize: tc.humanize, + topology: tc.topology, + } + sp.printVolumeInfo() + got := buf.String() + + if got != tc.want { + t.Errorf("for %v: got %v, want %v", tc.topology.Id, got, tc.want) + } + } +} + +func TestPrintStorageInfo(t *testing.T) { + testCases := []struct { + topology *master_pb.TopologyInfo + humanize bool + want string + }{ + { + testTopology2, true, + `storage: + total: 5.9 TB + regular volumes: 5.9 TB + EC volumes: 0 B + raw: 18 TB on volume replicas, 0 B on EC shards + +`, + }, + { + testTopology2, false, + `storage: + total: 5892610895448 byte(s) + regular volumes: 5892610895448 byte(s) + EC volumes: 0 byte(s) + raw: 17676186754616 byte(s) on volume replicas, 0 byte(s) on EC shards + +`, + }, + } + + for _, tc := range testCases { + var buf bytes.Buffer + sp := &ClusterStatusPrinter{ + writer: &buf, + humanize: tc.humanize, + topology: tc.topology, + } + sp.printStorageInfo() + got := buf.String() + + if got != tc.want { + t.Errorf("for %v: got %v, want %v", tc.topology.Id, got, tc.want) + } + } +} diff --git a/weed/shell/command_ec_common_test.go b/weed/shell/command_ec_common_test.go index 47bf9eea1..8f5f4a1b9 100644 --- a/weed/shell/command_ec_common_test.go +++ b/weed/shell/command_ec_common_test.go @@ -13,12 +13,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/types" ) -var ( - topology1 = parseOutput(topoData) - topology2 = parseOutput(topoData2) - topologyEc = parseOutput(topoDataEc) -) - func errorCheck(got error, want string) error { if got == nil && want == "" { return nil @@ -42,22 +36,22 @@ func TestCollectCollectionsForVolumeIds(t *testing.T) { want []string }{ // normal volumes - {topology1, nil, nil}, - {topology1, []needle.VolumeId{}, nil}, - {topology1, []needle.VolumeId{needle.VolumeId(9999)}, nil}, - {topology1, []needle.VolumeId{needle.VolumeId(2)}, []string{""}}, - {topology1, []needle.VolumeId{needle.VolumeId(2), needle.VolumeId(272)}, []string{"", "collection2"}}, - {topology1, []needle.VolumeId{needle.VolumeId(2), needle.VolumeId(272), needle.VolumeId(299)}, []string{"", "collection2"}}, - {topology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95)}, []string{"collection1", "collection2"}}, - {topology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95), needle.VolumeId(51)}, []string{"collection1", "collection2"}}, - {topology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95), needle.VolumeId(51), needle.VolumeId(15)}, []string{"collection0", "collection1", "collection2"}}, + {testTopology1, nil, nil}, + {testTopology1, []needle.VolumeId{}, nil}, + {testTopology1, []needle.VolumeId{needle.VolumeId(9999)}, nil}, + {testTopology1, []needle.VolumeId{needle.VolumeId(2)}, []string{""}}, + {testTopology1, []needle.VolumeId{needle.VolumeId(2), needle.VolumeId(272)}, []string{"", "collection2"}}, + {testTopology1, []needle.VolumeId{needle.VolumeId(2), needle.VolumeId(272), needle.VolumeId(299)}, []string{"", "collection2"}}, + {testTopology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95)}, []string{"collection1", "collection2"}}, + {testTopology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95), needle.VolumeId(51)}, []string{"collection1", "collection2"}}, + {testTopology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95), needle.VolumeId(51), needle.VolumeId(15)}, []string{"collection0", "collection1", "collection2"}}, // EC volumes - {topology2, []needle.VolumeId{needle.VolumeId(9577)}, []string{"s3qldata"}}, - {topology2, []needle.VolumeId{needle.VolumeId(9577), needle.VolumeId(12549)}, []string{"s3qldata"}}, + {testTopology2, []needle.VolumeId{needle.VolumeId(9577)}, []string{"s3qldata"}}, + {testTopology2, []needle.VolumeId{needle.VolumeId(9577), needle.VolumeId(12549)}, []string{"s3qldata"}}, // normal + EC volumes - {topology2, []needle.VolumeId{needle.VolumeId(18111)}, []string{"s3qldata"}}, - {topology2, []needle.VolumeId{needle.VolumeId(8677)}, []string{"s3qldata"}}, - {topology2, []needle.VolumeId{needle.VolumeId(18111), needle.VolumeId(8677)}, []string{"s3qldata"}}, + {testTopology2, []needle.VolumeId{needle.VolumeId(18111)}, []string{"s3qldata"}}, + {testTopology2, []needle.VolumeId{needle.VolumeId(8677)}, []string{"s3qldata"}}, + {testTopology2, []needle.VolumeId{needle.VolumeId(18111), needle.VolumeId(8677)}, []string{"s3qldata"}}, } for _, tc := range testCases { @@ -106,7 +100,7 @@ func TestParseReplicaPlacementArg(t *testing.T) { func TestEcDistribution(t *testing.T) { // find out all volume servers with one slot left. - ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(topology1, "", types.HardDriveType) + ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(testTopology1, "", types.HardDriveType) sortEcNodesByFreeslotsDescending(ecNodes) @@ -133,18 +127,18 @@ func TestPickRackToBalanceShardsInto(t *testing.T) { wantErr string }{ // Non-EC volumes. We don't care about these, but the function should return all racks as a safeguard. - {topologyEc, "", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""}, - {topologyEc, "6225", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""}, - {topologyEc, "6226", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""}, - {topologyEc, "6241", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""}, - {topologyEc, "6242", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""}, + {testTopologyEc, "", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""}, + {testTopologyEc, "6225", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""}, + {testTopologyEc, "6226", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""}, + {testTopologyEc, "6241", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""}, + {testTopologyEc, "6242", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""}, // EC volumes. - {topologyEc, "9577", "", nil, "shards 1 > replica placement limit for other racks (0)"}, - {topologyEc, "9577", "111", []string{"rack1", "rack2", "rack3"}, ""}, - {topologyEc, "9577", "222", []string{"rack1", "rack2", "rack3"}, ""}, - {topologyEc, "10457", "222", []string{"rack1"}, ""}, - {topologyEc, "12737", "222", []string{"rack2"}, ""}, - {topologyEc, "14322", "222", []string{"rack3"}, ""}, + {testTopologyEc, "9577", "", nil, "shards 1 > replica placement limit for other racks (0)"}, + {testTopologyEc, "9577", "111", []string{"rack1", "rack2", "rack3"}, ""}, + {testTopologyEc, "9577", "222", []string{"rack1", "rack2", "rack3"}, ""}, + {testTopologyEc, "10457", "222", []string{"rack1"}, ""}, + {testTopologyEc, "12737", "222", []string{"rack2"}, ""}, + {testTopologyEc, "14322", "222", []string{"rack3"}, ""}, } for _, tc := range testCases { @@ -190,11 +184,11 @@ func TestPickEcNodeToBalanceShardsInto(t *testing.T) { wantOneOf []string wantErr string }{ - {topologyEc, "", "", nil, "INTERNAL: missing source nodes"}, - {topologyEc, "idontexist", "12737", nil, "INTERNAL: missing source nodes"}, + {testTopologyEc, "", "", nil, "INTERNAL: missing source nodes"}, + {testTopologyEc, "idontexist", "12737", nil, "INTERNAL: missing source nodes"}, // Non-EC nodes. We don't care about these, but the function should return all available target nodes as a safeguard. { - topologyEc, "172.19.0.10:8702", "6225", []string{ + testTopologyEc, "172.19.0.10:8702", "6225", []string{ "172.19.0.13:8701", "172.19.0.14:8711", "172.19.0.16:8704", "172.19.0.17:8703", "172.19.0.19:8700", "172.19.0.20:8706", "172.19.0.21:8710", "172.19.0.3:8708", "172.19.0.4:8707", "172.19.0.5:8705", "172.19.0.6:8713", "172.19.0.8:8709", @@ -202,7 +196,7 @@ func TestPickEcNodeToBalanceShardsInto(t *testing.T) { "", }, { - topologyEc, "172.19.0.8:8709", "6226", []string{ + testTopologyEc, "172.19.0.8:8709", "6226", []string{ "172.19.0.10:8702", "172.19.0.13:8701", "172.19.0.14:8711", "172.19.0.16:8704", "172.19.0.17:8703", "172.19.0.19:8700", "172.19.0.20:8706", "172.19.0.21:8710", "172.19.0.3:8708", "172.19.0.4:8707", "172.19.0.5:8705", "172.19.0.6:8713", @@ -210,16 +204,16 @@ func TestPickEcNodeToBalanceShardsInto(t *testing.T) { "", }, // EC volumes. - {topologyEc, "172.19.0.10:8702", "14322", []string{ + {testTopologyEc, "172.19.0.10:8702", "14322", []string{ "172.19.0.14:8711", "172.19.0.5:8705", "172.19.0.6:8713"}, ""}, - {topologyEc, "172.19.0.13:8701", "10457", []string{ + {testTopologyEc, "172.19.0.13:8701", "10457", []string{ "172.19.0.10:8702", "172.19.0.6:8713"}, ""}, - {topologyEc, "172.19.0.17:8703", "12737", []string{ + {testTopologyEc, "172.19.0.17:8703", "12737", []string{ "172.19.0.13:8701"}, ""}, - {topologyEc, "172.19.0.20:8706", "14322", []string{ + {testTopologyEc, "172.19.0.20:8706", "14322", []string{ "172.19.0.14:8711", "172.19.0.5:8705", "172.19.0.6:8713"}, ""}, } @@ -277,7 +271,7 @@ func TestCountFreeShardSlots(t *testing.T) { }{ { name: "topology #1, free HDD shards", - topology: topology1, + topology: testTopology1, diskType: types.HardDriveType, want: map[string]int{ "192.168.1.1:8080": 17330, @@ -289,7 +283,7 @@ func TestCountFreeShardSlots(t *testing.T) { }, { name: "topology #1, no free SSD shards available", - topology: topology1, + topology: testTopology1, diskType: types.SsdType, want: map[string]int{ "192.168.1.1:8080": 0, @@ -301,7 +295,7 @@ func TestCountFreeShardSlots(t *testing.T) { }, { name: "topology #2, no negative free HDD shards", - topology: topology2, + topology: testTopology2, diskType: types.HardDriveType, want: map[string]int{ "172.19.0.3:8708": 0, @@ -322,7 +316,7 @@ func TestCountFreeShardSlots(t *testing.T) { }, { name: "topology #2, no free SSD shards available", - topology: topology2, + topology: testTopology2, diskType: types.SsdType, want: map[string]int{ "172.19.0.10:8702": 0, diff --git a/weed/shell/command_volume_list_test.go b/weed/shell/command_volume_list_test.go index 4d67ada01..8b85efc95 100644 --- a/weed/shell/command_volume_list_test.go +++ b/weed/shell/command_volume_list_test.go @@ -1,8 +1,6 @@ package shell import ( - _ "embed" - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/stretchr/testify/assert" @@ -32,6 +30,7 @@ func TestParsing(t *testing.T) { } +// TODO: actually parsing all fields would be nice... func parseOutput(output string) *master_pb.TopologyInfo { lines := strings.Split(output, "\n") var topo *master_pb.TopologyInfo @@ -45,7 +44,9 @@ func parseOutput(output string) *master_pb.TopologyInfo { switch parts[0] { case "Topology": if topo == nil { - topo = &master_pb.TopologyInfo{} + topo = &master_pb.TopologyInfo{ + Id: parts[1], + } } case "DataCenter": if dc == nil { @@ -124,12 +125,3 @@ func parseOutput(output string) *master_pb.TopologyInfo { return topo } - -//go:embed volume.list.txt -var topoData string - -//go:embed volume.list2.txt -var topoData2 string - -//go:embed volume.ecshards.txt -var topoDataEc string diff --git a/weed/shell/common_test.go b/weed/shell/common_test.go new file mode 100644 index 000000000..023bedfb8 --- /dev/null +++ b/weed/shell/common_test.go @@ -0,0 +1,20 @@ +package shell + +import ( + _ "embed" +) + +//go:embed volume.list.txt +var topoData string + +//go:embed volume.list2.txt +var topoData2 string + +//go:embed volume.ecshards.txt +var topoDataEc string + +var ( + testTopology1 = parseOutput(topoData) + testTopology2 = parseOutput(topoData2) + testTopologyEc = parseOutput(topoDataEc) +) diff --git a/weed/shell/volume.ecshards.txt b/weed/shell/volume.ecshards.txt index 2070187da..21e4cb756 100644 --- a/weed/shell/volume.ecshards.txt +++ b/weed/shell/volume.ecshards.txt @@ -1,4 +1,4 @@ -Topology volumeSizeLimit:1024 MB hdd(volume:15900/25063 active:15900 free:9163 remote:0) +Topology test_topo_ec volumeSizeLimit:1024 MB hdd(volume:15900/25063 active:15900 free:9163 remote:0) DataCenter DefaultDataCenter hdd(volume:15900/25063 active:15900 free:9163 remote:0) Rack rack1 hdd(volume:15900/25063 active:15900 free:9163 remote:0) DataNode 172.19.0.10:8702 hdd(volume:7/2225 active:7 free:2225 remote:0) @@ -131,4 +131,4 @@ Topology volumeSizeLimit:1024 MB hdd(volume:15900/25063 active:15900 free:9163 r DataNode 172.19.0.9:8712 total size:1629878746296 file_count:497502 deleted_file:11825 deleted_bytes:23550625989 Rack rack6 total size:17676186754616 file_count:5439969 deleted_file:127907 deleted_bytes:251707271029 DataCenter DefaultDataCenter total size:17676186754616 file_count:5439969 deleted_file:127907 deleted_bytes:251707271029 -total size:17676186754616 file_count:5439969 deleted_file:127907 deleted_bytes:251707271029 \ No newline at end of file +total size:17676186754616 file_count:5439969 deleted_file:127907 deleted_bytes:251707271029 diff --git a/weed/shell/volume.list.txt b/weed/shell/volume.list.txt index b3ceaa140..e2ea6ebf9 100644 --- a/weed/shell/volume.list.txt +++ b/weed/shell/volume.list.txt @@ -1,4 +1,4 @@ -Topology volumeSizeLimit:1024 MB hdd(volume:760/7280 active:760 free:6520 remote:0) +Topology test_topo_1 volumeSizeLimit:1024 MB hdd(volume:760/7280 active:760 free:6520 remote:0) DataCenter dc1 hdd(volume:0/0 active:0 free:0 remote:0) Rack DefaultRack hdd(volume:0/0 active:0 free:0 remote:0) Rack DefaultRack total size:0 file_count:0 diff --git a/weed/shell/volume.list2.txt b/weed/shell/volume.list2.txt index adc521f84..188ba8260 100644 --- a/weed/shell/volume.list2.txt +++ b/weed/shell/volume.list2.txt @@ -1,4 +1,4 @@ -Topology volumeSizeLimit:1024 MB hdd(volume:15900/25063 active:15900 free:9163 remote:0) +Topology test_topo_2 volumeSizeLimit:1024 MB hdd(volume:15900/25063 active:15900 free:9163 remote:0) DataCenter DefaultDataCenter hdd(volume:15900/25063 active:15900 free:9163 remote:0) Rack DefaultRack hdd(volume:15900/25063 active:15900 free:9163 remote:0) DataNode 172.19.0.10:8702 hdd(volume:1559/2232 active:1559 free:673 remote:0)