chrislu
4 weeks ago
62 changed files with 890 additions and 425 deletions
-
24go.mod
-
48go.sum
-
4k8s/charts/seaweedfs/Chart.yaml
-
24k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml
-
6other/java/client/pom.xml
-
6other/java/client/pom.xml.deploy
-
6other/java/client/pom_debug.xml
-
21other/java/client/src/main/java/seaweedfs/client/ReadChunks.java
-
4other/java/examples/pom.xml
-
2other/java/hdfs2/dependency-reduced-pom.xml
-
2other/java/hdfs2/pom.xml
-
2other/java/hdfs3/dependency-reduced-pom.xml
-
2other/java/hdfs3/pom.xml
-
52weed/command/filer_backup.go
-
29weed/command/filer_sync.go
-
1weed/command/scaffold/master.toml
-
10weed/filer/elastic/v7/elastic_store.go
-
4weed/filer/elastic/v7/elastic_store_kv.go
-
6weed/mount/page_writer/page_chunk_swapfile.go
-
2weed/operation/needle_parse_test.go
-
7weed/s3api/s3api_object_handlers_multipart.go
-
4weed/server/master_grpc_server_volume.go
-
8weed/server/master_ui/master.html
-
15weed/server/master_ui/templates.go
-
158weed/shell/command_ec_common.go
-
133weed/shell/command_ec_common_test.go
-
9weed/shell/command_ec_decode.go
-
4weed/shell/command_ec_encode.go
-
31weed/shell/command_ec_encode_test.go
-
2weed/shell/command_ec_test.go
-
40weed/shell/command_fs_merge_volumes.go
-
13weed/shell/command_fs_verify.go
-
5weed/shell/command_volume_balance_test.go
-
5weed/shell/command_volume_configure_replication.go
-
9weed/shell/command_volume_delete_empty.go
-
13weed/shell/command_volume_fix_replication.go
-
2weed/shell/command_volume_fsck.go
-
8weed/shell/command_volume_list_test.go
-
14weed/shell/command_volume_move.go
-
2weed/shell/command_volume_tier_download.go
-
2weed/shell/command_volume_tier_move.go
-
2weed/shell/commands.go
-
134weed/shell/volume.ecshards.txt
-
7weed/storage/backend/disk_file.go
-
2weed/storage/erasure_coding/ec_decoder.go
-
7weed/storage/erasure_coding/ec_shard.go
-
6weed/storage/erasure_coding/ec_volume.go
-
232weed/storage/needle/needle_parse_upload.go
-
10weed/storage/needle/needle_read.go
-
6weed/storage/needle/needle_read_page.go
-
3weed/storage/needle/needle_write.go
-
6weed/storage/needle_map_metric.go
-
9weed/storage/store_ec.go
-
9weed/storage/super_block/replica_placement.go
-
8weed/storage/super_block/super_block_read.go.go
-
2weed/storage/volume_backup.go
-
12weed/storage/volume_checking.go
-
7weed/topology/volume_layout.go
-
18weed/util/chunk_cache/chunk_cache_on_disk.go
-
2weed/util/constants.go
-
9weed/util/http/http_global_client_util.go
-
79weed/wdclient/exclusive_locks/exclusive_locker.go
@ -1,6 +1,6 @@ |
|||
apiVersion: v1 |
|||
description: SeaweedFS |
|||
name: seaweedfs |
|||
appVersion: "3.79" |
|||
appVersion: "3.80" |
|||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>` |
|||
version: 4.0.379 |
|||
version: 4.0.380 |
|||
@ -0,0 +1,133 @@ |
|||
package shell |
|||
|
|||
import ( |
|||
"fmt" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/needle" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block" |
|||
) |
|||
|
|||
var ( |
|||
topology1 = parseOutput(topoData) |
|||
topology2 = parseOutput(topoData2) |
|||
topologyEc = parseOutput(topoDataEc) |
|||
) |
|||
|
|||
func TestEcDistribution(t *testing.T) { |
|||
|
|||
// find out all volume servers with one slot left.
|
|||
ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(topology1, "") |
|||
|
|||
sortEcNodesByFreeslotsDescending(ecNodes) |
|||
|
|||
if totalFreeEcSlots < erasure_coding.TotalShardsCount { |
|||
t.Errorf("not enough free ec shard slots: %d", totalFreeEcSlots) |
|||
} |
|||
allocatedDataNodes := ecNodes |
|||
if len(allocatedDataNodes) > erasure_coding.TotalShardsCount { |
|||
allocatedDataNodes = allocatedDataNodes[:erasure_coding.TotalShardsCount] |
|||
} |
|||
|
|||
for _, dn := range allocatedDataNodes { |
|||
// fmt.Printf("info %+v %+v\n", dn.info, dn)
|
|||
fmt.Printf("=> %+v %+v\n", dn.info.Id, dn.freeEcSlot) |
|||
} |
|||
} |
|||
|
|||
func TestVolumeIdToReplicaPlacement(t *testing.T) { |
|||
testCases := []struct { |
|||
topology *master_pb.TopologyInfo |
|||
vid string |
|||
want string |
|||
wantErr string |
|||
}{ |
|||
{topology1, "", "", "failed to resolve replica placement for volume ID 0"}, |
|||
{topology1, "0", "", "failed to resolve replica placement for volume ID 0"}, |
|||
{topology1, "1", "100", ""}, |
|||
{topology1, "296", "100", ""}, |
|||
{topology2, "", "", "failed to resolve replica placement for volume ID 0"}, |
|||
{topology2, "19012", "", "failed to resolve replica placement for volume ID 19012"}, |
|||
{topology2, "6271", "002", ""}, |
|||
{topology2, "17932", "002", ""}, |
|||
} |
|||
|
|||
for _, tc := range testCases { |
|||
vid, _ := needle.NewVolumeId(tc.vid) |
|||
ecNodes, _ := collectEcVolumeServersByDc(tc.topology, "") |
|||
got, gotErr := volumeIdToReplicaPlacement(vid, ecNodes) |
|||
|
|||
if tc.wantErr == "" && gotErr != nil { |
|||
t.Errorf("expected no error for volume %q, got %q", tc.vid, gotErr.Error()) |
|||
continue |
|||
} |
|||
if tc.wantErr != "" { |
|||
if gotErr == nil { |
|||
t.Errorf("got no error for volume %q, expected %q", tc.vid, tc.wantErr) |
|||
continue |
|||
} |
|||
if gotErr.Error() != tc.wantErr { |
|||
t.Errorf("expected error %q for volume %q, got %q", tc.wantErr, tc.vid, gotErr.Error()) |
|||
continue |
|||
} |
|||
} |
|||
|
|||
if got == nil { |
|||
if tc.want != "" { |
|||
t.Errorf("expected replica placement %q for volume %q, got nil", tc.want, tc.vid) |
|||
} |
|||
continue |
|||
} |
|||
want, _ := super_block.NewReplicaPlacementFromString(tc.want) |
|||
if !got.Equals(want) { |
|||
t.Errorf("got replica placement %q for volune %q, want %q", got.String(), tc.vid, want.String()) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func TestPickRackToBalanceShardsInto(t *testing.T) { |
|||
testCases := []struct { |
|||
topology *master_pb.TopologyInfo |
|||
vid string |
|||
wantOneOf []string |
|||
}{ |
|||
// Non-EC volumes. We don't care about these, but the function should return all racks as a safeguard.
|
|||
{topologyEc, "", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}}, |
|||
{topologyEc, "6225", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}}, |
|||
{topologyEc, "6226", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}}, |
|||
{topologyEc, "6241", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}}, |
|||
{topologyEc, "6242", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}}, |
|||
// EC volumes.
|
|||
{topologyEc, "9577", []string{"rack1", "rack2", "rack3"}}, |
|||
{topologyEc, "10457", []string{"rack1"}}, |
|||
{topologyEc, "12737", []string{"rack2"}}, |
|||
{topologyEc, "14322", []string{"rack3"}}, |
|||
} |
|||
|
|||
for _, tc := range testCases { |
|||
vid, _ := needle.NewVolumeId(tc.vid) |
|||
ecNodes, _ := collectEcVolumeServersByDc(tc.topology, "") |
|||
racks := collectRacks(ecNodes) |
|||
|
|||
locations := ecNodes |
|||
rackToShardCount := countShardsByRack(vid, locations) |
|||
averageShardsPerEcRack := ceilDivide(erasure_coding.TotalShardsCount, len(racks)) |
|||
|
|||
got := pickRackToBalanceShardsInto(racks, rackToShardCount, nil, averageShardsPerEcRack) |
|||
if string(got) == "" && len(tc.wantOneOf) == 0 { |
|||
continue |
|||
} |
|||
found := false |
|||
for _, want := range tc.wantOneOf { |
|||
if got := string(got); got == want { |
|||
found = true |
|||
break |
|||
} |
|||
} |
|||
if !(found) { |
|||
t.Errorf("expected one of %v for volume %q, got %q", tc.wantOneOf, tc.vid, got) |
|||
} |
|||
} |
|||
} |
|||
@ -1,31 +0,0 @@ |
|||
package shell |
|||
|
|||
import ( |
|||
"fmt" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" |
|||
"testing" |
|||
) |
|||
|
|||
func TestEcDistribution(t *testing.T) { |
|||
|
|||
topologyInfo := parseOutput(topoData) |
|||
|
|||
// find out all volume servers with one slot left.
|
|||
ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(topologyInfo, "") |
|||
|
|||
sortEcNodesByFreeslotsDescending(ecNodes) |
|||
|
|||
if totalFreeEcSlots < erasure_coding.TotalShardsCount { |
|||
println("not enough free ec shard slots", totalFreeEcSlots) |
|||
} |
|||
allocatedDataNodes := ecNodes |
|||
if len(allocatedDataNodes) > erasure_coding.TotalShardsCount { |
|||
allocatedDataNodes = allocatedDataNodes[:erasure_coding.TotalShardsCount] |
|||
} |
|||
|
|||
for _, dn := range allocatedDataNodes { |
|||
// fmt.Printf("info %+v %+v\n", dn.info, dn)
|
|||
fmt.Printf("=> %+v %+v\n", dn.info.Id, dn.freeEcSlot) |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,134 @@ |
|||
Topology volumeSizeLimit:1024 MB hdd(volume:15900/25063 active:15900 free:9163 remote:0) |
|||
DataCenter DefaultDataCenter hdd(volume:15900/25063 active:15900 free:9163 remote:0) |
|||
Rack rack1 hdd(volume:15900/25063 active:15900 free:9163 remote:0) |
|||
DataNode 172.19.0.10:8702 hdd(volume:7/2225 active:7 free:2225 remote:0) |
|||
Disk hdd(volume:7/2232 active:7 free:2225 remote:0) |
|||
volume id:6225 size:24404408 file_count:275 replica_placement:2 version:3 compact_revision:2 modified_at_second:1664897660 |
|||
volume id:6226 size:20871152 file_count:258 replica_placement:2 version:3 compact_revision:2 modified_at_second:1664888660 |
|||
volume id:6241 size:34861224 file_count:274 replica_placement:2 version:3 compact_revision:1 modified_at_second:1664909248 |
|||
volume id:6242 size:40460472 file_count:236 replica_placement:2 version:3 compact_revision:1 modified_at_second:1664906607 |
|||
ec volume id:12737 collection:s3qldata shards:[3] |
|||
ec volume id:14322 collection:s3qldata shards:[5] |
|||
ec volume id:9577 collection:s3qldata shards:[11] |
|||
Disk hdd total size:1737345132344 file_count:533580 deleted_file:10764 deleted_bytes:22028207276 |
|||
DataNode 172.19.0.10:8702 total size:1737345132344 file_count:533580 deleted_file:10764 deleted_bytes:22028207276 |
|||
Rack rack1 total size:17676186754616 file_count:5439969 deleted_file:127907 deleted_bytes:251707271029 |
|||
Rack rack2 hdd(volume:3/25063 active:3 free:25060 remote:0) |
|||
DataNode 172.19.0.13:8701 hdd(volume:3/2187 active:3 free:2184 remote:0) |
|||
Disk hdd(volume:3/2187 active:3 free:2184 remote:0) |
|||
volume id:6241 size:34861256 file_count:275 delete_count:1 replica_placement:2 version:3 compact_revision:1 modified_at_second:1664909248 |
|||
ec volume id:10457 collection:s3qldata shards:[12] |
|||
ec volume id:14322 collection:s3qldata shards:[10] |
|||
ec volume id:9577 collection:s3qldata shards:[10] |
|||
Disk hdd total size:1695600546816 file_count:521054 deleted_file:9961 deleted_bytes:21063702677 |
|||
DataNode 172.19.0.13:8701 total size:1695600546816 file_count:521054 deleted_file:9961 deleted_bytes:21063702677 |
|||
Rack rack2 total size:17676186754616 file_count:5439969 deleted_file:127907 deleted_bytes:251707271029 |
|||
Rack rack3 hdd(volume:3/25063 active:28 free:25060 remote:0) |
|||
DataNode 172.19.0.14:8711 hdd(volume:3/1627 active:3 free:1624 remote:0) |
|||
Disk hdd(volume:3/1627 active:3 free:1624 remote:0) |
|||
ec volume id:10457 collection:s3qldata shards:[3] |
|||
ec volume id:12737 collection:s3qldata shards:[6] |
|||
ec volume id:9577 collection:s3qldata shards:[5] |
|||
Disk hdd total size:1050933775360 file_count:323231 deleted_file:8245 deleted_bytes:15595720358 |
|||
DataNode 172.19.0.14:8711 total size:1050933775360 file_count:323231 deleted_file:8245 deleted_bytes:15595720358 |
|||
Rack rack3 total size:17676186754616 file_count:5439969 deleted_file:127907 deleted_bytes:251707271029 |
|||
Rack rack4 hdd(volume:10/25063 active:4 free:25053 remote:0) |
|||
DataNode 172.19.0.16:8704 hdd(volume:10/2174 active:4 free:2164 remote:0) |
|||
Disk hdd(volume:4/2174 active:4 free:2170 remote:0) |
|||
ec volume id:10457 collection:s3qldata shards:[0 13] |
|||
ec volume id:12737 collection:s3qldata shards:[1] |
|||
ec volume id:14322 collection:s3qldata shards:[7] |
|||
ec volume id:9577 collection:s3qldata shards:[2] |
|||
Disk hdd total size:1653215155776 file_count:507914 deleted_file:11402 deleted_bytes:22641676340 |
|||
DataNode 172.19.0.16:8704 total size:1653215155776 file_count:507914 deleted_file:11402 deleted_bytes:22641676340 |
|||
DataNode 172.19.0.17:8703 hdd(volume:6/2214 active:6 free:2208 remote:0) |
|||
Disk hdd(volume:6/2214 active:6 free:2208 remote:0) |
|||
volume id:6226 size:20871152 file_count:258 replica_placement:2 version:3 compact_revision:2 modified_at_second:1664888660 |
|||
volume id:6241 size:34861256 file_count:275 delete_count:1 replica_placement:2 version:3 compact_revision:1 modified_at_second:1664909248 |
|||
ec volume id:10457 collection:s3qldata shards:[11] |
|||
ec volume id:12737 collection:s3qldata shards:[5] |
|||
ec volume id:14322 collection:s3qldata shards:[2 9] |
|||
ec volume id:9577 collection:s3qldata shards:[0] |
|||
Disk hdd total size:1715724688456 file_count:526901 deleted_file:10854 deleted_bytes:22441405472 |
|||
DataNode 172.19.0.17:8703 total size:1715724688456 file_count:526901 deleted_file:10854 deleted_bytes:22441405472 |
|||
Rack rack4 total size:17676186754616 file_count:5439969 deleted_file:127907 deleted_bytes:251707271029 |
|||
Rack rack5 hdd(volume:20/25063 active:20 free:25043 remote:0) |
|||
DataNode 172.19.0.19:8700 hdd(volume:6/2132 active:6 free:2126 remote:0) |
|||
Disk hdd(volume:6/2132 active:6 free:2126 remote:0) |
|||
volume id:6242 size:40460472 file_count:236 replica_placement:2 version:3 compact_revision:1 modified_at_second:1664906607 |
|||
volume id:6225 size:24398232 file_count:274 replica_placement:2 version:3 compact_revision:2 modified_at_second:1664897660 |
|||
ec volume id:10457 collection:s3qldata shards:[8] |
|||
ec volume id:12737 collection:s3qldata shards:[13] |
|||
ec volume id:14322 collection:s3qldata shards:[8] |
|||
ec volume id:9577 collection:s3qldata shards:[12] |
|||
Disk hdd total size:1635328083064 file_count:504512 deleted_file:11567 deleted_bytes:23202472281 |
|||
DataNode 172.19.0.19:8700 total size:1635328083064 file_count:504512 deleted_file:11567 deleted_bytes:23202472281 |
|||
DataNode 172.19.0.20:8706 hdd(volume:4/2153 active:4 free:2149 remote:0) |
|||
Disk hdd(volume:6/2153 active:1497 free:656 remote:0) |
|||
ec volume id:10457 collection:s3qldata shards:[1] |
|||
ec volume id:12737 collection:s3qldata shards:[7] |
|||
ec volume id:14322 collection:s3qldata shards:[5 11 12] |
|||
ec volume id:9577 collection:s3qldata shards:[1] |
|||
Disk hdd total size:1662887597128 file_count:510323 deleted_file:10919 deleted_bytes:22504428853 |
|||
DataNode 172.19.0.20:8706 total size:1662887597128 file_count:510323 deleted_file:10919 deleted_bytes:22504428853 |
|||
DataNode 172.19.0.21:8710 hdd(volume:6/2184 active:6 free:2178 remote:0) |
|||
Disk hdd(volume:6/2184 active:6 free:2178 remote:0) |
|||
volume id:6225 size:24398232 file_count:274 replica_placement:2 version:3 compact_revision:2 modified_at_second:1664897660 |
|||
volume id:6242 size:40460472 file_count:236 replica_placement:2 version:3 compact_revision:1 modified_at_second:1664906607 |
|||
ec volume id:10457 collection:s3qldata shards:[9] |
|||
ec volume id:12737 collection:s3qldata shards:[4] |
|||
ec volume id:14322 collection:s3qldata shards:[11] |
|||
ec volume id:9577 collection:s3qldata shards:[3] |
|||
Disk hdd total size:1685060737528 file_count:517231 deleted_file:10635 deleted_bytes:22306836236 |
|||
DataNode 172.19.0.21:8710 total size:1685060737528 file_count:517231 deleted_file:10635 deleted_bytes:22306836236 |
|||
DataNode 172.19.0.3:8708 hdd(volume:4/961 active:4 free:957 remote:0) |
|||
Disk hdd(volume:4/961 active:4 free:957 remote:0) |
|||
ec volume id:10457 collection:s3qldata shards:[4] |
|||
ec volume id:12737 collection:s3qldata shards:[10] |
|||
ec volume id:14322 collection:s3qldata shards:[3] |
|||
ec volume id:9577 collection:s3qldata shards:[7] |
|||
Disk hdd total size:377523488192 file_count:119577 deleted_file:5368 deleted_bytes:8596766559 |
|||
DataNode 172.19.0.3:8708 total size:377523488192 file_count:119577 deleted_file:5368 deleted_bytes:8596766559 |
|||
Rack rack5 total size:17676186754616 file_count:5439969 deleted_file:127907 deleted_bytes:251707271029 |
|||
Rack rack6 hdd(volume:18/25063 active:18 free:25045 remote:0) |
|||
DataNode 172.19.0.4:8707 hdd(volume:4/958 active:4 free:954 remote:0) |
|||
Disk hdd(volume:4/958 active:4 free:954 remote:0) |
|||
ec volume id:10457 collection:s3qldata shards:[6] |
|||
ec volume id:12737 collection:s3qldata shards:[9] |
|||
ec volume id:14322 collection:s3qldata shards:[4] |
|||
ec volume id:9577 collection:s3qldata shards:[9] |
|||
Disk hdd total size:378345005760 file_count:119036 deleted_file:5301 deleted_bytes:8574028334 |
|||
DataNode 172.19.0.4:8707 total size:378345005760 file_count:119036 deleted_file:5301 deleted_bytes:8574028334 |
|||
DataNode 172.19.0.5:8705 hdd(volume:3/983 active:3 free:980 remote:0) |
|||
Disk hdd(volume:3/983 active:3 free:980 remote:0) |
|||
ec volume id:10457 collection:s3qldata shards:[5] |
|||
ec volume id:12737 collection:s3qldata shards:[8] |
|||
ec volume id:9577 collection:s3qldata shards:[6] |
|||
Disk hdd total size:404653451288 file_count:126527 deleted_file:4789 deleted_bytes:8145619860 |
|||
DataNode 172.19.0.5:8705 total size:404653451288 file_count:126527 deleted_file:4789 deleted_bytes:8145619860 |
|||
DataNode 172.19.0.6:8713 hdd(volume:2/970 active:2 free:968 remote:0) |
|||
Disk hdd(volume:2/970 active:2 free:968 remote:0) |
|||
ec volume id:12737 collection:s3qldata shards:[11] |
|||
ec volume id:9577 collection:s3qldata shards:[8] |
|||
Disk hdd total size:401028073512 file_count:125448 deleted_file:4891 deleted_bytes:7914078769 |
|||
DataNode 172.19.0.6:8713 total size:401028073512 file_count:125448 deleted_file:4891 deleted_bytes:7914078769 |
|||
DataNode 172.19.0.8:8709 hdd(volume:5/2144 active:5 free:2139 remote:0) |
|||
Disk hdd(volume:5/2144 active:5 free:2139 remote:0) |
|||
volume id:6226 size:20871152 file_count:258 replica_placement:2 version:3 compact_revision:2 modified_at_second:1664888660 |
|||
ec volume id:10457 collection:s3qldata shards:[2] |
|||
ec volume id:12737 collection:s3qldata shards:[2 12] |
|||
ec volume id:14322 collection:s3qldata shards:[1 13] |
|||
ec volume id:9577 collection:s3qldata shards:[13] |
|||
Disk hdd total size:1648662273096 file_count:507133 deleted_file:11386 deleted_bytes:23141702025 |
|||
DataNode 172.19.0.8:8709 total size:1648662273096 file_count:507133 deleted_file:11386 deleted_bytes:23141702025 |
|||
DataNode 172.19.0.9:8712 hdd(volume:4/2144 active:4 free:2140 remote:0) |
|||
Disk hdd(volume:4/2144 active:4 free:2140 remote:0) |
|||
ec volume id:10457 collection:s3qldata shards:[7] |
|||
ec volume id:12737 collection:s3qldata shards:[0] |
|||
ec volume id:14322 collection:s3qldata shards:[0 6] |
|||
ec volume id:9577 collection:s3qldata shards:[4] |
|||
Disk hdd total size:1629878746296 file_count:497502 deleted_file:11825 deleted_bytes:23550625989 |
|||
DataNode 172.19.0.9:8712 total size:1629878746296 file_count:497502 deleted_file:11825 deleted_bytes:23550625989 |
|||
Rack rack6 total size:17676186754616 file_count:5439969 deleted_file:127907 deleted_bytes:251707271029 |
|||
DataCenter DefaultDataCenter total size:17676186754616 file_count:5439969 deleted_file:127907 deleted_bytes:251707271029 |
|||
total size:17676186754616 file_count:5439969 deleted_file:127907 deleted_bytes:251707271029 |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue