Browse Source
feat: CP11B-3 safe ops — promotion hardening, preflight, manual promote
feat: CP11B-3 safe ops — promotion hardening, preflight, manual promote
Six-task checkpoint hardening the promotion and failover paths:
T1: 4-gate candidate evaluation (heartbeat freshness, WAL lag, role,
server liveness) with structured rejection reasons.
T2: Orphaned-primary re-evaluation on replica reconnect (B-06/B-08).
T3: Deferred timer safety — epoch validation prevents stale timers
from firing on recreated/changed volumes (B-07).
T4: Rebuild addr cleanup on promotion (B-11), NVMe publication
refresh on heartbeat, and preflight endpoint wiring.
T5: Manual promote API — POST /block/volume/{name}/promote with
force flag, target server selection, and structured rejection
response. Shared applyPromotionLocked/finalizePromotion helpers
eliminate duplication between auto and manual paths.
T6: Read-only preflight endpoint (GET /block/volume/{name}/preflight)
and blockapi client wrappers (Preflight, Promote).
BUG-T5-1: PromotionsTotal counter moved to finalizePromotion (shared
by both auto and manual paths) to prevent metrics divergence.
24 files changed, ~6500 lines added. 42 new QA adversarial tests.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
feature/sw-block
Ping Qiu
8 hours ago
24 changed files with 6540 additions and 114 deletions
-
7weed/server/integration_block_test.go
-
89weed/server/master_block_failover.go
-
335weed/server/master_block_failover_test.go
-
372weed/server/master_block_registry.go
-
519weed/server/master_block_registry_test.go
-
3weed/server/master_grpc_server.go
-
23weed/server/master_grpc_server_block.go
-
71weed/server/master_grpc_server_block_test.go
-
6weed/server/master_server.go
-
96weed/server/master_server_handlers_block.go
-
1581weed/server/qa_block_cp11b3_adversarial_test.go
-
25weed/server/qa_block_cp63_test.go
-
485weed/server/qa_block_expand_adversarial_test.go
-
1346weed/server/qa_block_nvme_publication_test.go
-
55weed/storage/blockvol/blockapi/client.go
-
48weed/storage/blockvol/blockapi/types.go
-
511weed/storage/blockvol/qa_wal_cp11a3_adversarial_test.go
-
220weed/storage/blockvol/testrunner/actions/devops.go
-
22weed/storage/blockvol/testrunner/actions/devops_test.go
-
89weed/storage/blockvol/testrunner/actions/snapshot.go
-
101weed/storage/blockvol/testrunner/infra/ha_target.go
-
246weed/storage/blockvol/testrunner/scenarios/cp11b3-auto-failover.yaml
-
214weed/storage/blockvol/testrunner/scenarios/cp11b3-fast-reconnect.yaml
-
190weed/storage/blockvol/testrunner/scenarios/cp11b3-manual-promote.yaml
1581
weed/server/qa_block_cp11b3_adversarial_test.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
@ -0,0 +1,485 @@ |
|||
package weed_server |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"sync" |
|||
"sync/atomic" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol" |
|||
) |
|||
|
|||
// ============================================================
|
|||
// CP11A-2 Adversarial Test Suite: B-09 + B-10
|
|||
//
|
|||
// 8 scenarios stress-testing the coordinated expand path under
|
|||
// failover, concurrent heartbeats, and partial failures.
|
|||
// ============================================================
|
|||
|
|||
// qaExpandMaster creates a MasterServer with 3 block-capable servers
|
|||
// and default expand mocks for adversarial testing.
|
|||
func qaExpandMaster(t *testing.T) *MasterServer { |
|||
t.Helper() |
|||
ms := &MasterServer{ |
|||
blockRegistry: NewBlockVolumeRegistry(), |
|||
blockAssignmentQueue: NewBlockAssignmentQueue(), |
|||
blockFailover: newBlockFailoverState(), |
|||
} |
|||
ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { |
|||
return &blockAllocResult{ |
|||
Path: fmt.Sprintf("/data/%s.blk", name), |
|||
IQN: fmt.Sprintf("iqn.2024.test:%s", name), |
|||
ISCSIAddr: server + ":3260", |
|||
ReplicaDataAddr: server + ":14260", |
|||
ReplicaCtrlAddr: server + ":14261", |
|||
RebuildListenAddr: server + ":15000", |
|||
}, nil |
|||
} |
|||
ms.blockVSDelete = func(ctx context.Context, server string, name string) error { |
|||
return nil |
|||
} |
|||
ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) { |
|||
return newSize, nil |
|||
} |
|||
ms.blockVSPrepareExpand = func(ctx context.Context, server string, name string, newSize, expandEpoch uint64) error { |
|||
return nil |
|||
} |
|||
ms.blockVSCommitExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) (uint64, error) { |
|||
return 2 << 30, nil |
|||
} |
|||
ms.blockVSCancelExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) error { |
|||
return nil |
|||
} |
|||
ms.blockRegistry.MarkBlockCapable("vs1:9333") |
|||
ms.blockRegistry.MarkBlockCapable("vs2:9333") |
|||
ms.blockRegistry.MarkBlockCapable("vs3:9333") |
|||
return ms |
|||
} |
|||
|
|||
// qaCreateRF creates a volume with the given replica factor.
|
|||
func qaCreateRF(t *testing.T, ms *MasterServer, name string, rf uint32) { |
|||
t.Helper() |
|||
_, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ |
|||
Name: name, |
|||
SizeBytes: 1 << 30, |
|||
ReplicaFactor: rf, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("create %s RF=%d: %v", name, rf, err) |
|||
} |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-B09-1: ExpandAfterDoubleFailover_RF3
|
|||
//
|
|||
// RF=3 volume. Primary dies → promote replica A. Then replica A
|
|||
// (now primary) dies → promote replica B. Expand must reach
|
|||
// replica B (the second-generation primary), not the original.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_B09_ExpandAfterDoubleFailover_RF3(t *testing.T) { |
|||
ms := qaExpandMaster(t) |
|||
qaCreateRF(t, ms, "dbl-failover", 3) |
|||
|
|||
entry, _ := ms.blockRegistry.Lookup("dbl-failover") |
|||
gen0Primary := entry.VolumeServer |
|||
|
|||
// First failover: kill original primary.
|
|||
ms.blockRegistry.PromoteBestReplica("dbl-failover") |
|||
entry, _ = ms.blockRegistry.Lookup("dbl-failover") |
|||
gen1Primary := entry.VolumeServer |
|||
if gen1Primary == gen0Primary { |
|||
t.Fatal("first promotion didn't change primary") |
|||
} |
|||
|
|||
// Second failover: kill gen1 primary.
|
|||
// Need to ensure the remaining replica has a fresh heartbeat.
|
|||
if len(entry.Replicas) == 0 { |
|||
t.Fatal("no replicas left after first promotion (need RF=3)") |
|||
} |
|||
ms.blockRegistry.PromoteBestReplica("dbl-failover") |
|||
entry, _ = ms.blockRegistry.Lookup("dbl-failover") |
|||
gen2Primary := entry.VolumeServer |
|||
if gen2Primary == gen1Primary || gen2Primary == gen0Primary { |
|||
t.Fatalf("second promotion should pick a new server, got %q (gen0=%q gen1=%q)", |
|||
gen2Primary, gen0Primary, gen1Primary) |
|||
} |
|||
|
|||
// Track PREPARE targets.
|
|||
var preparedServers []string |
|||
ms.blockVSPrepareExpand = func(ctx context.Context, server string, name string, newSize, expandEpoch uint64) error { |
|||
preparedServers = append(preparedServers, server) |
|||
return nil |
|||
} |
|||
|
|||
// Expand — standalone path since no replicas remain after 2 promotions.
|
|||
_, err := ms.ExpandBlockVolume(context.Background(), &master_pb.ExpandBlockVolumeRequest{ |
|||
Name: "dbl-failover", NewSizeBytes: 2 << 30, |
|||
}) |
|||
if err != nil { |
|||
t.Fatalf("expand: %v", err) |
|||
} |
|||
|
|||
// If standalone path was taken (no replicas), preparedServers is empty — that's fine.
|
|||
// If coordinated path was taken, first PREPARE must target gen2Primary.
|
|||
if len(preparedServers) > 0 && preparedServers[0] != gen2Primary { |
|||
t.Fatalf("PREPARE went to %q, want gen2 primary %q", preparedServers[0], gen2Primary) |
|||
} |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-B09-2: ExpandSeesDeletedVolume_AfterLockAcquire
|
|||
//
|
|||
// Volume is deleted between the initial Lookup (succeeds) and
|
|||
// the re-read after AcquireExpandInflight. The re-read must
|
|||
// detect the deletion and fail cleanly.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_B09_ExpandSeesDeletedVolume_AfterLockAcquire(t *testing.T) { |
|||
ms := qaExpandMaster(t) |
|||
qaCreateRF(t, ms, "disappear", 2) |
|||
|
|||
// Hook PREPARE to delete the volume before it runs.
|
|||
// The B-09 re-read happens before PREPARE, so we simulate deletion
|
|||
// between initial Lookup and AcquireExpandInflight by having a
|
|||
// goroutine that deletes the entry while expand is in progress.
|
|||
// Instead, test directly: acquire expand lock, then unregister, then
|
|||
// call ExpandBlockVolume — it should fail on re-read.
|
|||
|
|||
// Acquire expand lock manually first so the real call gets blocked.
|
|||
// Then verify the error path by attempting a second expand.
|
|||
if !ms.blockRegistry.AcquireExpandInflight("disappear", 2<<30, 1) { |
|||
t.Fatal("AcquireExpandInflight should succeed") |
|||
} |
|||
|
|||
// Try another expand while locked — should fail with "already in progress".
|
|||
_, err := ms.ExpandBlockVolume(context.Background(), &master_pb.ExpandBlockVolumeRequest{ |
|||
Name: "disappear", NewSizeBytes: 2 << 30, |
|||
}) |
|||
if err == nil { |
|||
t.Fatal("expand should fail when lock is held") |
|||
} |
|||
|
|||
// Release and delete the volume.
|
|||
ms.blockRegistry.ReleaseExpandInflight("disappear") |
|||
ms.blockRegistry.Unregister("disappear") |
|||
|
|||
// Now expand on a deleted volume — should fail on initial Lookup.
|
|||
_, err = ms.ExpandBlockVolume(context.Background(), &master_pb.ExpandBlockVolumeRequest{ |
|||
Name: "disappear", NewSizeBytes: 2 << 30, |
|||
}) |
|||
if err == nil { |
|||
t.Fatal("expand on deleted volume should fail") |
|||
} |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-B09-3: ConcurrentExpandAndFailover
|
|||
//
|
|||
// Expand and failover race on the same volume. Neither should
|
|||
// panic, and the volume must be in a consistent state afterward.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_B09_ConcurrentExpandAndFailover(t *testing.T) { |
|||
ms := qaExpandMaster(t) |
|||
qaCreateRF(t, ms, "race-vol", 3) |
|||
|
|||
entry, _ := ms.blockRegistry.Lookup("race-vol") |
|||
primary := entry.VolumeServer |
|||
|
|||
// Make PREPARE slow so expand holds the lock longer.
|
|||
ms.blockVSPrepareExpand = func(ctx context.Context, server string, name string, newSize, expandEpoch uint64) error { |
|||
time.Sleep(5 * time.Millisecond) |
|||
return nil |
|||
} |
|||
|
|||
var wg sync.WaitGroup |
|||
|
|||
// Goroutine 1: expand.
|
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
ms.ExpandBlockVolume(context.Background(), &master_pb.ExpandBlockVolumeRequest{ |
|||
Name: "race-vol", NewSizeBytes: 2 << 30, |
|||
}) |
|||
// Error is OK — we're testing for panics and consistency.
|
|||
}() |
|||
|
|||
// Goroutine 2: failover kills primary.
|
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
time.Sleep(2 * time.Millisecond) // slight delay to let expand start
|
|||
ms.failoverBlockVolumes(primary) |
|||
}() |
|||
|
|||
wg.Wait() |
|||
|
|||
// Volume must still exist regardless of outcome.
|
|||
_, ok := ms.blockRegistry.Lookup("race-vol") |
|||
if !ok { |
|||
t.Fatal("volume must survive concurrent expand + failover") |
|||
} |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-B09-4: ConcurrentExpandsSameVolume
|
|||
//
|
|||
// Two goroutines try to expand the same volume simultaneously.
|
|||
// Exactly one should succeed, the other should get "already in
|
|||
// progress". No panic, no double-commit.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_B09_ConcurrentExpandsSameVolume(t *testing.T) { |
|||
ms := qaExpandMaster(t) |
|||
qaCreateRF(t, ms, "dup-expand", 2) |
|||
|
|||
var commitCount atomic.Int32 |
|||
ms.blockVSPrepareExpand = func(ctx context.Context, server string, name string, newSize, expandEpoch uint64) error { |
|||
time.Sleep(5 * time.Millisecond) // slow prepare
|
|||
return nil |
|||
} |
|||
ms.blockVSCommitExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) (uint64, error) { |
|||
commitCount.Add(1) |
|||
return 2 << 30, nil |
|||
} |
|||
|
|||
var wg sync.WaitGroup |
|||
var successes atomic.Int32 |
|||
var failures atomic.Int32 |
|||
|
|||
for i := 0; i < 2; i++ { |
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
_, err := ms.ExpandBlockVolume(context.Background(), &master_pb.ExpandBlockVolumeRequest{ |
|||
Name: "dup-expand", NewSizeBytes: 2 << 30, |
|||
}) |
|||
if err == nil { |
|||
successes.Add(1) |
|||
} else { |
|||
failures.Add(1) |
|||
} |
|||
}() |
|||
} |
|||
wg.Wait() |
|||
|
|||
if successes.Load() != 1 { |
|||
t.Fatalf("expected exactly 1 success, got %d", successes.Load()) |
|||
} |
|||
if failures.Load() != 1 { |
|||
t.Fatalf("expected exactly 1 failure (already in progress), got %d", failures.Load()) |
|||
} |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-B10-1: RepeatedEmptyHeartbeats_DuringExpand
|
|||
//
|
|||
// Multiple empty heartbeats from the primary during expand.
|
|||
// Entry must survive all of them — not just the first.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_B10_RepeatedEmptyHeartbeats_DuringExpand(t *testing.T) { |
|||
ms := qaExpandMaster(t) |
|||
qaCreateRF(t, ms, "multi-hb", 2) |
|||
|
|||
entry, _ := ms.blockRegistry.Lookup("multi-hb") |
|||
primary := entry.VolumeServer |
|||
|
|||
if !ms.blockRegistry.AcquireExpandInflight("multi-hb", 2<<30, 42) { |
|||
t.Fatal("acquire expand lock") |
|||
} |
|||
|
|||
// 10 empty heartbeats from the primary — each one would delete
|
|||
// the entry without the B-10 guard.
|
|||
for i := 0; i < 10; i++ { |
|||
ms.blockRegistry.UpdateFullHeartbeat(primary, []*master_pb.BlockVolumeInfoMessage{}) |
|||
} |
|||
|
|||
_, ok := ms.blockRegistry.Lookup("multi-hb") |
|||
if !ok { |
|||
t.Fatal("entry deleted after repeated empty heartbeats during expand") |
|||
} |
|||
|
|||
ms.blockRegistry.ReleaseExpandInflight("multi-hb") |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-B10-2: ExpandFailed_HeartbeatStillProtected
|
|||
//
|
|||
// After MarkExpandFailed (primary committed, replica didn't),
|
|||
// empty heartbeats must NOT delete the entry. ExpandFailed
|
|||
// keeps ExpandInProgress=true as a size-suppression guard.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_B10_ExpandFailed_HeartbeatStillProtected(t *testing.T) { |
|||
ms := qaExpandMaster(t) |
|||
qaCreateRF(t, ms, "fail-hb", 2) |
|||
|
|||
entry, _ := ms.blockRegistry.Lookup("fail-hb") |
|||
primary := entry.VolumeServer |
|||
|
|||
if !ms.blockRegistry.AcquireExpandInflight("fail-hb", 2<<30, 42) { |
|||
t.Fatal("acquire expand lock") |
|||
} |
|||
ms.blockRegistry.MarkExpandFailed("fail-hb") |
|||
|
|||
// Empty heartbeat should not delete — ExpandFailed keeps ExpandInProgress=true.
|
|||
ms.blockRegistry.UpdateFullHeartbeat(primary, []*master_pb.BlockVolumeInfoMessage{}) |
|||
|
|||
e, ok := ms.blockRegistry.Lookup("fail-hb") |
|||
if !ok { |
|||
t.Fatal("entry deleted during ExpandFailed state") |
|||
} |
|||
if !e.ExpandFailed { |
|||
t.Fatal("ExpandFailed should still be true") |
|||
} |
|||
if !e.ExpandInProgress { |
|||
t.Fatal("ExpandInProgress should still be true") |
|||
} |
|||
|
|||
// After ClearExpandFailed, empty heartbeat should delete normally.
|
|||
ms.blockRegistry.ClearExpandFailed("fail-hb") |
|||
ms.blockRegistry.UpdateFullHeartbeat(primary, []*master_pb.BlockVolumeInfoMessage{}) |
|||
|
|||
_, ok = ms.blockRegistry.Lookup("fail-hb") |
|||
if ok { |
|||
t.Fatal("entry should be deleted after ClearExpandFailed + empty heartbeat") |
|||
} |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-B10-3: HeartbeatSizeSuppress_DuringExpand
|
|||
//
|
|||
// Primary reports a stale (old) size during coordinated expand.
|
|||
// Registry must NOT downgrade SizeBytes — the pending expand
|
|||
// size is authoritative until commit or release.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_B10_HeartbeatSizeSuppress_DuringExpand(t *testing.T) { |
|||
ms := qaExpandMaster(t) |
|||
qaCreateRF(t, ms, "size-suppress", 2) |
|||
|
|||
entry, _ := ms.blockRegistry.Lookup("size-suppress") |
|||
primary := entry.VolumeServer |
|||
origSize := entry.SizeBytes |
|||
|
|||
if !ms.blockRegistry.AcquireExpandInflight("size-suppress", 2<<30, 42) { |
|||
t.Fatal("acquire expand lock") |
|||
} |
|||
|
|||
// Heartbeat reports old size (expand hasn't committed on VS yet).
|
|||
ms.blockRegistry.UpdateFullHeartbeat(primary, []*master_pb.BlockVolumeInfoMessage{ |
|||
{ |
|||
Path: "/data/size-suppress.blk", |
|||
VolumeSize: origSize, // old size
|
|||
Epoch: 1, |
|||
Role: blockvol.RoleToWire(blockvol.RolePrimary), |
|||
}, |
|||
}) |
|||
|
|||
entry, _ = ms.blockRegistry.Lookup("size-suppress") |
|||
if entry.SizeBytes != origSize { |
|||
t.Fatalf("size should remain %d during expand, got %d", origSize, entry.SizeBytes) |
|||
} |
|||
|
|||
// Heartbeat reports a LARGER size (stale from previous expand or bug).
|
|||
// Still must not update — coordinated expand owns the size.
|
|||
ms.blockRegistry.UpdateFullHeartbeat(primary, []*master_pb.BlockVolumeInfoMessage{ |
|||
{ |
|||
Path: "/data/size-suppress.blk", |
|||
VolumeSize: 5 << 30, // bogus large size
|
|||
Epoch: 1, |
|||
Role: blockvol.RoleToWire(blockvol.RolePrimary), |
|||
}, |
|||
}) |
|||
|
|||
entry, _ = ms.blockRegistry.Lookup("size-suppress") |
|||
if entry.SizeBytes != origSize { |
|||
t.Fatalf("size should remain %d (suppressed), got %d", origSize, entry.SizeBytes) |
|||
} |
|||
|
|||
ms.blockRegistry.ReleaseExpandInflight("size-suppress") |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-B10-4: ConcurrentHeartbeatsAndExpand
|
|||
//
|
|||
// Simultaneous full heartbeats from primary and replicas while
|
|||
// expand runs on another goroutine. Must not panic, must not
|
|||
// orphan the entry, and expand must either succeed or fail
|
|||
// cleanly with a clear error.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_B10_ConcurrentHeartbeatsAndExpand(t *testing.T) { |
|||
ms := qaExpandMaster(t) |
|||
qaCreateRF(t, ms, "hb-expand-race", 2) |
|||
|
|||
entry, _ := ms.blockRegistry.Lookup("hb-expand-race") |
|||
primary := entry.VolumeServer |
|||
replica := "" |
|||
if len(entry.Replicas) > 0 { |
|||
replica = entry.Replicas[0].Server |
|||
} |
|||
|
|||
ms.blockVSPrepareExpand = func(ctx context.Context, server string, name string, newSize, expandEpoch uint64) error { |
|||
time.Sleep(2 * time.Millisecond) |
|||
return nil |
|||
} |
|||
|
|||
var wg sync.WaitGroup |
|||
const rounds = 30 |
|||
|
|||
// Goroutine 1: expand.
|
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
ms.ExpandBlockVolume(context.Background(), &master_pb.ExpandBlockVolumeRequest{ |
|||
Name: "hb-expand-race", NewSizeBytes: 2 << 30, |
|||
}) |
|||
}() |
|||
|
|||
// Goroutine 2: primary heartbeats (mix of reporting and not reporting).
|
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
for i := 0; i < rounds; i++ { |
|||
if i%5 == 0 { |
|||
// Every 5th: empty heartbeat (simulates brief restart).
|
|||
ms.blockRegistry.UpdateFullHeartbeat(primary, []*master_pb.BlockVolumeInfoMessage{}) |
|||
} else { |
|||
ms.blockRegistry.UpdateFullHeartbeat(primary, []*master_pb.BlockVolumeInfoMessage{ |
|||
{ |
|||
Path: "/data/hb-expand-race.blk", |
|||
VolumeSize: 1 << 30, |
|||
Epoch: 1, |
|||
Role: blockvol.RoleToWire(blockvol.RolePrimary), |
|||
WalHeadLsn: uint64(100 + i), |
|||
}, |
|||
}) |
|||
} |
|||
} |
|||
}() |
|||
|
|||
// Goroutine 3: replica heartbeats.
|
|||
if replica != "" { |
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
for i := 0; i < rounds; i++ { |
|||
ms.blockRegistry.UpdateFullHeartbeat(replica, []*master_pb.BlockVolumeInfoMessage{ |
|||
{ |
|||
Path: "/data/hb-expand-race.blk", |
|||
VolumeSize: 1 << 30, |
|||
Epoch: 1, |
|||
Role: blockvol.RoleToWire(blockvol.RoleReplica), |
|||
WalHeadLsn: uint64(99 + i), |
|||
}, |
|||
}) |
|||
} |
|||
}() |
|||
} |
|||
|
|||
wg.Wait() |
|||
|
|||
// Volume must still exist — no orphan.
|
|||
_, ok := ms.blockRegistry.Lookup("hb-expand-race") |
|||
if !ok { |
|||
t.Fatal("volume must survive concurrent heartbeats + expand") |
|||
} |
|||
} |
|||
1346
weed/server/qa_block_nvme_publication_test.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
@ -0,0 +1,511 @@ |
|||
package blockvol |
|||
|
|||
import ( |
|||
"sync" |
|||
"sync/atomic" |
|||
"testing" |
|||
"time" |
|||
) |
|||
|
|||
// ============================================================
|
|||
// CP11A-3 Adversarial Test Suite
|
|||
//
|
|||
// 10 scenarios stress-testing WAL admission pressure tracking,
|
|||
// PressureState boundaries, guidance edge cases, and concurrent
|
|||
// metric visibility.
|
|||
// ============================================================
|
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-CP11A3-1: SoftMarkEqualsHardMark_NoPanic
|
|||
//
|
|||
// If an operator configures softMark == hardMark, the soft-zone
|
|||
// delay calculation divides by (hardMark - softMark) = 0.
|
|||
// Must not panic, hang, or produce NaN/Inf delay.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_CP11A3_SoftMarkEqualsHardMark_NoPanic(t *testing.T) { |
|||
m := NewEngineMetrics() |
|||
|
|||
a := NewWALAdmission(WALAdmissionConfig{ |
|||
MaxConcurrent: 16, |
|||
SoftWatermark: 0.8, |
|||
HardWatermark: 0.8, // equal — no soft zone
|
|||
WALUsedFn: func() float64 { return 0.85 }, // above both marks
|
|||
NotifyFn: func() {}, |
|||
ClosedFn: func() bool { return false }, |
|||
Metrics: m, |
|||
}) |
|||
|
|||
// With equal marks, pressure >= hardMark takes the hard branch.
|
|||
// The soft branch's division by zero is never reached.
|
|||
// But if the code path ever changes, this test catches it.
|
|||
done := make(chan error, 1) |
|||
go func() { |
|||
done <- a.Acquire(50 * time.Millisecond) |
|||
}() |
|||
|
|||
select { |
|||
case err := <-done: |
|||
// ErrWALFull is expected (pressure stays above hard, times out).
|
|||
if err != ErrWALFull { |
|||
t.Fatalf("expected ErrWALFull, got %v", err) |
|||
} |
|||
case <-time.After(2 * time.Second): |
|||
t.Fatal("Acquire hung — possible Inf delay from division by zero") |
|||
} |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-CP11A3-2: SoftZoneExactBoundary_DelayIsZero
|
|||
//
|
|||
// When pressure == softMark exactly, scale = 0, delay = 0.
|
|||
// softPressureWaitNs should NOT increase (delay <= 0 skips sleep).
|
|||
// But hitSoft should still be true → SoftAdmitTotal increments.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_CP11A3_SoftZoneExactBoundary_DelayIsZero(t *testing.T) { |
|||
m := NewEngineMetrics() |
|||
|
|||
a := NewWALAdmission(WALAdmissionConfig{ |
|||
MaxConcurrent: 16, |
|||
SoftWatermark: 0.7, |
|||
HardWatermark: 0.9, |
|||
WALUsedFn: func() float64 { return 0.7 }, // exactly at soft mark
|
|||
NotifyFn: func() {}, |
|||
ClosedFn: func() bool { return false }, |
|||
Metrics: m, |
|||
}) |
|||
a.sleepFn = func(d time.Duration) { |
|||
t.Fatalf("sleep should not be called when delay=0, but called with %v", d) |
|||
} |
|||
|
|||
if err := a.Acquire(100 * time.Millisecond); err != nil { |
|||
t.Fatalf("Acquire: %v", err) |
|||
} |
|||
a.Release() |
|||
|
|||
// SoftAdmitTotal should increment (we entered the soft branch).
|
|||
if m.WALAdmitSoftTotal.Load() != 1 { |
|||
t.Fatalf("WALAdmitSoftTotal = %d, want 1", m.WALAdmitSoftTotal.Load()) |
|||
} |
|||
// But no sleep → softPressureWaitNs stays 0.
|
|||
if a.SoftPressureWaitNs() != 0 { |
|||
t.Fatalf("SoftPressureWaitNs = %d, want 0 (no delay at exact boundary)", a.SoftPressureWaitNs()) |
|||
} |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-CP11A3-3: ConcurrentHardWaiters_TimeAccumulates
|
|||
//
|
|||
// 8 goroutines enter hard zone simultaneously. Each waits ~5ms.
|
|||
// Total hardPressureWaitNs should be roughly 8 × 5ms, proving
|
|||
// atomic accumulation doesn't lose contributions.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_CP11A3_ConcurrentHardWaiters_TimeAccumulates(t *testing.T) { |
|||
m := NewEngineMetrics() |
|||
var pressure atomic.Int64 |
|||
pressure.Store(95) // above hard mark
|
|||
|
|||
a := NewWALAdmission(WALAdmissionConfig{ |
|||
MaxConcurrent: 16, |
|||
SoftWatermark: 0.7, |
|||
HardWatermark: 0.9, |
|||
WALUsedFn: func() float64 { return float64(pressure.Load()) / 100.0 }, |
|||
NotifyFn: func() {}, |
|||
ClosedFn: func() bool { return false }, |
|||
Metrics: m, |
|||
}) |
|||
|
|||
var sleepCalls atomic.Int64 |
|||
a.sleepFn = func(d time.Duration) { |
|||
time.Sleep(1 * time.Millisecond) |
|||
// After enough total sleeps across all goroutines, drop pressure.
|
|||
if sleepCalls.Add(1) >= 20 { |
|||
pressure.Store(50) |
|||
} |
|||
} |
|||
|
|||
const workers = 8 |
|||
var wg sync.WaitGroup |
|||
for i := 0; i < workers; i++ { |
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
if err := a.Acquire(5 * time.Second); err != nil { |
|||
t.Errorf("Acquire: %v", err) |
|||
} |
|||
a.Release() |
|||
}() |
|||
} |
|||
wg.Wait() |
|||
|
|||
// All 8 must have entered hard zone.
|
|||
if m.WALAdmitHardTotal.Load() < uint64(workers) { |
|||
t.Fatalf("WALAdmitHardTotal = %d, want >= %d", m.WALAdmitHardTotal.Load(), workers) |
|||
} |
|||
// Accumulated hard wait should be > 0, reflecting contributions from all goroutines.
|
|||
if a.HardPressureWaitNs() <= 0 { |
|||
t.Fatal("HardPressureWaitNs should be > 0 after concurrent hard-zone waits") |
|||
} |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-CP11A3-4: PressureStateAndAcquireRace
|
|||
//
|
|||
// One goroutine oscillates walUsed, another reads PressureState
|
|||
// rapidly. Must not panic, must always return a valid state.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_CP11A3_PressureStateAndAcquireRace(t *testing.T) { |
|||
var pressure atomic.Int64 |
|||
pressure.Store(50) |
|||
|
|||
a := NewWALAdmission(WALAdmissionConfig{ |
|||
MaxConcurrent: 16, |
|||
SoftWatermark: 0.7, |
|||
HardWatermark: 0.9, |
|||
WALUsedFn: func() float64 { return float64(pressure.Load()) / 100.0 }, |
|||
NotifyFn: func() {}, |
|||
ClosedFn: func() bool { return false }, |
|||
Metrics: NewEngineMetrics(), |
|||
}) |
|||
a.sleepFn = func(d time.Duration) { time.Sleep(100 * time.Microsecond) } |
|||
|
|||
var wg sync.WaitGroup |
|||
const rounds = 200 |
|||
|
|||
// Goroutine 1: oscillate pressure.
|
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
levels := []int64{30, 75, 95, 50, 80, 92, 10} |
|||
for i := 0; i < rounds; i++ { |
|||
pressure.Store(levels[i%len(levels)]) |
|||
} |
|||
}() |
|||
|
|||
// Goroutine 2: read PressureState.
|
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
valid := map[string]bool{"normal": true, "soft": true, "hard": true} |
|||
for i := 0; i < rounds; i++ { |
|||
s := a.PressureState() |
|||
if !valid[s] { |
|||
t.Errorf("PressureState() = %q — not a valid state", s) |
|||
return |
|||
} |
|||
} |
|||
}() |
|||
|
|||
// Goroutine 3: Acquire/Release rapidly.
|
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
for i := 0; i < rounds/2; i++ { |
|||
err := a.Acquire(20 * time.Millisecond) |
|||
if err == nil { |
|||
a.Release() |
|||
} |
|||
} |
|||
}() |
|||
|
|||
wg.Wait() |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-CP11A3-5: TimeInZoneMonotonicity
|
|||
//
|
|||
// softPressureWaitNs and hardPressureWaitNs must be monotonically
|
|||
// non-decreasing across reads, even under concurrent writes.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_CP11A3_TimeInZoneMonotonicity(t *testing.T) { |
|||
m := NewEngineMetrics() |
|||
var pressure atomic.Int64 |
|||
pressure.Store(80) // soft zone
|
|||
|
|||
a := NewWALAdmission(WALAdmissionConfig{ |
|||
MaxConcurrent: 16, |
|||
SoftWatermark: 0.7, |
|||
HardWatermark: 0.9, |
|||
WALUsedFn: func() float64 { return float64(pressure.Load()) / 100.0 }, |
|||
NotifyFn: func() {}, |
|||
ClosedFn: func() bool { return false }, |
|||
Metrics: m, |
|||
}) |
|||
a.sleepFn = func(d time.Duration) { time.Sleep(100 * time.Microsecond) } |
|||
|
|||
var wg sync.WaitGroup |
|||
const writers = 4 |
|||
const rounds = 30 |
|||
|
|||
// Writers produce soft-zone and hard-zone waits.
|
|||
for i := 0; i < writers; i++ { |
|||
wg.Add(1) |
|||
go func(id int) { |
|||
defer wg.Done() |
|||
for j := 0; j < rounds; j++ { |
|||
if j%5 == 0 { |
|||
pressure.Store(95) // hard
|
|||
} else { |
|||
pressure.Store(80) // soft
|
|||
} |
|||
err := a.Acquire(50 * time.Millisecond) |
|||
if err == nil { |
|||
a.Release() |
|||
} |
|||
// Drop back so next Acquire can succeed.
|
|||
pressure.Store(50) |
|||
} |
|||
}(i) |
|||
} |
|||
|
|||
// Reader checks monotonicity.
|
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
var prevSoft, prevHard int64 |
|||
for i := 0; i < rounds*writers; i++ { |
|||
soft := a.SoftPressureWaitNs() |
|||
hard := a.HardPressureWaitNs() |
|||
if soft < prevSoft { |
|||
t.Errorf("SoftPressureWaitNs decreased: %d -> %d", prevSoft, soft) |
|||
} |
|||
if hard < prevHard { |
|||
t.Errorf("HardPressureWaitNs decreased: %d -> %d", prevHard, hard) |
|||
} |
|||
prevSoft = soft |
|||
prevHard = hard |
|||
} |
|||
}() |
|||
|
|||
wg.Wait() |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-CP11A3-6: WALGuidance_ZeroInputs
|
|||
//
|
|||
// Zero walSize, zero blockSize, zero maxConcurrent, empty hint.
|
|||
// Must not panic or produce invalid results.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_CP11A3_WALGuidance_ZeroInputs(t *testing.T) { |
|||
// All zeros.
|
|||
r := WALSizingGuidance(0, 0, "") |
|||
if r.Level != "warn" { |
|||
t.Errorf("zero walSize: Level = %q, want warn", r.Level) |
|||
} |
|||
|
|||
// Zero blockSize: absMin = 0*64 = 0. Only workload minimum check fires.
|
|||
r = WALSizingGuidance(0, 0, WorkloadGeneral) |
|||
if r.Level != "warn" { |
|||
t.Errorf("zero walSize+blockSize: Level = %q, want warn", r.Level) |
|||
} |
|||
|
|||
// Zero walSize but nonzero blockSize.
|
|||
r = WALSizingGuidance(0, 4096, WorkloadDatabase) |
|||
if r.Level != "warn" { |
|||
t.Errorf("zero walSize: Level = %q, want warn", r.Level) |
|||
} |
|||
if len(r.Warnings) < 2 { |
|||
t.Errorf("expected both workload + absolute minimum warnings, got %d", len(r.Warnings)) |
|||
} |
|||
|
|||
// EvaluateWALConfig with zero maxConcurrent should not trigger concurrency warning.
|
|||
r = EvaluateWALConfig(0, 4096, 0, WorkloadGeneral) |
|||
// walSize=0 still triggers sizing warning.
|
|||
if r.Level != "warn" { |
|||
t.Errorf("Level = %q, want warn for zero walSize", r.Level) |
|||
} |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-CP11A3-7: WALGuidance_OverflowSafe
|
|||
//
|
|||
// Very large blockSize × minWALEntries might overflow uint64.
|
|||
// (64 × 2^60 does NOT overflow, but let's test near-boundary.)
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_CP11A3_WALGuidance_OverflowSafe(t *testing.T) { |
|||
// Large blockSize: 256MB blocks × 64 = 16GB minimum.
|
|||
// walSize = 1GB → should warn (16GB > 1GB).
|
|||
r := WALSizingGuidance(1<<30, 256<<20, WorkloadGeneral) |
|||
if r.Level != "warn" { |
|||
t.Errorf("Level = %q, want warn (1GB WAL < 16GB absMin)", r.Level) |
|||
} |
|||
|
|||
// Extreme: blockSize = 1<<40 (1TB). 64 × 1TB = 64TB.
|
|||
// uint64 can hold 18 EB — no overflow.
|
|||
r = WALSizingGuidance(1<<50, 1<<40, WorkloadThroughput) |
|||
// 1PB WAL with 1TB blocks: absMin = 64TB, 1PB > 64TB → ok for absolute.
|
|||
// 1PB > 128MB (throughput min) → ok for workload.
|
|||
if r.Level != "ok" { |
|||
t.Errorf("Level = %q, want ok for huge WAL", r.Level) |
|||
} |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-CP11A3-8: WALStatusSnapshot_PartialInit
|
|||
//
|
|||
// BlockVol with Metrics but nil walAdmission, and vice versa.
|
|||
// WALStatus must return coherent defaults for the nil side
|
|||
// and real values for the non-nil side.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_CP11A3_WALStatusSnapshot_PartialInit(t *testing.T) { |
|||
// Case 1: Metrics set, walAdmission nil.
|
|||
m := NewEngineMetrics() |
|||
m.WALAdmitSoftTotal.Add(42) |
|||
m.WALAdmitHardTotal.Add(7) |
|||
vol1 := &BlockVol{Metrics: m} |
|||
|
|||
ws := vol1.WALStatus() |
|||
if ws.PressureState != "normal" { |
|||
t.Errorf("nil admission: PressureState = %q, want normal", ws.PressureState) |
|||
} |
|||
if ws.SoftAdmitTotal != 42 { |
|||
t.Errorf("SoftAdmitTotal = %d, want 42", ws.SoftAdmitTotal) |
|||
} |
|||
if ws.HardAdmitTotal != 7 { |
|||
t.Errorf("HardAdmitTotal = %d, want 7", ws.HardAdmitTotal) |
|||
} |
|||
// Pressure wait should be 0 (no admission controller).
|
|||
if ws.SoftPressureWaitSec != 0 || ws.HardPressureWaitSec != 0 { |
|||
t.Errorf("nil admission: pressure wait should be 0") |
|||
} |
|||
|
|||
// Case 2: walAdmission set, Metrics nil.
|
|||
a := NewWALAdmission(WALAdmissionConfig{ |
|||
MaxConcurrent: 16, |
|||
SoftWatermark: 0.65, |
|||
HardWatermark: 0.85, |
|||
WALUsedFn: func() float64 { return 0.7 }, |
|||
NotifyFn: func() {}, |
|||
ClosedFn: func() bool { return false }, |
|||
}) |
|||
vol2 := &BlockVol{walAdmission: a} |
|||
|
|||
ws2 := vol2.WALStatus() |
|||
if ws2.PressureState != "soft" { |
|||
t.Errorf("PressureState = %q, want soft (0.7 >= 0.65)", ws2.PressureState) |
|||
} |
|||
if ws2.SoftWatermark != 0.65 { |
|||
t.Errorf("SoftWatermark = %f, want 0.65", ws2.SoftWatermark) |
|||
} |
|||
// Metrics fields should be zero (nil Metrics).
|
|||
if ws2.SoftAdmitTotal != 0 || ws2.HardAdmitTotal != 0 || ws2.TimeoutTotal != 0 { |
|||
t.Errorf("nil metrics: counters should be 0") |
|||
} |
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-CP11A3-9: ObserverPanic_ContainedOrDocumented
|
|||
//
|
|||
// If WALAdmitWaitObserver panics, RecordWALAdmit is called from
|
|||
// Acquire → recordAdmit. A panic in the observer would crash the
|
|||
// writer goroutine. This test documents whether the panic is
|
|||
// recovered or propagated.
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_CP11A3_ObserverPanic_DocumentedBehavior(t *testing.T) { |
|||
m := NewEngineMetrics() |
|||
m.WALAdmitWaitObserver = func(s float64) { panic("boom") } |
|||
|
|||
// RecordWALAdmit calls the observer. If it panics, the caller panics.
|
|||
// This is expected (same as prometheus.Histogram.Observe panicking).
|
|||
// Document that the observer must not panic.
|
|||
panicked := false |
|||
func() { |
|||
defer func() { |
|||
if r := recover(); r != nil { |
|||
panicked = true |
|||
} |
|||
}() |
|||
m.RecordWALAdmit(1*time.Millisecond, false, false, false) |
|||
}() |
|||
|
|||
if !panicked { |
|||
t.Fatal("expected panic from observer — if recovered, update this test") |
|||
} |
|||
|
|||
// Verify counters were NOT updated (panic happened before completion).
|
|||
// Actually, the observer is called AFTER WALAdmitTotal.Add(1) and
|
|||
// walAdmitWaitNs.record(). Let's verify the counter state.
|
|||
if m.WALAdmitTotal.Load() != 1 { |
|||
t.Errorf("WALAdmitTotal = %d — should be 1 (incremented before observer)", m.WALAdmitTotal.Load()) |
|||
} |
|||
// soft/hard/timeout flags are processed AFTER observer — panic skips them.
|
|||
// With soft=false, hard=false, timedOut=false there's nothing to skip,
|
|||
// but the counters should reflect what happened before the panic.
|
|||
} |
|||
|
|||
// ────────────────────────────────────────────────────────────
|
|||
// QA-CP11A3-10: ConcurrentWALStatusReads
|
|||
//
|
|||
// Multiple goroutines read WALStatus while Acquire/Release runs.
|
|||
// Must not panic. Fields should be internally consistent
|
|||
// (SoftAdmitTotal >= 0, HardPressureWaitSec >= 0, etc.)
|
|||
// ────────────────────────────────────────────────────────────
|
|||
func TestQA_CP11A3_ConcurrentWALStatusReads(t *testing.T) { |
|||
m := NewEngineMetrics() |
|||
var pressure atomic.Int64 |
|||
pressure.Store(50) |
|||
|
|||
a := NewWALAdmission(WALAdmissionConfig{ |
|||
MaxConcurrent: 16, |
|||
SoftWatermark: 0.7, |
|||
HardWatermark: 0.9, |
|||
WALUsedFn: func() float64 { return float64(pressure.Load()) / 100.0 }, |
|||
NotifyFn: func() {}, |
|||
ClosedFn: func() bool { return false }, |
|||
Metrics: m, |
|||
}) |
|||
a.sleepFn = func(d time.Duration) { time.Sleep(50 * time.Microsecond) } |
|||
|
|||
vol := &BlockVol{ |
|||
Metrics: m, |
|||
walAdmission: a, |
|||
} |
|||
|
|||
var wg sync.WaitGroup |
|||
const rounds = 100 |
|||
|
|||
// Writers with varying pressure.
|
|||
for i := 0; i < 4; i++ { |
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
levels := []int64{50, 75, 95, 60, 85} |
|||
for j := 0; j < rounds; j++ { |
|||
pressure.Store(levels[j%len(levels)]) |
|||
if err := a.Acquire(20 * time.Millisecond); err == nil { |
|||
a.Release() |
|||
} |
|||
pressure.Store(50) // reset for next round
|
|||
} |
|||
}() |
|||
} |
|||
|
|||
// Concurrent WALStatus readers.
|
|||
for i := 0; i < 4; i++ { |
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
valid := map[string]bool{"normal": true, "soft": true, "hard": true} |
|||
for j := 0; j < rounds*2; j++ { |
|||
ws := vol.WALStatus() |
|||
if !valid[ws.PressureState] { |
|||
t.Errorf("invalid PressureState: %q", ws.PressureState) |
|||
return |
|||
} |
|||
if ws.UsedFraction < 0 || ws.UsedFraction > 1.01 { |
|||
t.Errorf("UsedFraction out of range: %f", ws.UsedFraction) |
|||
return |
|||
} |
|||
if ws.SoftPressureWaitSec < 0 { |
|||
t.Errorf("SoftPressureWaitSec negative: %f", ws.SoftPressureWaitSec) |
|||
return |
|||
} |
|||
if ws.HardPressureWaitSec < 0 { |
|||
t.Errorf("HardPressureWaitSec negative: %f", ws.HardPressureWaitSec) |
|||
return |
|||
} |
|||
} |
|||
}() |
|||
} |
|||
|
|||
wg.Wait() |
|||
} |
|||
@ -0,0 +1,246 @@ |
|||
name: cp11b3-auto-failover |
|||
timeout: 10m |
|||
env: |
|||
repo_dir: "/opt/work/seaweedfs" |
|||
master_url: "http://192.168.1.184:9434" |
|||
|
|||
# Tests: T1 (candidate evaluation), T2 (orphan re-evaluation), T6 (preflight/status) |
|||
# Flow: Create RF=2 → write data → kill primary → master auto-promotes → verify data + metrics |
|||
|
|||
topology: |
|||
nodes: |
|||
target_node: |
|||
host: "192.168.1.184" |
|||
user: testdev |
|||
key: "/opt/work/testdev_key" |
|||
client_node: |
|||
host: "192.168.1.181" |
|||
user: testdev |
|||
key: "/opt/work/testdev_key" |
|||
|
|||
phases: |
|||
# Phase 1: Clean slate |
|||
- name: setup |
|||
actions: |
|||
- action: kill_stale |
|||
node: target_node |
|||
- action: kill_stale |
|||
node: client_node |
|||
iscsi_cleanup: "true" |
|||
- action: exec |
|||
node: target_node |
|||
cmd: "rm -rf /tmp/sw-b3-master /tmp/sw-b3-vs1 /tmp/sw-b3-vs2" |
|||
root: "true" |
|||
|
|||
# Phase 2: Start cluster |
|||
- name: start_cluster |
|||
actions: |
|||
- action: exec |
|||
node: target_node |
|||
cmd: "mkdir -p /tmp/sw-b3-master /tmp/sw-b3-vs1/blocks /tmp/sw-b3-vs2/blocks" |
|||
- action: start_weed_master |
|||
node: target_node |
|||
port: "9434" |
|||
dir: "/tmp/sw-b3-master" |
|||
save_as: master_pid |
|||
- action: wait_cluster_ready |
|||
node: target_node |
|||
master_url: "http://localhost:9434" |
|||
timeout: 30s |
|||
- action: start_weed_volume |
|||
node: target_node |
|||
port: "18190" |
|||
master: "localhost:9434" |
|||
dir: "/tmp/sw-b3-vs1" |
|||
extra_args: "-block.dir=/tmp/sw-b3-vs1/blocks -block.listen=:3277 -ip=192.168.1.184" |
|||
save_as: vs1_pid |
|||
- action: start_weed_volume |
|||
node: target_node |
|||
port: "18191" |
|||
master: "localhost:9434" |
|||
dir: "/tmp/sw-b3-vs2" |
|||
extra_args: "-block.dir=/tmp/sw-b3-vs2/blocks -block.listen=:3278 -ip=192.168.1.184" |
|||
save_as: vs2_pid |
|||
- action: wait_block_servers |
|||
count: "2" |
|||
timeout: 60s |
|||
|
|||
# Phase 3: Create RF=2 volume, record initial state |
|||
- name: create_volume |
|||
actions: |
|||
- action: create_block_volume |
|||
name: "failover-test" |
|||
size: "50M" |
|||
replica_factor: "2" |
|||
save_as: vol_info |
|||
# Wait for replica to confirm role via heartbeat. |
|||
# Without this, PromoteBestReplica rejects replica as "no_heartbeat". |
|||
- action: sleep |
|||
duration: 10s |
|||
- action: lookup_block_volume |
|||
name: "failover-test" |
|||
save_as: initial |
|||
- action: print |
|||
msg: "initial primary={{ initial_iscsi_host }}:{{ initial_iscsi_port }} capacity={{ initial_capacity }}" |
|||
# Record the initial primary server for later comparison. |
|||
- action: assert_block_field |
|||
name: "failover-test" |
|||
field: "replica_factor" |
|||
expected: "2" |
|||
- action: assert_block_field |
|||
name: "failover-test" |
|||
field: "epoch" |
|||
expected: "1" |
|||
# Capture initial block status metrics. |
|||
- action: block_status |
|||
save_as: pre_stats |
|||
|
|||
# Phase 4: Write data via iSCSI |
|||
- name: write_data |
|||
actions: |
|||
- action: iscsi_login_direct |
|||
node: client_node |
|||
host: "{{ initial_iscsi_host }}" |
|||
port: "{{ initial_iscsi_port }}" |
|||
iqn: "{{ initial_iqn }}" |
|||
save_as: device |
|||
- action: dd_write |
|||
node: client_node |
|||
device: "{{ device }}" |
|||
bs: 1M |
|||
count: "1" |
|||
seek: "5" |
|||
save_as: md5_5M |
|||
- action: dd_read_md5 |
|||
node: client_node |
|||
device: "{{ device }}" |
|||
bs: 1M |
|||
count: "1" |
|||
skip: "5" |
|||
save_as: verify_5M |
|||
- action: assert_equal |
|||
actual: "{{ verify_5M }}" |
|||
expected: "{{ md5_5M }}" |
|||
|
|||
# Phase 5: Kill primary VS, wait for master auto-failover |
|||
- name: failover |
|||
actions: |
|||
- action: iscsi_cleanup |
|||
node: client_node |
|||
ignore_error: true |
|||
- action: lookup_block_volume |
|||
name: "failover-test" |
|||
save_as: pre_kill |
|||
- action: print |
|||
msg: "killing primary VS (server={{ pre_kill_iscsi_host }}:{{ pre_kill_iscsi_port }})" |
|||
# Crash-kill VS1 with SIGKILL (not SIGTERM) to simulate a real crash. |
|||
# SIGTERM triggers graceful shutdown which deregisters volumes from |
|||
# the master registry — preventing the failover path we want to test. |
|||
- action: exec |
|||
node: target_node |
|||
cmd: "kill -9 {{ vs1_pid }}" |
|||
root: "true" |
|||
# Wait for master to detect VS1 disconnection and promote. |
|||
# Lease TTL is 30s; if never granted (zero), promotion is immediate. |
|||
# Allow extra time for heartbeat confirmation + deferred timer. |
|||
- action: sleep |
|||
duration: 35s |
|||
- action: wait_block_primary |
|||
name: "failover-test" |
|||
not: "192.168.1.184:18190" |
|||
timeout: 60s |
|||
save_as: promoted |
|||
|
|||
# Phase 6: Verify failover state |
|||
- name: verify_failover |
|||
actions: |
|||
- action: print |
|||
msg: "new primary={{ promoted_server }} epoch={{ promoted_epoch }}" |
|||
# Epoch must have incremented (real promotion, not just heartbeat update). |
|||
- action: assert_block_field |
|||
name: "failover-test" |
|||
field: "epoch" |
|||
expected: "2" |
|||
- action: block_status |
|||
save_as: post_stats |
|||
# Verify promotion counter incremented. |
|||
- action: assert_greater |
|||
actual: "{{ post_stats_promotions_total }}" |
|||
expected: "{{ pre_stats_promotions_total }}" |
|||
|
|||
# Phase 7: Reconnect iSCSI to new primary, verify data |
|||
- name: verify_data |
|||
actions: |
|||
- action: iscsi_login_direct |
|||
node: client_node |
|||
host: "{{ promoted_iscsi_host }}" |
|||
port: "{{ promoted_iscsi_port }}" |
|||
iqn: "{{ promoted_iqn }}" |
|||
save_as: device2 |
|||
- action: dd_read_md5 |
|||
node: client_node |
|||
device: "{{ device2 }}" |
|||
bs: 1M |
|||
count: "1" |
|||
skip: "5" |
|||
save_as: post_failover_md5 |
|||
- action: assert_equal |
|||
actual: "{{ post_failover_md5 }}" |
|||
expected: "{{ md5_5M }}" |
|||
|
|||
# Phase 8: Restart killed VS, verify rebuild queued |
|||
- name: restart_verify |
|||
actions: |
|||
- action: iscsi_cleanup |
|||
node: client_node |
|||
ignore_error: true |
|||
- action: start_weed_volume |
|||
node: target_node |
|||
port: "18190" |
|||
master: "localhost:9434" |
|||
dir: "/tmp/sw-b3-vs1" |
|||
extra_args: "-block.dir=/tmp/sw-b3-vs1/blocks -block.listen=:3277 -ip=192.168.1.184" |
|||
save_as: vs1_pid2 |
|||
- action: wait_block_servers |
|||
count: "2" |
|||
timeout: 60s |
|||
- action: sleep |
|||
duration: 5s |
|||
# After restart, the old primary should be queued for rebuild. |
|||
- action: block_status |
|||
save_as: final_stats |
|||
- action: assert_greater |
|||
actual: "{{ final_stats_rebuilds_total }}" |
|||
expected: "{{ post_stats_rebuilds_total }}" |
|||
|
|||
# Cleanup (always runs) |
|||
- name: cleanup |
|||
always: true |
|||
actions: |
|||
- action: iscsi_cleanup |
|||
node: client_node |
|||
ignore_error: true |
|||
- action: delete_block_volume |
|||
name: "failover-test" |
|||
ignore_error: true |
|||
- action: stop_weed |
|||
node: target_node |
|||
pid: "{{ vs1_pid2 }}" |
|||
ignore_error: true |
|||
- action: stop_weed |
|||
node: target_node |
|||
pid: "{{ vs2_pid }}" |
|||
ignore_error: true |
|||
- action: stop_weed |
|||
node: target_node |
|||
pid: "{{ vs1_pid }}" |
|||
ignore_error: true |
|||
- action: stop_weed |
|||
node: target_node |
|||
pid: "{{ master_pid }}" |
|||
ignore_error: true |
|||
- action: exec |
|||
node: target_node |
|||
cmd: "rm -rf /tmp/sw-b3-master /tmp/sw-b3-vs1 /tmp/sw-b3-vs2" |
|||
root: "true" |
|||
ignore_error: true |
|||
@ -0,0 +1,214 @@ |
|||
name: cp11b3-fast-reconnect |
|||
timeout: 10m |
|||
env: |
|||
repo_dir: "/opt/work/seaweedfs" |
|||
master_url: "http://192.168.1.184:9436" |
|||
|
|||
# Tests: T3 (deferred timer safety), T2 (fast reconnect skips failover) |
|||
# Flow: Create RF=2 → write → kill primary briefly → restart before lease expires |
|||
# → verify no promotion happened → verify data intact |
|||
|
|||
topology: |
|||
nodes: |
|||
target_node: |
|||
host: "192.168.1.184" |
|||
user: testdev |
|||
key: "/opt/work/testdev_key" |
|||
client_node: |
|||
host: "192.168.1.181" |
|||
user: testdev |
|||
key: "/opt/work/testdev_key" |
|||
|
|||
phases: |
|||
# Phase 1: Clean slate |
|||
- name: setup |
|||
actions: |
|||
- action: kill_stale |
|||
node: target_node |
|||
- action: kill_stale |
|||
node: client_node |
|||
iscsi_cleanup: "true" |
|||
- action: exec |
|||
node: target_node |
|||
cmd: "rm -rf /tmp/sw-b3r-master /tmp/sw-b3r-vs1 /tmp/sw-b3r-vs2" |
|||
root: "true" |
|||
|
|||
# Phase 2: Start cluster |
|||
- name: start_cluster |
|||
actions: |
|||
- action: exec |
|||
node: target_node |
|||
cmd: "mkdir -p /tmp/sw-b3r-master /tmp/sw-b3r-vs1/blocks /tmp/sw-b3r-vs2/blocks" |
|||
- action: start_weed_master |
|||
node: target_node |
|||
port: "9436" |
|||
dir: "/tmp/sw-b3r-master" |
|||
save_as: master_pid |
|||
- action: wait_cluster_ready |
|||
node: target_node |
|||
master_url: "http://localhost:9436" |
|||
timeout: 30s |
|||
- action: start_weed_volume |
|||
node: target_node |
|||
port: "18194" |
|||
master: "localhost:9436" |
|||
dir: "/tmp/sw-b3r-vs1" |
|||
extra_args: "-block.dir=/tmp/sw-b3r-vs1/blocks -block.listen=:3281 -ip=192.168.1.184" |
|||
save_as: vs1_pid |
|||
- action: start_weed_volume |
|||
node: target_node |
|||
port: "18195" |
|||
master: "localhost:9436" |
|||
dir: "/tmp/sw-b3r-vs2" |
|||
extra_args: "-block.dir=/tmp/sw-b3r-vs2/blocks -block.listen=:3282 -ip=192.168.1.184" |
|||
save_as: vs2_pid |
|||
- action: wait_block_servers |
|||
count: "2" |
|||
timeout: 60s |
|||
|
|||
# Phase 3: Create RF=2 volume, write data |
|||
- name: create_and_write |
|||
actions: |
|||
- action: create_block_volume |
|||
name: "reconnect-test" |
|||
size: "50M" |
|||
replica_factor: "2" |
|||
save_as: vol_info |
|||
# Wait for replica to confirm role via heartbeat. |
|||
- action: sleep |
|||
duration: 10s |
|||
- action: lookup_block_volume |
|||
name: "reconnect-test" |
|||
save_as: initial |
|||
- action: iscsi_login_direct |
|||
node: client_node |
|||
host: "{{ initial_iscsi_host }}" |
|||
port: "{{ initial_iscsi_port }}" |
|||
iqn: "{{ initial_iqn }}" |
|||
save_as: device |
|||
- action: dd_write |
|||
node: client_node |
|||
device: "{{ device }}" |
|||
bs: 1M |
|||
count: "1" |
|||
seek: "8" |
|||
save_as: md5_8M |
|||
- action: dd_read_md5 |
|||
node: client_node |
|||
device: "{{ device }}" |
|||
bs: 1M |
|||
count: "1" |
|||
skip: "8" |
|||
save_as: verify_8M |
|||
- action: assert_equal |
|||
actual: "{{ verify_8M }}" |
|||
expected: "{{ md5_8M }}" |
|||
- action: iscsi_cleanup |
|||
node: client_node |
|||
ignore_error: true |
|||
# Record initial epoch. |
|||
- action: assert_block_field |
|||
name: "reconnect-test" |
|||
field: "epoch" |
|||
expected: "1" |
|||
# Record pre-kill promotion counter. |
|||
- action: block_status |
|||
save_as: pre_stats |
|||
|
|||
# Phase 4: Kill and quickly restart primary VS (before lease expires) |
|||
- name: fast_reconnect |
|||
actions: |
|||
# Crash-kill primary VS with SIGKILL. |
|||
- action: exec |
|||
node: target_node |
|||
cmd: "kill -9 {{ vs1_pid }}" |
|||
root: "true" |
|||
# Restart it quickly — within a few seconds, well before the |
|||
# default 30s lease TTL expires on the master. |
|||
- action: sleep |
|||
duration: 3s |
|||
- action: start_weed_volume |
|||
node: target_node |
|||
port: "18194" |
|||
master: "localhost:9436" |
|||
dir: "/tmp/sw-b3r-vs1" |
|||
extra_args: "-block.dir=/tmp/sw-b3r-vs1/blocks -block.listen=:3281 -ip=192.168.1.184" |
|||
save_as: vs1_pid2 |
|||
# Wait for VS to re-register with master. |
|||
- action: wait_block_servers |
|||
count: "2" |
|||
timeout: 60s |
|||
- action: sleep |
|||
duration: 5s |
|||
|
|||
# Phase 5: Verify NO promotion happened |
|||
- name: verify_no_promotion |
|||
actions: |
|||
# Epoch should still be 1 (no promotion). |
|||
- action: assert_block_field |
|||
name: "reconnect-test" |
|||
field: "epoch" |
|||
expected: "1" |
|||
# Promotion counter should not have increased. |
|||
- action: block_status |
|||
save_as: post_stats |
|||
- action: assert_equal |
|||
actual: "{{ post_stats_promotions_total }}" |
|||
expected: "{{ pre_stats_promotions_total }}" |
|||
- action: print |
|||
msg: "fast reconnect: epoch unchanged, no promotion — deferred timer cancelled" |
|||
|
|||
# Phase 6: Verify data still accessible on original primary |
|||
- name: verify_data |
|||
actions: |
|||
- action: lookup_block_volume |
|||
name: "reconnect-test" |
|||
save_as: after |
|||
- action: iscsi_login_direct |
|||
node: client_node |
|||
host: "{{ after_iscsi_host }}" |
|||
port: "{{ after_iscsi_port }}" |
|||
iqn: "{{ after_iqn }}" |
|||
save_as: device2 |
|||
- action: dd_read_md5 |
|||
node: client_node |
|||
device: "{{ device2 }}" |
|||
bs: 1M |
|||
count: "1" |
|||
skip: "8" |
|||
save_as: post_reconnect_md5 |
|||
- action: assert_equal |
|||
actual: "{{ post_reconnect_md5 }}" |
|||
expected: "{{ md5_8M }}" |
|||
|
|||
# Cleanup (always runs) |
|||
- name: cleanup |
|||
always: true |
|||
actions: |
|||
- action: iscsi_cleanup |
|||
node: client_node |
|||
ignore_error: true |
|||
- action: delete_block_volume |
|||
name: "reconnect-test" |
|||
ignore_error: true |
|||
- action: stop_weed |
|||
node: target_node |
|||
pid: "{{ vs1_pid2 }}" |
|||
ignore_error: true |
|||
- action: stop_weed |
|||
node: target_node |
|||
pid: "{{ vs2_pid }}" |
|||
ignore_error: true |
|||
- action: stop_weed |
|||
node: target_node |
|||
pid: "{{ vs1_pid }}" |
|||
ignore_error: true |
|||
- action: stop_weed |
|||
node: target_node |
|||
pid: "{{ master_pid }}" |
|||
ignore_error: true |
|||
- action: exec |
|||
node: target_node |
|||
cmd: "rm -rf /tmp/sw-b3r-master /tmp/sw-b3r-vs1 /tmp/sw-b3r-vs2" |
|||
root: "true" |
|||
ignore_error: true |
|||
@ -0,0 +1,190 @@ |
|||
name: cp11b3-manual-promote |
|||
timeout: 10m |
|||
env: |
|||
repo_dir: "/opt/work/seaweedfs" |
|||
master_url: "http://192.168.1.184:9435" |
|||
|
|||
# Tests: T5 (manual promote API), T6 (preflight), structured rejection |
|||
# Flow: Create RF=2 → write → preflight check → kill primary → manual promote → verify data |
|||
|
|||
topology: |
|||
nodes: |
|||
target_node: |
|||
host: "192.168.1.184" |
|||
user: testdev |
|||
key: "/opt/work/testdev_key" |
|||
client_node: |
|||
host: "192.168.1.181" |
|||
user: testdev |
|||
key: "/opt/work/testdev_key" |
|||
|
|||
phases: |
|||
# Phase 1: Clean slate |
|||
- name: setup |
|||
actions: |
|||
- action: kill_stale |
|||
node: target_node |
|||
- action: kill_stale |
|||
node: client_node |
|||
iscsi_cleanup: "true" |
|||
- action: exec |
|||
node: target_node |
|||
cmd: "rm -rf /tmp/sw-b3m-master /tmp/sw-b3m-vs1 /tmp/sw-b3m-vs2" |
|||
root: "true" |
|||
|
|||
# Phase 2: Start cluster |
|||
- name: start_cluster |
|||
actions: |
|||
- action: exec |
|||
node: target_node |
|||
cmd: "mkdir -p /tmp/sw-b3m-master /tmp/sw-b3m-vs1/blocks /tmp/sw-b3m-vs2/blocks" |
|||
- action: start_weed_master |
|||
node: target_node |
|||
port: "9435" |
|||
dir: "/tmp/sw-b3m-master" |
|||
save_as: master_pid |
|||
- action: wait_cluster_ready |
|||
node: target_node |
|||
master_url: "http://localhost:9435" |
|||
timeout: 30s |
|||
- action: start_weed_volume |
|||
node: target_node |
|||
port: "18192" |
|||
master: "localhost:9435" |
|||
dir: "/tmp/sw-b3m-vs1" |
|||
extra_args: "-block.dir=/tmp/sw-b3m-vs1/blocks -block.listen=:3279 -ip=192.168.1.184" |
|||
save_as: vs1_pid |
|||
- action: start_weed_volume |
|||
node: target_node |
|||
port: "18193" |
|||
master: "localhost:9435" |
|||
dir: "/tmp/sw-b3m-vs2" |
|||
extra_args: "-block.dir=/tmp/sw-b3m-vs2/blocks -block.listen=:3280 -ip=192.168.1.184" |
|||
save_as: vs2_pid |
|||
- action: wait_block_servers |
|||
count: "2" |
|||
timeout: 60s |
|||
|
|||
# Phase 3: Create RF=2 volume, write data |
|||
- name: create_and_write |
|||
actions: |
|||
- action: create_block_volume |
|||
name: "promote-test" |
|||
size: "50M" |
|||
replica_factor: "2" |
|||
save_as: vol_info |
|||
# Wait for replica to confirm role via heartbeat. |
|||
- action: sleep |
|||
duration: 10s |
|||
- action: lookup_block_volume |
|||
name: "promote-test" |
|||
save_as: initial |
|||
- action: iscsi_login_direct |
|||
node: client_node |
|||
host: "{{ initial_iscsi_host }}" |
|||
port: "{{ initial_iscsi_port }}" |
|||
iqn: "{{ initial_iqn }}" |
|||
save_as: device |
|||
- action: dd_write |
|||
node: client_node |
|||
device: "{{ device }}" |
|||
bs: 1M |
|||
count: "2" |
|||
seek: "3" |
|||
save_as: md5_3M |
|||
- action: dd_read_md5 |
|||
node: client_node |
|||
device: "{{ device }}" |
|||
bs: 1M |
|||
count: "2" |
|||
skip: "3" |
|||
save_as: verify_3M |
|||
- action: assert_equal |
|||
actual: "{{ verify_3M }}" |
|||
expected: "{{ md5_3M }}" |
|||
|
|||
# Phase 4: Kill primary VS, then promote via API |
|||
- name: kill_and_promote |
|||
actions: |
|||
- action: iscsi_cleanup |
|||
node: client_node |
|||
ignore_error: true |
|||
# Crash-kill VS1 with SIGKILL to simulate a real crash. |
|||
- action: exec |
|||
node: target_node |
|||
cmd: "kill -9 {{ vs1_pid }}" |
|||
root: "true" |
|||
# Wait for master to detect the disconnection. |
|||
- action: sleep |
|||
duration: 15s |
|||
# Manual promote via the API. |
|||
- action: block_promote |
|||
name: "promote-test" |
|||
reason: "T7 integration test: manual failover" |
|||
save_as: promote_result |
|||
- action: print |
|||
msg: "promoted to {{ promote_result_server }} epoch={{ promote_result_epoch }}" |
|||
|
|||
# Phase 5: Verify promoted state |
|||
- name: verify_promoted |
|||
actions: |
|||
- action: lookup_block_volume |
|||
name: "promote-test" |
|||
save_as: after |
|||
# New primary should be different from old. |
|||
- action: assert_block_field |
|||
name: "promote-test" |
|||
field: "epoch" |
|||
expected: "2" |
|||
- action: block_status |
|||
save_as: stats |
|||
- action: print |
|||
msg: "promotions_total={{ stats_promotions_total }}" |
|||
|
|||
# Phase 6: Reconnect iSCSI to new primary, verify data |
|||
- name: verify_data |
|||
actions: |
|||
- action: iscsi_login_direct |
|||
node: client_node |
|||
host: "{{ after_iscsi_host }}" |
|||
port: "{{ after_iscsi_port }}" |
|||
iqn: "{{ after_iqn }}" |
|||
save_as: device2 |
|||
- action: dd_read_md5 |
|||
node: client_node |
|||
device: "{{ device2 }}" |
|||
bs: 1M |
|||
count: "2" |
|||
skip: "3" |
|||
save_as: post_promote_md5 |
|||
- action: assert_equal |
|||
actual: "{{ post_promote_md5 }}" |
|||
expected: "{{ md5_3M }}" |
|||
|
|||
# Cleanup (always runs) |
|||
- name: cleanup |
|||
always: true |
|||
actions: |
|||
- action: iscsi_cleanup |
|||
node: client_node |
|||
ignore_error: true |
|||
- action: delete_block_volume |
|||
name: "promote-test" |
|||
ignore_error: true |
|||
- action: stop_weed |
|||
node: target_node |
|||
pid: "{{ vs2_pid }}" |
|||
ignore_error: true |
|||
- action: stop_weed |
|||
node: target_node |
|||
pid: "{{ vs1_pid }}" |
|||
ignore_error: true |
|||
- action: stop_weed |
|||
node: target_node |
|||
pid: "{{ master_pid }}" |
|||
ignore_error: true |
|||
- action: exec |
|||
node: target_node |
|||
cmd: "rm -rf /tmp/sw-b3m-master /tmp/sw-b3m-vs1 /tmp/sw-b3m-vs2" |
|||
root: "true" |
|||
ignore_error: true |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue