You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
887 lines
31 KiB
887 lines
31 KiB
package weed_server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
|
|
)
|
|
|
|
// ============================================================
|
|
// CP8-2 Adversarial Test Suite
|
|
//
|
|
// 12 scenarios covering multi-replica, scrub, promotion gating,
|
|
// heartbeat reconciliation, and partial-failure edge cases.
|
|
// ============================================================
|
|
|
|
// qaCP82Master creates a MasterServer with 3 block-capable servers
|
|
// and configurable mocks for adversarial testing.
|
|
func qaCP82Master(t *testing.T) *MasterServer {
|
|
t.Helper()
|
|
ms := &MasterServer{
|
|
blockRegistry: NewBlockVolumeRegistry(),
|
|
blockAssignmentQueue: NewBlockAssignmentQueue(),
|
|
blockFailover: newBlockFailoverState(),
|
|
}
|
|
ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
|
|
return &blockAllocResult{
|
|
Path: fmt.Sprintf("/data/%s.blk", name),
|
|
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
|
|
ISCSIAddr: server + ":3260",
|
|
ReplicaDataAddr: server + ":14260",
|
|
ReplicaCtrlAddr: server + ":14261",
|
|
RebuildListenAddr: server + ":15000",
|
|
}, nil
|
|
}
|
|
ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
|
|
return nil
|
|
}
|
|
ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) {
|
|
return newSize, nil
|
|
}
|
|
ms.blockVSPrepareExpand = func(ctx context.Context, server string, name string, newSize, expandEpoch uint64) error {
|
|
return nil
|
|
}
|
|
ms.blockVSCommitExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) (uint64, error) {
|
|
return 2 << 30, nil
|
|
}
|
|
ms.blockVSCancelExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) error {
|
|
return nil
|
|
}
|
|
ms.blockRegistry.MarkBlockCapable("vs1:9333")
|
|
ms.blockRegistry.MarkBlockCapable("vs2:9333")
|
|
ms.blockRegistry.MarkBlockCapable("vs3:9333")
|
|
return ms
|
|
}
|
|
|
|
// qaRegisterRF2WithLSN creates an RF=2 volume with explicit WAL LSN state.
|
|
func qaRegisterRF2WithLSN(t *testing.T, ms *MasterServer, name, primary, replica string, epoch, primaryLSN, replicaLSN uint64) {
|
|
t.Helper()
|
|
entry := &BlockVolumeEntry{
|
|
Name: name,
|
|
VolumeServer: primary,
|
|
Path: fmt.Sprintf("/data/%s.blk", name),
|
|
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
|
|
ISCSIAddr: primary + ":3260",
|
|
SizeBytes: 1 << 30,
|
|
Epoch: epoch,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
|
Status: StatusActive,
|
|
WALHeadLSN: primaryLSN,
|
|
LeaseTTL: 5 * time.Second,
|
|
LastLeaseGrant: time.Now().Add(-10 * time.Second), // expired
|
|
ReplicaServer: replica,
|
|
ReplicaPath: fmt.Sprintf("/data/%s.blk", name),
|
|
ReplicaIQN: fmt.Sprintf("iqn.2024.test:%s-r", name),
|
|
ReplicaISCSIAddr: replica + ":3260",
|
|
Replicas: []ReplicaInfo{
|
|
{
|
|
Server: replica,
|
|
Path: fmt.Sprintf("/data/%s.blk", name),
|
|
IQN: fmt.Sprintf("iqn.2024.test:%s-r", name),
|
|
ISCSIAddr: replica + ":3260",
|
|
HealthScore: 1.0,
|
|
WALHeadLSN: replicaLSN,
|
|
LastHeartbeat: time.Now(), // fresh
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
|
},
|
|
},
|
|
}
|
|
if err := ms.blockRegistry.Register(entry); err != nil {
|
|
t.Fatalf("register %s: %v", name, err)
|
|
}
|
|
}
|
|
|
|
// qaRegisterRF3WithState creates an RF=3 volume with per-replica state control.
|
|
func qaRegisterRF3WithState(t *testing.T, ms *MasterServer, name, primary string,
|
|
replicas []struct {
|
|
Server string
|
|
HealthScore float64
|
|
WALHeadLSN uint64
|
|
Role blockvol.Role
|
|
Heartbeat time.Time
|
|
},
|
|
primaryLSN uint64,
|
|
) {
|
|
t.Helper()
|
|
entry := &BlockVolumeEntry{
|
|
Name: name,
|
|
VolumeServer: primary,
|
|
Path: fmt.Sprintf("/data/%s.blk", name),
|
|
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
|
|
ISCSIAddr: primary + ":3260",
|
|
SizeBytes: 1 << 30,
|
|
Epoch: 1,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
|
Status: StatusActive,
|
|
WALHeadLSN: primaryLSN,
|
|
ReplicaFactor: len(replicas) + 1,
|
|
LeaseTTL: 5 * time.Second,
|
|
LastLeaseGrant: time.Now().Add(-10 * time.Second), // expired
|
|
}
|
|
for i, r := range replicas {
|
|
ri := ReplicaInfo{
|
|
Server: r.Server,
|
|
Path: fmt.Sprintf("/data/%s.blk", name),
|
|
IQN: fmt.Sprintf("iqn.2024.test:%s-r%d", name, i+1),
|
|
ISCSIAddr: r.Server + ":3260",
|
|
HealthScore: r.HealthScore,
|
|
WALHeadLSN: r.WALHeadLSN,
|
|
Role: blockvol.RoleToWire(r.Role),
|
|
LastHeartbeat: r.Heartbeat,
|
|
}
|
|
entry.Replicas = append(entry.Replicas, ri)
|
|
}
|
|
// Sync scalar fields from first replica.
|
|
if len(entry.Replicas) > 0 {
|
|
entry.ReplicaServer = entry.Replicas[0].Server
|
|
entry.ReplicaPath = entry.Replicas[0].Path
|
|
entry.ReplicaIQN = entry.Replicas[0].IQN
|
|
entry.ReplicaISCSIAddr = entry.Replicas[0].ISCSIAddr
|
|
}
|
|
if err := ms.blockRegistry.Register(entry); err != nil {
|
|
t.Fatalf("register %s: %v", name, err)
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// QA-1: PrimaryCrash_AfterAck_BeforeReplicaBarrier
|
|
//
|
|
// Verify failover data loss window is bounded by barrier_lag_lsn.
|
|
// Simulates primary at LSN 200, replica at barrier LSN 190.
|
|
// After failover, new primary should be at replica's LSN (190).
|
|
// The gap (10 LSN entries) is the bounded loss window.
|
|
// ────────────────────────────────────────────────────────────
|
|
func TestQA_CP82_PrimaryCrash_AfterAck_BeforeReplicaBarrier(t *testing.T) {
|
|
ms := qaCP82Master(t)
|
|
qaRegisterRF2WithLSN(t, ms, "vol-lag", "vs1:9333", "vs2:9333", 1, 200, 190)
|
|
|
|
// Kill primary.
|
|
ms.failoverBlockVolumes("vs1:9333")
|
|
|
|
entry, ok := ms.blockRegistry.Lookup("vol-lag")
|
|
if !ok {
|
|
t.Fatal("volume should still exist after failover")
|
|
}
|
|
if entry.VolumeServer != "vs2:9333" {
|
|
t.Fatalf("replica should be promoted, got %q", entry.VolumeServer)
|
|
}
|
|
// The promoted replica was at LSN 190. The 10-LSN gap represents
|
|
// acknowledged-but-unbarriered writes (bounded data loss).
|
|
// New epoch should be 2.
|
|
if entry.Epoch != 2 {
|
|
t.Fatalf("epoch: got %d, want 2", entry.Epoch)
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// QA-2: ReplicaHeartbeatSpoof_DoesNotDeletePrimary
|
|
//
|
|
// Regression test for Fix #1. A replica heartbeat that does NOT
|
|
// include the primary path must NOT delete the volume entry.
|
|
// ────────────────────────────────────────────────────────────
|
|
func TestQA_CP82_ReplicaHeartbeatSpoof_DoesNotDeletePrimary(t *testing.T) {
|
|
ms := qaCP82Master(t)
|
|
qaRegisterRF2WithLSN(t, ms, "vol-spoof", "vs1:9333", "vs2:9333", 1, 100, 100)
|
|
|
|
// Simulate replica heartbeat from vs2 with ONLY its replica path.
|
|
// This must NOT remove the volume (vs1 is the primary, not vs2).
|
|
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
|
|
{
|
|
Path: "/data/vol-spoof.blk",
|
|
VolumeSize: 1 << 30,
|
|
Epoch: 1,
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
|
},
|
|
}, "")
|
|
|
|
entry, ok := ms.blockRegistry.Lookup("vol-spoof")
|
|
if !ok {
|
|
t.Fatal("volume must NOT be deleted by replica heartbeat (Fix #1 regression)")
|
|
}
|
|
if entry.VolumeServer != "vs1:9333" {
|
|
t.Fatalf("primary must remain vs1:9333, got %q", entry.VolumeServer)
|
|
}
|
|
|
|
// Now simulate a full heartbeat from vs2 with NO volumes at all.
|
|
// This should remove vs2 as replica but NOT delete the volume.
|
|
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{}, "")
|
|
|
|
entry, ok = ms.blockRegistry.Lookup("vol-spoof")
|
|
if !ok {
|
|
t.Fatal("volume must still exist after empty replica heartbeat")
|
|
}
|
|
if entry.VolumeServer != "vs1:9333" {
|
|
t.Fatal("primary must still be vs1:9333")
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// QA-3: PromotionRejects_StaleButHealthyReplica
|
|
//
|
|
// Replica has perfect health (1.0) but stale heartbeat.
|
|
// Must NOT be promoted (Gate 1: heartbeat freshness).
|
|
// ────────────────────────────────────────────────────────────
|
|
func TestQA_CP82_PromotionRejects_StaleButHealthyReplica(t *testing.T) {
|
|
ms := qaCP82Master(t)
|
|
staleTime := time.Now().Add(-5 * time.Minute) // way beyond 2×LeaseTTL
|
|
qaRegisterRF3WithState(t, ms, "vol-stale", "vs1:9333",
|
|
[]struct {
|
|
Server string
|
|
HealthScore float64
|
|
WALHeadLSN uint64
|
|
Role blockvol.Role
|
|
Heartbeat time.Time
|
|
}{
|
|
{"vs2:9333", 1.0, 100, blockvol.RoleReplica, staleTime}, // healthy but stale
|
|
{"vs3:9333", 0.5, 100, blockvol.RoleReplica, staleTime}, // also stale
|
|
},
|
|
100,
|
|
)
|
|
|
|
_, err := ms.blockRegistry.PromoteBestReplica("vol-stale")
|
|
if err == nil {
|
|
t.Fatal("expected promotion to fail — all replicas have stale heartbeats")
|
|
}
|
|
// Volume should remain unchanged.
|
|
entry, _ := ms.blockRegistry.Lookup("vol-stale")
|
|
if entry.VolumeServer != "vs1:9333" {
|
|
t.Fatalf("primary must not change, got %q", entry.VolumeServer)
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// QA-4: PromotionRejects_RebuildingReplica
|
|
//
|
|
// Replica is fresh and healthy but in RoleRebuilding.
|
|
// Must NOT be promoted (Gate 3: role check).
|
|
// ────────────────────────────────────────────────────────────
|
|
func TestQA_CP82_PromotionRejects_RebuildingReplica(t *testing.T) {
|
|
ms := qaCP82Master(t)
|
|
now := time.Now()
|
|
qaRegisterRF3WithState(t, ms, "vol-rebuild", "vs1:9333",
|
|
[]struct {
|
|
Server string
|
|
HealthScore float64
|
|
WALHeadLSN uint64
|
|
Role blockvol.Role
|
|
Heartbeat time.Time
|
|
}{
|
|
{"vs2:9333", 1.0, 100, blockvol.RoleRebuilding, now}, // rebuilding
|
|
{"vs3:9333", 1.0, 100, blockvol.RoleRebuilding, now}, // also rebuilding
|
|
},
|
|
100,
|
|
)
|
|
|
|
_, err := ms.blockRegistry.PromoteBestReplica("vol-rebuild")
|
|
if err == nil {
|
|
t.Fatal("expected promotion to fail — all replicas are rebuilding")
|
|
}
|
|
|
|
// One rebuilding, one ready: only the ready one should be promoted.
|
|
ms2 := qaCP82Master(t)
|
|
qaRegisterRF3WithState(t, ms2, "vol-mixed", "vs1:9333",
|
|
[]struct {
|
|
Server string
|
|
HealthScore float64
|
|
WALHeadLSN uint64
|
|
Role blockvol.Role
|
|
Heartbeat time.Time
|
|
}{
|
|
{"vs2:9333", 1.0, 100, blockvol.RoleRebuilding, now},
|
|
{"vs3:9333", 0.7, 100, blockvol.RoleReplica, now}, // eligible despite lower health
|
|
},
|
|
100,
|
|
)
|
|
|
|
_, err = ms2.blockRegistry.PromoteBestReplica("vol-mixed")
|
|
if err != nil {
|
|
t.Fatalf("expected promotion to succeed: %v", err)
|
|
}
|
|
entry, _ := ms2.blockRegistry.Lookup("vol-mixed")
|
|
if entry.VolumeServer != "vs3:9333" {
|
|
t.Fatalf("vs3 should be promoted (only eligible), got %q", entry.VolumeServer)
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// QA-5: PromotionToleranceBoundary_ExactLSN
|
|
//
|
|
// Tests promotion_lsn_tolerance boundary: tolerance-1, exact, tolerance+1.
|
|
// Catches off-by-one in the eligibility gate.
|
|
// ────────────────────────────────────────────────────────────
|
|
func TestQA_CP82_PromotionToleranceBoundary_ExactLSN(t *testing.T) {
|
|
const tolerance uint64 = 50
|
|
const primaryLSN uint64 = 200
|
|
|
|
cases := []struct {
|
|
name string
|
|
replicaLSN uint64
|
|
wantErr bool
|
|
}{
|
|
// replicaLSN + tolerance < primaryLSN → ineligible
|
|
// 149 + 50 = 199 < 200 → INELIGIBLE
|
|
{"below_tolerance", primaryLSN - tolerance - 1, true},
|
|
// 150 + 50 = 200 == 200 → ELIGIBLE (not strict less-than)
|
|
{"at_tolerance", primaryLSN - tolerance, false},
|
|
// 151 + 50 = 201 > 200 → ELIGIBLE
|
|
{"above_tolerance", primaryLSN - tolerance + 1, false},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ms := qaCP82Master(t)
|
|
ms.blockRegistry.SetPromotionLSNTolerance(tolerance)
|
|
|
|
now := time.Now()
|
|
qaRegisterRF3WithState(t, ms, "vol-tol", "vs1:9333",
|
|
[]struct {
|
|
Server string
|
|
HealthScore float64
|
|
WALHeadLSN uint64
|
|
Role blockvol.Role
|
|
Heartbeat time.Time
|
|
}{
|
|
{"vs2:9333", 1.0, tc.replicaLSN, blockvol.RoleReplica, now},
|
|
},
|
|
primaryLSN,
|
|
)
|
|
|
|
_, err := ms.blockRegistry.PromoteBestReplica("vol-tol")
|
|
if tc.wantErr && err == nil {
|
|
t.Fatalf("replicaLSN=%d: expected promotion to fail", tc.replicaLSN)
|
|
}
|
|
if !tc.wantErr && err != nil {
|
|
t.Fatalf("replicaLSN=%d: expected promotion to succeed: %v", tc.replicaLSN, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// QA-6: MasterRestart_ReconstructReplicas_ThenFailover
|
|
//
|
|
// Simulates master restart: empty registry, full heartbeats
|
|
// from primary and replica servers reconstruct state, then
|
|
// failover correctly promotes the replica.
|
|
// ────────────────────────────────────────────────────────────
|
|
func TestQA_CP82_MasterRestart_ReconstructReplicas_ThenFailover(t *testing.T) {
|
|
ms := qaCP82Master(t)
|
|
|
|
// Phase 1: Primary heartbeat — auto-registers volume from heartbeat.
|
|
ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{
|
|
{
|
|
Path: "/data/vol-restart.blk",
|
|
VolumeSize: 1 << 30,
|
|
Epoch: 5,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
|
WalHeadLsn: 500,
|
|
},
|
|
}, "")
|
|
|
|
entry, ok := ms.blockRegistry.Lookup("vol-restart")
|
|
if !ok {
|
|
t.Fatal("volume should be auto-registered from primary heartbeat")
|
|
}
|
|
if entry.VolumeServer != "vs1:9333" {
|
|
t.Fatalf("primary: got %q, want vs1:9333", entry.VolumeServer)
|
|
}
|
|
|
|
// Phase 2: Replica heartbeat — should reconstruct ReplicaInfo.
|
|
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
|
|
{
|
|
Path: "/data/vol-restart.blk",
|
|
VolumeSize: 1 << 30,
|
|
Epoch: 5,
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
|
WalHeadLsn: 498,
|
|
},
|
|
}, "")
|
|
|
|
entry, _ = ms.blockRegistry.Lookup("vol-restart")
|
|
if len(entry.Replicas) == 0 {
|
|
t.Fatal("replica should be reconstructed from heartbeat (Fix #3)")
|
|
}
|
|
if entry.Replicas[0].Server != "vs2:9333" {
|
|
t.Fatalf("replica server: got %q, want vs2:9333", entry.Replicas[0].Server)
|
|
}
|
|
|
|
// Phase 3: Set lease expired and trigger failover.
|
|
ms.blockRegistry.UpdateEntry("vol-restart", func(e *BlockVolumeEntry) {
|
|
e.LeaseTTL = 5 * time.Second
|
|
e.LastLeaseGrant = time.Now().Add(-1 * time.Minute)
|
|
// Ensure replica is eligible for promotion.
|
|
e.Replicas[0].LastHeartbeat = time.Now()
|
|
e.Replicas[0].Role = blockvol.RoleToWire(blockvol.RoleReplica)
|
|
})
|
|
|
|
ms.failoverBlockVolumes("vs1:9333")
|
|
|
|
entry, ok = ms.blockRegistry.Lookup("vol-restart")
|
|
if !ok {
|
|
t.Fatal("volume should survive failover")
|
|
}
|
|
if entry.VolumeServer != "vs2:9333" {
|
|
t.Fatalf("after failover: primary should be vs2:9333, got %q", entry.VolumeServer)
|
|
}
|
|
if entry.Epoch <= 5 {
|
|
t.Fatalf("epoch should be bumped, got %d", entry.Epoch)
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// QA-7: RF3_OneReplicaFlaps_UnderWriteLoad
|
|
//
|
|
// One replica rapidly disconnects/reconnects while concurrent
|
|
// failover checks run. Must not cause split-brain or panic.
|
|
// Simulates via concurrent UpdateFullHeartbeat + failoverBlockVolumes.
|
|
// ────────────────────────────────────────────────────────────
|
|
func TestQA_CP82_RF3_OneReplicaFlaps_UnderWriteLoad(t *testing.T) {
|
|
ms := qaCP82Master(t)
|
|
now := time.Now()
|
|
qaRegisterRF3WithState(t, ms, "vol-flap", "vs1:9333",
|
|
[]struct {
|
|
Server string
|
|
HealthScore float64
|
|
WALHeadLSN uint64
|
|
Role blockvol.Role
|
|
Heartbeat time.Time
|
|
}{
|
|
{"vs2:9333", 1.0, 100, blockvol.RoleReplica, now},
|
|
{"vs3:9333", 1.0, 100, blockvol.RoleReplica, now},
|
|
},
|
|
100,
|
|
)
|
|
|
|
var wg sync.WaitGroup
|
|
const rounds = 50
|
|
|
|
// Goroutine 1: flapping replica vs3 (heartbeat with/without volume).
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for i := 0; i < rounds; i++ {
|
|
if i%2 == 0 {
|
|
// Replica reports volume.
|
|
ms.blockRegistry.UpdateFullHeartbeat("vs3:9333", []*master_pb.BlockVolumeInfoMessage{
|
|
{
|
|
Path: "/data/vol-flap.blk",
|
|
VolumeSize: 1 << 30,
|
|
Epoch: 1,
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
|
WalHeadLsn: 100,
|
|
},
|
|
}, "")
|
|
} else {
|
|
// Replica reports no volumes (simulates disconnect).
|
|
ms.blockRegistry.UpdateFullHeartbeat("vs3:9333", []*master_pb.BlockVolumeInfoMessage{}, "")
|
|
}
|
|
}
|
|
}()
|
|
|
|
// Goroutine 2: concurrent heartbeats from primary.
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for i := 0; i < rounds; i++ {
|
|
ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{
|
|
{
|
|
Path: "/data/vol-flap.blk",
|
|
VolumeSize: 1 << 30,
|
|
Epoch: 1,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
|
WalHeadLsn: uint64(100 + i),
|
|
},
|
|
}, "")
|
|
}
|
|
}()
|
|
|
|
// Goroutine 3: concurrent heartbeats from stable replica vs2.
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for i := 0; i < rounds; i++ {
|
|
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
|
|
{
|
|
Path: "/data/vol-flap.blk",
|
|
VolumeSize: 1 << 30,
|
|
Epoch: 1,
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
|
WalHeadLsn: uint64(100 + i),
|
|
},
|
|
}, "")
|
|
}
|
|
}()
|
|
|
|
wg.Wait()
|
|
|
|
// Invariant: volume must still exist, primary must be vs1.
|
|
entry, ok := ms.blockRegistry.Lookup("vol-flap")
|
|
if !ok {
|
|
t.Fatal("volume must survive flapping replica")
|
|
}
|
|
if entry.VolumeServer != "vs1:9333" {
|
|
t.Fatalf("primary must remain vs1:9333, got %q (split-brain!)", entry.VolumeServer)
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// QA-8: AssignmentPrecedence_ReplicaAddrsVsScalar
|
|
//
|
|
// Creates RF=3 volume and verifies primary assignment includes
|
|
// ReplicaAddrs (not just scalar fields). VS should use
|
|
// ReplicaAddrs when non-empty.
|
|
// ────────────────────────────────────────────────────────────
|
|
func TestQA_CP82_AssignmentPrecedence_ReplicaAddrsVsScalar(t *testing.T) {
|
|
ms := qaCP82Master(t)
|
|
ctx := context.Background()
|
|
|
|
resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
|
|
Name: "vol-addrs",
|
|
SizeBytes: 1 << 30,
|
|
ReplicaFactor: 3,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create: %v", err)
|
|
}
|
|
|
|
primary := resp.VolumeServer
|
|
assignments := ms.blockAssignmentQueue.Peek(primary)
|
|
|
|
// Find the primary assignment.
|
|
var primaryAssignment *blockvol.BlockVolumeAssignment
|
|
for i := range assignments {
|
|
if blockvol.RoleFromWire(assignments[i].Role) == blockvol.RolePrimary {
|
|
primaryAssignment = &assignments[i]
|
|
break
|
|
}
|
|
}
|
|
|
|
if primaryAssignment == nil {
|
|
t.Fatal("no primary assignment found")
|
|
}
|
|
|
|
// RF=3 → 2 replicas → ReplicaAddrs should have 2 entries.
|
|
if len(primaryAssignment.ReplicaAddrs) != 2 {
|
|
t.Fatalf("primary assignment ReplicaAddrs: got %d, want 2", len(primaryAssignment.ReplicaAddrs))
|
|
}
|
|
|
|
// Verify each ReplicaAddr has data+ctrl.
|
|
for i, ra := range primaryAssignment.ReplicaAddrs {
|
|
if ra.DataAddr == "" || ra.CtrlAddr == "" {
|
|
t.Errorf("ReplicaAddrs[%d]: data=%q ctrl=%q — both must be non-empty", i, ra.DataAddr, ra.CtrlAddr)
|
|
}
|
|
}
|
|
|
|
// For RF=3, legacy scalar ReplicaDataAddr is NOT set on the primary
|
|
// assignment (only ReplicaAddrs is used). Verify precedence: ReplicaAddrs
|
|
// is the authoritative source when non-empty.
|
|
if len(primaryAssignment.ReplicaAddrs) > 0 && primaryAssignment.ReplicaDataAddr != "" {
|
|
t.Error("legacy ReplicaDataAddr should be empty when ReplicaAddrs is populated (RF=3)")
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// QA-9: ScrubConcurrentWrites_NoFalseCorruption
|
|
//
|
|
// Heavy writes while scrub runs. Expect zero false positives.
|
|
// Tests at the HealthScore level (engine scrub tested separately).
|
|
// ────────────────────────────────────────────────────────────
|
|
func TestQA_CP82_ScrubConcurrentWrites_NoFalseCorruption(t *testing.T) {
|
|
ms := qaCP82Master(t)
|
|
now := time.Now()
|
|
qaRegisterRF3WithState(t, ms, "vol-scrub", "vs1:9333",
|
|
[]struct {
|
|
Server string
|
|
HealthScore float64
|
|
WALHeadLSN uint64
|
|
Role blockvol.Role
|
|
Heartbeat time.Time
|
|
}{
|
|
{"vs2:9333", 1.0, 500, blockvol.RoleReplica, now},
|
|
{"vs3:9333", 1.0, 500, blockvol.RoleReplica, now},
|
|
},
|
|
500,
|
|
)
|
|
|
|
var wg sync.WaitGroup
|
|
const rounds = 100
|
|
|
|
// Simulate heartbeats with varying health scores
|
|
// (as would happen during scrub with concurrent writes).
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for i := 0; i < rounds; i++ {
|
|
// Health stays 1.0 — no corruption detected (correct behavior
|
|
// when writes are excluded from scrub checks).
|
|
ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{
|
|
{
|
|
Path: "/data/vol-scrub.blk",
|
|
VolumeSize: 1 << 30,
|
|
Epoch: 1,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
|
WalHeadLsn: uint64(500 + i),
|
|
HealthScore: 1.0, // Clean — no false positives
|
|
},
|
|
}, "")
|
|
}
|
|
}()
|
|
|
|
// Concurrent replica heartbeats.
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for i := 0; i < rounds; i++ {
|
|
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
|
|
{
|
|
Path: "/data/vol-scrub.blk",
|
|
VolumeSize: 1 << 30,
|
|
Epoch: 1,
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
|
WalHeadLsn: uint64(500 + i),
|
|
HealthScore: 1.0,
|
|
},
|
|
}, "")
|
|
}
|
|
}()
|
|
|
|
wg.Wait()
|
|
|
|
entry, _ := ms.blockRegistry.Lookup("vol-scrub")
|
|
if entry.HealthScore < 1.0 {
|
|
t.Fatalf("health score should be 1.0 (no corruption), got %f", entry.HealthScore)
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// QA-10: ScrubDetectsCorruption_HealthDrops_PromotionAvoids
|
|
//
|
|
// One replica has corruption (low health), the other is clean.
|
|
// Promotion must prefer the healthy replica.
|
|
// ────────────────────────────────────────────────────────────
|
|
func TestQA_CP82_ScrubDetectsCorruption_HealthDrops_PromotionAvoids(t *testing.T) {
|
|
ms := qaCP82Master(t)
|
|
now := time.Now()
|
|
qaRegisterRF3WithState(t, ms, "vol-corrupt", "vs1:9333",
|
|
[]struct {
|
|
Server string
|
|
HealthScore float64
|
|
WALHeadLSN uint64
|
|
Role blockvol.Role
|
|
Heartbeat time.Time
|
|
}{
|
|
{"vs2:9333", 0.3, 100, blockvol.RoleReplica, now}, // corrupted (low health)
|
|
{"vs3:9333", 1.0, 100, blockvol.RoleReplica, now}, // clean
|
|
},
|
|
100,
|
|
)
|
|
|
|
// Simulate heartbeat reporting low health on vs2 (scrub found errors).
|
|
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
|
|
{
|
|
Path: "/data/vol-corrupt.blk",
|
|
VolumeSize: 1 << 30,
|
|
Epoch: 1,
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
|
WalHeadLsn: 100,
|
|
HealthScore: 0.3,
|
|
ScrubErrors: 5,
|
|
},
|
|
}, "")
|
|
|
|
// Trigger failover — vs3 (healthy) should be promoted, not vs2.
|
|
newEpoch, err := ms.blockRegistry.PromoteBestReplica("vol-corrupt")
|
|
if err != nil {
|
|
t.Fatalf("promotion should succeed: %v", err)
|
|
}
|
|
if newEpoch != 2 {
|
|
t.Fatalf("epoch: got %d, want 2", newEpoch)
|
|
}
|
|
|
|
entry, _ := ms.blockRegistry.Lookup("vol-corrupt")
|
|
if entry.VolumeServer != "vs3:9333" {
|
|
t.Fatalf("vs3 (healthy) should be promoted, got %q", entry.VolumeServer)
|
|
}
|
|
// vs2 (corrupted) should remain as replica.
|
|
if len(entry.Replicas) != 1 || entry.Replicas[0].Server != "vs2:9333" {
|
|
t.Fatalf("vs2 should remain as replica, got %+v", entry.Replicas)
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// QA-11: ExpandRF3_PartialReplicaFailure
|
|
//
|
|
// Primary expand succeeds, one replica expand fails.
|
|
// Verify: operation succeeds (best-effort), registry updated,
|
|
// failed replica state is degraded.
|
|
// ────────────────────────────────────────────────────────────
|
|
func TestQA_CP82_ExpandRF3_PartialReplicaFailure(t *testing.T) {
|
|
ms := qaCP82Master(t)
|
|
ctx := context.Background()
|
|
|
|
// Create RF=3 volume first.
|
|
_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
|
|
Name: "vol-expand",
|
|
SizeBytes: 1 << 30,
|
|
ReplicaFactor: 3,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create: %v", err)
|
|
}
|
|
|
|
entry, _ := ms.blockRegistry.Lookup("vol-expand")
|
|
failServer := entry.Replicas[1].Server
|
|
|
|
// CP11A-2: coordinated expand — set up prepare/commit/cancel mocks.
|
|
ms.blockVSPrepareExpand = func(ctx context.Context, server string, name string, newSize, expandEpoch uint64) error {
|
|
return nil
|
|
}
|
|
ms.blockVSCommitExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) (uint64, error) {
|
|
if server == failServer {
|
|
return 0, fmt.Errorf("disk full on %s", server)
|
|
}
|
|
return 2 << 30, nil
|
|
}
|
|
ms.blockVSCancelExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) error {
|
|
return nil
|
|
}
|
|
|
|
// Under coordinated expand, partial replica commit failure marks the volume degraded.
|
|
_, err = ms.ExpandBlockVolume(ctx, &master_pb.ExpandBlockVolumeRequest{
|
|
Name: "vol-expand",
|
|
NewSizeBytes: 2 << 30,
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expand should fail when a required replica commit fails")
|
|
}
|
|
|
|
// Registry size should NOT be updated (primary committed but replica failed → degraded).
|
|
entry, _ = ms.blockRegistry.Lookup("vol-expand")
|
|
if entry.SizeBytes != 1<<30 {
|
|
t.Fatalf("registry size should be unchanged: got %d, want %d", entry.SizeBytes, uint64(1<<30))
|
|
}
|
|
if !entry.ExpandFailed {
|
|
t.Fatal("ExpandFailed should be true after partial commit failure")
|
|
}
|
|
if !entry.ExpandInProgress {
|
|
t.Fatal("ExpandInProgress should stay true to suppress heartbeat size updates")
|
|
}
|
|
|
|
// Cleanup: ClearExpandFailed allows future operations.
|
|
ms.blockRegistry.ClearExpandFailed("vol-expand")
|
|
|
|
// Now expand with all mocks succeeding should work.
|
|
ms.blockVSCommitExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) (uint64, error) {
|
|
return 2 << 30, nil
|
|
}
|
|
resp, err := ms.ExpandBlockVolume(ctx, &master_pb.ExpandBlockVolumeRequest{
|
|
Name: "vol-expand",
|
|
NewSizeBytes: 2 << 30,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("retry expand after clear: %v", err)
|
|
}
|
|
if resp.CapacityBytes != 2<<30 {
|
|
t.Fatalf("capacity: got %d, want %d", resp.CapacityBytes, uint64(2<<30))
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// QA-12: ByServerIndexConsistency_AfterReplicaMove
|
|
//
|
|
// Moves replica between servers repeatedly. ListByServer and
|
|
// byServer counts must stay exact — no dangling references.
|
|
// ────────────────────────────────────────────────────────────
|
|
func TestQA_CP82_ByServerIndexConsistency_AfterReplicaMove(t *testing.T) {
|
|
ms := qaCP82Master(t)
|
|
now := time.Now()
|
|
qaRegisterRF3WithState(t, ms, "vol-move", "vs1:9333",
|
|
[]struct {
|
|
Server string
|
|
HealthScore float64
|
|
WALHeadLSN uint64
|
|
Role blockvol.Role
|
|
Heartbeat time.Time
|
|
}{
|
|
{"vs2:9333", 1.0, 100, blockvol.RoleReplica, now},
|
|
},
|
|
100,
|
|
)
|
|
|
|
// Verify initial state: vs1 and vs2 both have vol-move.
|
|
assertServerHasVolume(t, ms, "vs1:9333", "vol-move")
|
|
assertServerHasVolume(t, ms, "vs2:9333", "vol-move")
|
|
|
|
// Move replica from vs2 to vs3 repeatedly.
|
|
servers := []string{"vs3:9333", "vs2:9333", "vs3:9333", "vs2:9333", "vs3:9333"}
|
|
for _, newServer := range servers {
|
|
// Remove old replica.
|
|
entry, _ := ms.blockRegistry.Lookup("vol-move")
|
|
oldServer := ""
|
|
if len(entry.Replicas) > 0 {
|
|
oldServer = entry.Replicas[0].Server
|
|
}
|
|
if oldServer != "" {
|
|
if err := ms.blockRegistry.RemoveReplica("vol-move", oldServer); err != nil {
|
|
t.Fatalf("remove replica from %s: %v", oldServer, err)
|
|
}
|
|
}
|
|
|
|
// Add new replica.
|
|
if err := ms.blockRegistry.AddReplica("vol-move", ReplicaInfo{
|
|
Server: newServer,
|
|
Path: "/data/vol-move.blk",
|
|
IQN: fmt.Sprintf("iqn.2024.test:vol-move-r"),
|
|
ISCSIAddr: newServer + ":3260",
|
|
HealthScore: 1.0,
|
|
}); err != nil {
|
|
t.Fatalf("add replica on %s: %v", newServer, err)
|
|
}
|
|
|
|
// Invariant: primary always indexed.
|
|
assertServerHasVolume(t, ms, "vs1:9333", "vol-move")
|
|
// New replica indexed.
|
|
assertServerHasVolume(t, ms, newServer, "vol-move")
|
|
// Old server should NOT have it (unless it's the primary).
|
|
if oldServer != "" && oldServer != "vs1:9333" && oldServer != newServer {
|
|
assertServerDoesNotHaveVolume(t, ms, oldServer, "vol-move")
|
|
}
|
|
}
|
|
|
|
// Final check: ListAll should have exactly 1 volume.
|
|
all := ms.blockRegistry.ListAll()
|
|
if len(all) != 1 {
|
|
t.Fatalf("expected 1 volume, got %d", len(all))
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// Helpers
|
|
// ────────────────────────────────────────────────────────────
|
|
|
|
func assertServerHasVolume(t *testing.T, ms *MasterServer, server, volName string) {
|
|
t.Helper()
|
|
entries := ms.blockRegistry.ListByServer(server)
|
|
for _, e := range entries {
|
|
if e.Name == volName {
|
|
return
|
|
}
|
|
}
|
|
t.Errorf("server %q should have volume %q in byServer index", server, volName)
|
|
}
|
|
|
|
func assertServerDoesNotHaveVolume(t *testing.T, ms *MasterServer, server, volName string) {
|
|
t.Helper()
|
|
entries := ms.blockRegistry.ListByServer(server)
|
|
for _, e := range entries {
|
|
if e.Name == volName {
|
|
t.Errorf("server %q should NOT have volume %q in byServer index", server, volName)
|
|
return
|
|
}
|
|
}
|
|
}
|