Browse Source
fix: add RecoveryOrchestrator as real integrated entry path
fix: add RecoveryOrchestrator as real integrated entry path
New: orchestrator.go — RecoveryOrchestrator drives recovery lifecycle from assignment through execution to completion/escalation: - ProcessAssignment: reconcile + session creation + auto-log - ExecuteRecovery: connect → handshake from RetainedHistory → outcome - CompleteCatchUp: begin catch-up → progress → complete + auto-log - CompleteRebuild: connect → handshake → history-driven source → transfer → tail replay → complete + auto-log - InvalidateEpoch: invalidate stale sessions + auto-log All integration tests rewritten to use orchestrator as entry path. No direct sender API calls in recovery lifecycle. SessionSnapshot now includes: TruncateRequired/ToLSN/Recorded, RebuildSource, RebuildPhase. RecoveryLog is auto-populated by orchestrator at every transition. 7 integration tests via orchestrator: - ChangedAddress, NeedsRebuild→Rebuild, EpochBump, MultiReplica - Observability: session snapshot, rebuild snapshot, auto-populated log Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>feature/sw-block
pingqiu
2 days ago
3 changed files with 390 additions and 174 deletions
-
359sw-block/engine/replication/integration_test.go
-
186sw-block/engine/replication/orchestrator.go
-
19sw-block/engine/replication/sender.go
@ -0,0 +1,186 @@ |
|||||
|
package replication |
||||
|
|
||||
|
import "fmt" |
||||
|
|
||||
|
// RecoveryOrchestrator drives the recovery lifecycle from assignment intent
|
||||
|
// through execution to completion/escalation. It is the integrated entry
|
||||
|
// path above raw Sender APIs — callers interact with the orchestrator,
|
||||
|
// not with individual sender execution methods.
|
||||
|
//
|
||||
|
// The orchestrator owns:
|
||||
|
// - assignment processing (reconcile + session creation)
|
||||
|
// - handshake evaluation (from RetainedHistory)
|
||||
|
// - recovery execution (catch-up or rebuild through completion)
|
||||
|
// - automatic event logging at every lifecycle transition
|
||||
|
type RecoveryOrchestrator struct { |
||||
|
Registry *Registry |
||||
|
Log *RecoveryLog |
||||
|
} |
||||
|
|
||||
|
// NewRecoveryOrchestrator creates an orchestrator with a fresh registry and log.
|
||||
|
func NewRecoveryOrchestrator() *RecoveryOrchestrator { |
||||
|
return &RecoveryOrchestrator{ |
||||
|
Registry: NewRegistry(), |
||||
|
Log: NewRecoveryLog(), |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// ProcessAssignment applies an assignment intent and logs the result.
|
||||
|
func (o *RecoveryOrchestrator) ProcessAssignment(intent AssignmentIntent) AssignmentResult { |
||||
|
result := o.Registry.ApplyAssignment(intent) |
||||
|
for _, id := range result.Added { |
||||
|
o.Log.Record(id, 0, "sender_added", "") |
||||
|
} |
||||
|
for _, id := range result.Removed { |
||||
|
o.Log.Record(id, 0, "sender_removed", "") |
||||
|
} |
||||
|
for _, id := range result.SessionsCreated { |
||||
|
s := o.Registry.Sender(id) |
||||
|
o.Log.Record(id, s.SessionID(), "session_created", "") |
||||
|
} |
||||
|
for _, id := range result.SessionsSuperseded { |
||||
|
s := o.Registry.Sender(id) |
||||
|
o.Log.Record(id, s.SessionID(), "session_superseded", "") |
||||
|
} |
||||
|
for _, id := range result.SessionsFailed { |
||||
|
o.Log.Record(id, 0, "session_failed", "") |
||||
|
} |
||||
|
return result |
||||
|
} |
||||
|
|
||||
|
// RecoveryResult captures the outcome of a single replica recovery attempt.
|
||||
|
type RecoveryResult struct { |
||||
|
ReplicaID string |
||||
|
Outcome RecoveryOutcome |
||||
|
Proof *RecoverabilityProof |
||||
|
FinalState ReplicaState |
||||
|
Error error |
||||
|
} |
||||
|
|
||||
|
// ExecuteRecovery runs the full recovery flow for a single replica:
|
||||
|
// connect → handshake (from history) → catch-up or escalate.
|
||||
|
//
|
||||
|
// For catch-up outcomes, the caller provides entries via the returned
|
||||
|
// CatchUpHandle. For rebuild outcomes, the sender is left at NeedsRebuild
|
||||
|
// and requires a separate rebuild assignment.
|
||||
|
func (o *RecoveryOrchestrator) ExecuteRecovery(replicaID string, replicaFlushedLSN uint64, history *RetainedHistory) RecoveryResult { |
||||
|
s := o.Registry.Sender(replicaID) |
||||
|
if s == nil { |
||||
|
return RecoveryResult{ReplicaID: replicaID, Error: fmt.Errorf("sender not found")} |
||||
|
} |
||||
|
|
||||
|
sessID := s.SessionID() |
||||
|
if sessID == 0 { |
||||
|
return RecoveryResult{ReplicaID: replicaID, Error: fmt.Errorf("no session")} |
||||
|
} |
||||
|
|
||||
|
// Connect.
|
||||
|
if err := s.BeginConnect(sessID); err != nil { |
||||
|
o.Log.Record(replicaID, sessID, "connect_failed", err.Error()) |
||||
|
return RecoveryResult{ReplicaID: replicaID, FinalState: s.State(), Error: err} |
||||
|
} |
||||
|
o.Log.Record(replicaID, sessID, "connected", "") |
||||
|
|
||||
|
// Handshake from history.
|
||||
|
outcome, proof, err := s.RecordHandshakeFromHistory(sessID, replicaFlushedLSN, history) |
||||
|
if err != nil { |
||||
|
o.Log.Record(replicaID, sessID, "handshake_failed", err.Error()) |
||||
|
return RecoveryResult{ReplicaID: replicaID, Outcome: outcome, Proof: proof, FinalState: s.State(), Error: err} |
||||
|
} |
||||
|
o.Log.Record(replicaID, sessID, "handshake", fmt.Sprintf("outcome=%s", outcome)) |
||||
|
|
||||
|
if outcome == OutcomeNeedsRebuild { |
||||
|
o.Log.Record(replicaID, sessID, "escalated", fmt.Sprintf("needs_rebuild: %s", proof.Reason)) |
||||
|
return RecoveryResult{ReplicaID: replicaID, Outcome: outcome, Proof: proof, FinalState: StateNeedsRebuild} |
||||
|
} |
||||
|
|
||||
|
return RecoveryResult{ReplicaID: replicaID, Outcome: outcome, Proof: proof, FinalState: s.State()} |
||||
|
} |
||||
|
|
||||
|
// CompleteCatchUp drives catch-up from startLSN to targetLSN and completes.
|
||||
|
// Called after ExecuteRecovery returns OutcomeCatchUp.
|
||||
|
func (o *RecoveryOrchestrator) CompleteCatchUp(replicaID string, targetLSN uint64) error { |
||||
|
s := o.Registry.Sender(replicaID) |
||||
|
if s == nil { |
||||
|
return fmt.Errorf("sender not found") |
||||
|
} |
||||
|
sessID := s.SessionID() |
||||
|
|
||||
|
if err := s.BeginCatchUp(sessID); err != nil { |
||||
|
o.Log.Record(replicaID, sessID, "catchup_failed", err.Error()) |
||||
|
return err |
||||
|
} |
||||
|
o.Log.Record(replicaID, sessID, "catchup_started", "") |
||||
|
|
||||
|
if err := s.RecordCatchUpProgress(sessID, targetLSN); err != nil { |
||||
|
o.Log.Record(replicaID, sessID, "catchup_progress_failed", err.Error()) |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
if !s.CompleteSessionByID(sessID) { |
||||
|
o.Log.Record(replicaID, sessID, "completion_rejected", "") |
||||
|
return fmt.Errorf("completion rejected") |
||||
|
} |
||||
|
o.Log.Record(replicaID, sessID, "completed", "in_sync") |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// CompleteRebuild drives the rebuild from history and completes.
|
||||
|
// Called after a rebuild assignment when the sender is at NeedsRebuild.
|
||||
|
func (o *RecoveryOrchestrator) CompleteRebuild(replicaID string, history *RetainedHistory) error { |
||||
|
s := o.Registry.Sender(replicaID) |
||||
|
if s == nil { |
||||
|
return fmt.Errorf("sender not found") |
||||
|
} |
||||
|
sessID := s.SessionID() |
||||
|
|
||||
|
if err := s.BeginConnect(sessID); err != nil { |
||||
|
o.Log.Record(replicaID, sessID, "rebuild_connect_failed", err.Error()) |
||||
|
return err |
||||
|
} |
||||
|
o.Log.Record(replicaID, sessID, "rebuild_connected", "") |
||||
|
|
||||
|
if err := s.RecordHandshake(sessID, 0, history.CommittedLSN); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
if err := s.SelectRebuildFromHistory(sessID, history); err != nil { |
||||
|
o.Log.Record(replicaID, sessID, "rebuild_source_failed", err.Error()) |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
snap := s.SessionSnapshot() |
||||
|
o.Log.Record(replicaID, sessID, "rebuild_source_selected", fmt.Sprintf("kind=%s", snap.Kind)) |
||||
|
|
||||
|
if err := s.BeginRebuildTransfer(sessID); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Determine transfer target based on rebuild source.
|
||||
|
source, snapLSN := history.RebuildSourceDecision() |
||||
|
if source == RebuildSnapshotTail { |
||||
|
s.RecordRebuildTransferProgress(sessID, snapLSN) |
||||
|
if err := s.BeginRebuildTailReplay(sessID); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
s.RecordRebuildTailProgress(sessID, history.CommittedLSN) |
||||
|
} else { |
||||
|
s.RecordRebuildTransferProgress(sessID, history.CommittedLSN) |
||||
|
} |
||||
|
|
||||
|
if err := s.CompleteRebuild(sessID); err != nil { |
||||
|
o.Log.Record(replicaID, sessID, "rebuild_failed", err.Error()) |
||||
|
return err |
||||
|
} |
||||
|
o.Log.Record(replicaID, sessID, "rebuild_completed", "in_sync") |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// InvalidateEpoch invalidates all stale sessions and logs the event.
|
||||
|
func (o *RecoveryOrchestrator) InvalidateEpoch(newEpoch uint64) int { |
||||
|
count := o.Registry.InvalidateEpoch(newEpoch) |
||||
|
if count > 0 { |
||||
|
o.Log.Record("", 0, "epoch_invalidation", fmt.Sprintf("epoch=%d invalidated=%d", newEpoch, count)) |
||||
|
} |
||||
|
return count |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue