From 560ffee182e8ce12b1748df91f590bdd8c1aea6f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 26 Sep 2026 15:45:41 +0000 Subject: [PATCH] fix(failover): emit chain.switched when leaving degraded in place recomputeLocked only fired the event on an active-target change or on entering degraded. When the active target itself recovered while every target was down, the chain silently left degraded with no event, so SSE/realtime consumers tracking chain.switched.state got stuck on "degraded". Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Ettore Di Giacinto --- core/services/failover/manager.go | 6 ++++++ core/services/failover/manager_test.go | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/core/services/failover/manager.go b/core/services/failover/manager.go index c52aa2b7d..1169061bf 100644 --- a/core/services/failover/manager.go +++ b/core/services/failover/manager.go @@ -305,7 +305,13 @@ func (m *Manager) recomputeLocked(ch *chainState, override Reason) { ch.activeSince = now m.emitLocked(Event{Type: EventChainSwitched, Chain: ch.name, From: ch.targets[prev], To: ch.targets[next], State: string(state), Reason: reason, At: now}) case state == ChainDegraded && ch.state != ChainDegraded: + // Entering degraded with no active-target change (every target is down). m.emitLocked(Event{Type: EventChainSwitched, Chain: ch.name, From: ch.targets[prev], To: ch.targets[next], State: string(state), Reason: ReasonDegraded, At: now}) + case state != ChainDegraded && ch.state == ChainDegraded: + // Leaving degraded with no active-target change (the active target + // itself recovered): SSE/realtime consumers watch chain.switched.state, + // so this must fire or they stay on "degraded" forever. + m.emitLocked(Event{Type: EventChainSwitched, Chain: ch.name, From: ch.targets[prev], To: ch.targets[next], State: string(state), Reason: ReasonRecovery, At: now}) } ch.state = state } diff --git a/core/services/failover/manager_test.go b/core/services/failover/manager_test.go index f10a04f2c..3bef7945e 100644 --- a/core/services/failover/manager_test.go +++ b/core/services/failover/manager_test.go @@ -132,6 +132,25 @@ var _ = Describe("Manager", func() { Expect(att.Fail(errBoom)).To(BeFalse()) }) + It("emits chain.switched when leaving degraded without an active-target change", func() { + m.ReportFailure("a", errBoom) // active moves to b + m.ReportFailure("b", errBoom) // both down: degraded, active stays b + st, _ := m.ChainStatus("chain") + Expect(st.State).To(Equal(ChainDegraded)) + Expect(st.Active).To(Equal("b")) + events, cancel := m.Subscribe(16) + defer cancel() + m.ReportSuccess("b") // b is cold local: one success recovers it in place + st, _ = m.ChainStatus("chain") + Expect(st.State).To(Equal(ChainFallback)) + Expect(st.Active).To(Equal("b"), "the active target itself recovered, no switch needed") + sw := switched(drain(events)) + Expect(sw).To(HaveLen(1), "leaving degraded must still notify chain.switched listeners") + Expect(sw[0]).To(MatchFields(IgnoreExtras, Fields{ + "Chain": Equal("chain"), "State": Equal("fallback"), "Reason": Equal(ReasonRecovery), + })) + }) + It("pins a target regardless of health", func() { Expect(m.Pin("chain", "b")).To(Succeed()) att, _ := m.Plan("chain")