Keep Fleet outage state visible and reject superseded responses

docs/fleet-recovery-qualification-20261007.md

@@ -0,0 +1,47 @@
+# Fleet recovery qualification — 7 October 2026
+
+Status: source regression fixes qualified; deployed-browser and distributed
+transport acceptance remain open. This supplements tasks 8, 10 and 12 without
+claiming that the full Fleet acceptance matrix is complete.
+
+## Findings and changes
+
+A failed status poll only surfaced an error during the initial load. Once reports
+were cached, network failures silently left the previous view in place. The view
+now keeps those reports, selection and sort order, with an explicit failed-refresh
+notice and Retry. An initial failure without reports remains an unavailable state.
+A successful refresh clears the notice. Report timestamps continue ageing; a
+monitoring failure is not relabelled as proof that a peer is offline.
+
+Concurrent refreshes could overwrite newer status/alerts with older replies.
+Status, alerts and refresh completion now track request generations. Replies
+arriving after unmount cannot repopulate the session cache or change the view.
+Malformed status envelopes cannot advance the successful-refresh timestamp.
+
+## Evidence
+
+Focused Vitest run: **21 tests passed in four files**, zero failures, including
+actual Fleet view rendering with retained cards, outage notice and successful
+Retry. Other cases cover a missing cache, late status/alert replies, a malformed
+status response, late responses after unmount, real zero versus unavailable
+metrics, report ageing, sorting and selected-node history isolation.
+
+Log: `/tmp/archy-fleet-recovery-tests-final.log`.
+This is local component/composable evidence, not a live-node outage test.
+No node service, peer relationship, wallet or payment was changed.
+
+## Remaining acceptance matrix
+
+| Area | Established evidence | Remaining |
+| --- | --- | --- |
+| Report rendering | Local normalization/render tests; earlier dev/Yaya metrics evidence in progress ledger | Updated sender/receiver comparison across supported versions |
+| Status freshness | Clock-driven ageing, unknown versus stale, no synthetic metric zero | Real partition, reconnect and transport delivery |
+| Refresh failures | Visible retained reports; empty failure; Retry recovery | Candidate browser at phone/desktop widths and deployed artifact |
+| Concurrent replies | Newest status/alerts win; no unmount cache repopulation | Distributed delay/out-of-order delivery |
+| Selection/sorting | Retained across failure; history request isolation | Larger-fleet navigation and keyboard checks |
+| Authorization | No permission changes in this patch | Unauthorized/expired-trust actual-handler matrix |
+| Remote actions/updates | Outside this patch | Disposable-node success/partial failure/restart/update matrix |
+| FIPS | Existing preferred authenticated transport unchanged | Measured latency, fallback, reconnect and mixed capabilities |
+
+No current-node restart or deliberate network outage was performed. Live fault
+qualification must preserve wallets, installed applications and operator intent.

neode-ui/src/views/Fleet.vue

@@ -71,13 +71,19 @@
     </div>
 
     <!-- Error State -->
-    <div v-else-if="fleet.errorMessage.value" class="glass-card p-6 mb-6">
+    <div v-else-if="fleet.errorMessage.value && !fleet.nodes.value.length" class="glass-card p-6 mb-6">
       <div class="alert-error rounded-lg mb-4">{{ fleet.errorMessage.value }}</div>
       <button class="glass-button text-sm px-4 py-2" @click="fleet.refreshAll">Retry</button>
     </div>
 
     <!-- Dashboard Content -->
     <template v-else>
+      <div v-if="fleet.errorMessage.value" role="status" class="glass-card p-4 mb-4 flex flex-wrap items-center gap-3">
+        <p class="text-sm text-white/80 flex-1 min-w-0">{{ fleet.errorMessage.value }}</p>
+        <button class="glass-button text-sm px-4 py-2 disabled:opacity-50" :disabled="fleet.refreshing.value" @click="fleet.refreshAll">
+          {{ fleet.refreshing.value ? 'Retrying...' : 'Retry' }}
+        </button>
+      </div>
       <FleetOverviewCards
         :node-count="fleet.nodes.value.length"
         :online-count="fleet.onlineCount.value"

neode-ui/src/views/fleet/__tests__/FleetFreshness.test.ts

@@ -68,3 +68,71 @@ it('keeps a late node history response out of the newly selected node', async ()
   expect(fleet.nodeHistoryLoading.value).toBe(false)
   wrapper.unmount()
 })
+
+it('preserves cached reports and selection with a visible failure, then clears it on recovery', async () => {
+  sessionStorage.setItem('archipelago.fleet.cache.v2', JSON.stringify({
+    nodes: [{ node_id: 'retained', reported_at: '2026-10-07T12:00:00Z', cpu_pct: 0 }],
+    selectedNodeId: 'retained', sortBy: 'name', lastRefreshed: '2026-10-07T12:00:00Z',
+  }))
+  vi.mocked(rpcClient.call).mockRejectedValue(new Error('Disconnected'))
+  let fleet!: ReturnType<typeof useFleetData>
+  const wrapper = mount(defineComponent({ setup() { fleet = useFleetData(); return () => null } }))
+  try {
+    await flushPromises()
+    expect(fleet.errorMessage.value).toContain('Showing the last received reports')
+    expect(fleet.nodes.value[0]?.cpu_pct).toBe(0)
+    expect(fleet.selectedNodeId.value).toBe('retained')
+    expect(fleet.sortBy.value).toBe('name')
+    expect(fleet.lastRefreshed.value).toBe('2026-10-07T12:00:00Z')
+    vi.mocked(rpcClient.call).mockResolvedValue({ nodes: [{ node_id: 'retained', cpu_pct: 22 }], alerts: [] })
+    await fleet.refreshAll()
+    expect(fleet.errorMessage.value).toBe('')
+    expect(fleet.nodes.value[0]?.cpu_pct).toBe(22)
+    expect(fleet.selectedNodeId.value).toBe('retained')
+  } finally { wrapper.unmount() }
+})
+
+it('ignores older status and alert responses arriving after a newer refresh', async () => {
+  const statuses: Array<(value: unknown) => void> = []
+  const alerts: Array<(value: unknown) => void> = []
+  vi.mocked(rpcClient.call).mockImplementation(({ method }) => new Promise(done => {
+    (method === 'telemetry.fleet-status' ? statuses : alerts).push(done)
+  }) as never)
+  let fleet!: ReturnType<typeof useFleetData>
+  const wrapper = mount(defineComponent({ setup() { fleet = useFleetData(); return () => null } }))
+  try {
+    const latest = fleet.refreshAll()
+    statuses[1]!({ nodes: [{ node_id: 'newer', cpu_pct: 20 }] })
+    alerts[1]!({ alerts: [{ node_id: 'newer', rule: 'cpu_high', message: 'New', timestamp: '' }] })
+    await latest
+    statuses[0]!({ nodes: [{ node_id: 'older', cpu_pct: 99 }] })
+    alerts[0]!({ alerts: [] })
+    await flushPromises()
+    expect(fleet.nodes.value.map(n => n.node_id)).toEqual(['newer'])
+    expect(fleet.fleetAlerts.value[0]?.message).toBe('New')
+    expect(JSON.parse(sessionStorage.getItem('archipelago.fleet.cache.v2')!).nodes[0].node_id).toBe('newer')
+  } finally { wrapper.unmount() }
+})
+
+it('does not mark a malformed status response as a successful refresh', async () => {
+  vi.mocked(rpcClient.call).mockResolvedValue({ nodes: [null], alerts: [] })
+  let fleet!: ReturnType<typeof useFleetData>
+  const wrapper = mount(defineComponent({ setup() { fleet = useFleetData(); return () => null } }))
+  try {
+    await flushPromises()
+    expect(fleet.errorMessage.value).toContain('unavailable')
+    expect(fleet.lastRefreshed.value).toBe('')
+    expect(fleet.nodes.value).toEqual([])
+  } finally { wrapper.unmount() }
+})
+
+it('does not repopulate the session cache when requests finish after unmount', async () => {
+  const completions: Array<(value: unknown) => void> = []
+  vi.mocked(rpcClient.call).mockImplementation(() => new Promise(done => { completions.push(done) }) as never)
+  const wrapper = mount(defineComponent({ setup() { useFleetData(); return () => null } }))
+  wrapper.unmount()
+  sessionStorage.clear()
+  completions.forEach(done => done({ nodes: [{ node_id: 'old-session' }], alerts: [] }))
+  await flushPromises()
+  expect(sessionStorage.getItem('archipelago.fleet.cache.v2')).toBeNull()
+})

neode-ui/src/views/fleet/__tests__/FleetRecovery.test.ts

@@ -0,0 +1,42 @@
+import { afterEach, expect, it, vi } from 'vitest'
+import { flushPromises, mount } from '@vue/test-utils'
+import Fleet from '../../Fleet.vue'
+import { rpcClient } from '@/api/rpc-client'
+
+vi.mock('@/api/rpc-client', () => ({ rpcClient: { call: vi.fn() } }))
+vi.mock('vue-router', () => ({ useRouter: () => ({ push: vi.fn() }) }))
+afterEach(() => { sessionStorage.clear(); vi.clearAllMocks() })
+
+const stubs = {
+  BackButton: true, FleetOverviewCards: true, FleetAlerts: true,
+  FleetNodeDetail: true, FleetContainerMatrix: true,
+}
+
+it('keeps cached node cards visible beside the outage warning and recovers with Retry', async () => {
+  sessionStorage.setItem('archipelago.fleet.cache.v2', JSON.stringify({
+    nodes: [{ node_id: 'retained', node_name: 'Retained node', cpu_pct: 0 }],
+  }))
+  vi.mocked(rpcClient.call).mockRejectedValue(new Error('offline'))
+  const wrapper = mount(Fleet, { global: { stubs } })
+  try {
+    await flushPromises()
+    expect(wrapper.get('[role="status"]').text()).toContain('Showing the last received reports')
+    expect(wrapper.get('.fleet-node-card').text()).toContain('Retained node')
+    vi.mocked(rpcClient.call).mockResolvedValue({ nodes: [{ node_id: 'retained', node_name: 'Recovered node', cpu_pct: 0 }], alerts: [] })
+    await wrapper.get('[role="status"] button').trigger('click')
+    await flushPromises()
+    expect(wrapper.find('[role="status"]').exists()).toBe(false)
+    expect(wrapper.get('.fleet-node-card').text()).toContain('Recovered node')
+  } finally { wrapper.unmount() }
+})
+
+it('shows an actionable unavailable state when there are no saved reports', async () => {
+  vi.mocked(rpcClient.call).mockRejectedValue(new Error('offline'))
+  const wrapper = mount(Fleet, { global: { stubs } })
+  try {
+    await flushPromises()
+    expect(wrapper.text()).toContain('Fleet data is unavailable')
+    expect(wrapper.find('.fleet-node-card').exists()).toBe(false)
+    expect(wrapper.findAll('button').some(button => button.text() === 'Retry')).toBe(true)
+  } finally { wrapper.unmount() }
+})

neode-ui/src/views/fleet/useFleetData.ts

@@ -274,6 +274,10 @@ export function useFleetData() {
   const chartWidth = ref(300)
   let pollTimer: ReturnType<typeof setInterval> | null = null
   let historyRequest = 0
+  let statusRequest = 0
+  let alertsRequest = 0
+  let refreshRequest = 0
+  let disposed = false
 
   // --- Computed ---
 
@@ -339,35 +343,43 @@ export function useFleetData() {
   // --- Data Fetching ---
 
   async function fetchFleetStatus() {
+    const request = ++statusRequest
     try {
       const data = await rpcClient.call<{ nodes: Partial<FleetNode>[] }>({
         method: 'telemetry.fleet-status',
       })
-      if (data?.nodes) {
-        nodes.value = data.nodes.map(normalizeFleetNode)
-        lastRefreshed.value = new Date().toISOString()
-        writeFleetCache({
-          nodes: nodes.value,
-          fleetAlerts: fleetAlerts.value,
-          lastRefreshed: lastRefreshed.value,
-          selectedNodeId: selectedNodeId.value,
-          sortBy: sortBy.value,
-        })
+      if (disposed || request !== statusRequest) return
+      if (!Array.isArray(data?.nodes) || data.nodes.some(node => !node || typeof node !== 'object')) {
+        throw new Error('Invalid fleet response')
       }
-    } catch (err) {
-      if (loading.value) {
-        errorMessage.value = err instanceof Error ? err.message : 'Failed to load fleet data'
+      nodes.value = data.nodes.map(normalizeFleetNode)
+      lastRefreshed.value = new Date().toISOString()
+      errorMessage.value = ''
+      writeFleetCache({
+        nodes: nodes.value,
+        fleetAlerts: fleetAlerts.value,
+        lastRefreshed: lastRefreshed.value,
+        selectedNodeId: selectedNodeId.value,
+        sortBy: sortBy.value,
+      })
+    } catch {
+      if (!disposed && request === statusRequest) {
+        errorMessage.value = nodes.value.length
+          ? 'Fleet refresh failed. Showing the last received reports; current node status could not be checked.'
+          : 'Fleet data is unavailable. Check your connection and retry.'
       }
     }
   }
 
   async function fetchFleetAlerts() {
+    const request = ++alertsRequest
     alertsLoading.value = true
     try {
       const data = await rpcClient.call<{ alerts: FleetAlert[] }>({
         method: 'telemetry.fleet-alerts',
       })
-      if (data?.alerts) {
+      if (disposed || request !== alertsRequest) return
+      if (Array.isArray(data?.alerts)) {
         fleetAlerts.value = data.alerts
         writeFleetCache({
           nodes: nodes.value,
@@ -380,7 +392,7 @@ export function useFleetData() {
     } catch {
       // Non-critical, retry on next poll
     } finally {
-      alertsLoading.value = false
+      if (!disposed && request === alertsRequest) alertsLoading.value = false
     }
   }
 
@@ -403,14 +415,16 @@ export function useFleetData() {
   }
 
   async function refreshAll() {
+    const request = ++refreshRequest
     loading.value = !nodes.value.length
     refreshing.value = true
-    errorMessage.value = ''
     try {
       await Promise.all([fetchFleetStatus(), fetchFleetAlerts()])
     } finally {
-      loading.value = false
-      refreshing.value = false
+      if (!disposed && request === refreshRequest) {
+        loading.value = false
+        refreshing.value = false
+      }
     }
   }
 
@@ -554,6 +568,8 @@ export function useFleetData() {
   })
 
   onUnmounted(() => {
+    disposed = true
+    ++historyRequest
     disarmFleetPoll()
     window.removeEventListener('resize', updateChartWidth)
   })