Age Fleet status locally during stalled or paused telemetry refresh

2026-10-06 06:50:29 -04:00
parent ea3367ed45
commit c2d2b00c5c
5 changed files with 100 additions and 27 deletions
@@ -99,6 +99,7 @@
      </div>

      <FleetNodeGrid
+        :now="fleet.now.value"
        :nodes="fleet.nodes.value"
        :sorted-nodes="fleet.sortedNodes.value"
        :sort-by="fleet.sortBy.value"
@@ -133,7 +134,7 @@

      <p class="text-xs text-white/30 mt-4 text-center">
        {{ fleet.autoRefresh.value ? 'Auto-refreshing every 60s' : 'Auto-refresh paused' }}
-        &middot; Last updated {{ fleet.lastRefreshed.value ? timeAgo(fleet.lastRefreshed.value) : 'never' }}
+        &middot; Last updated {{ fleet.lastRefreshed.value ? timeAgo(fleet.lastRefreshed.value, fleet.now.value) : 'never' }}
      </p>
    </template>
  </div>
@@ -32,7 +32,7 @@
          <div class="flex items-center gap-2">
            <span
              class="fleet-status-dot"
-              :class="fleetStatus(node.reported_at) === 'unknown' ? 'bg-white/30' : isOnline(node.reported_at) ? 'fleet-dot-online' : 'fleet-dot-offline'"
+              :class="fleetStatus(node.reported_at, now) === 'unknown' ? 'bg-white/30' : isOnline(node.reported_at, now) ? 'fleet-dot-online' : 'fleet-dot-offline'"
            ></span>
            <span class="text-sm font-semibold text-white truncate">{{ fleetNodeDisplayName(node) }}</span>
          </div>
@@ -84,7 +84,7 @@
        </div>
        <div class="flex flex-wrap gap-x-3 gap-y-1 items-center justify-between text-xs text-white/40 mt-1">
          <span>{{ node.uptime_secs === null ? 'Uptime unavailable' : 'Up ' + formatUptime(node.uptime_secs) }}</span>
-          <span>{{ fleetStatus(node.reported_at) === 'unknown' ? 'Status unknown' : fleetStatus(node.reported_at) === 'stale' ? 'Not reporting · last seen ' + timeAgo(node.reported_at) : 'Seen ' + timeAgo(node.reported_at) }}</span>
+          <span>{{ fleetStatus(node.reported_at, now) === 'unknown' ? 'Status unknown' : fleetStatus(node.reported_at, now) === 'stale' ? 'Not reporting · last seen ' + timeAgo(node.reported_at, now) : 'Seen ' + timeAgo(node.reported_at, now) }}</span>
        </div>
      </div>
    </div>
@@ -98,6 +98,7 @@ import {
 } from './useFleetData'

 defineProps<{
+  now?: number
  nodes: FleetNode[]
  sortedNodes: FleetNode[]
  sortBy: SortOption
@@ -0,0 +1,48 @@
+import { afterEach, expect, it, vi } from 'vitest'
+import { defineComponent } from 'vue'
+import { flushPromises, mount } from '@vue/test-utils'
+import { rpcClient } from '@/api/rpc-client'
+import { useFleetData } from '../useFleetData'
+
+vi.mock('@/api/rpc-client', () => ({ rpcClient: { call: vi.fn() } }))
+afterEach(() => { vi.useRealTimers(); sessionStorage.clear(); vi.clearAllMocks() })
+
+it('ages cached reports while refresh is paused and cleans up its clock', async () => {
+  vi.useFakeTimers()
+  vi.setSystemTime(new Date('2026-10-06T12:00:00Z'))
+  sessionStorage.setItem('archipelago.fleet.cache.v2', JSON.stringify({ nodes: [{
+    node_id: 'node', reported_at: '2026-10-06T11:30:10Z', cpu_pct: 20, mem_pct: 30, disk_pct: 40,
+  }] }))
+  vi.mocked(rpcClient.call).mockRejectedValue(new Error('Offline'))
+  let fleet!: ReturnType<typeof useFleetData>
+  const wrapper = mount(defineComponent({ setup() { fleet = useFleetData(); return () => null } }))
+  await flushPromises()
+  expect(fleet.onlineCount.value).toBe(1)
+  expect(fleet.avgCpu.value).toBe(20)
+  fleet.toggleAutoRefresh()
+  const calls = vi.mocked(rpcClient.call).mock.calls.length
+  await vi.advanceTimersByTimeAsync(15_000)
+  expect(fleet.onlineCount.value).toBe(0)
+  expect(fleet.offlineCount.value).toBe(1)
+  expect(fleet.avgCpu.value).toBeNull()
+  expect(rpcClient.call).toHaveBeenCalledTimes(calls)
+  wrapper.unmount()
+  expect(vi.getTimerCount()).toBe(0)
+})
+
+it('does not rearm timers when requests finish after unmount', async () => {
+  vi.useFakeTimers()
+  const intervals = vi.spyOn(globalThis, 'setInterval')
+  const clear = vi.spyOn(globalThis, 'clearInterval')
+  const completions: Array<(value: unknown) => void> = []
+  vi.mocked(rpcClient.call).mockImplementation(() => new Promise(done => { completions.push(done) }) as never)
+  const wrapper = mount(defineComponent({ setup() { useFleetData(); return () => null } }))
+  const started = intervals.mock.results.map(result => result.value)
+  wrapper.unmount()
+  for (const timer of started) expect(clear).toHaveBeenCalledWith(timer)
+  const intervalCalls = intervals.mock.calls.length
+  completions.forEach(done => done({ nodes: [], alerts: [] }))
+  await flushPromises()
+  expect(intervals).toHaveBeenCalledTimes(intervalCalls)
+  intervals.mockRestore(); clear.mockRestore()
+})
@@ -25,6 +25,20 @@ describe('fleet unavailable readings', () => {
    expect(wrapper.text()).not.toContain('NaN')
    wrapper.unmount()
  })
+  it('updates last-seen text and status when only the local clock changes', async () => {
+    const start = Date.parse('2026-10-06T12:00:00Z')
+    const nodes = [normalizeFleetNode({ node_id: 'node', reported_at: '2026-10-06T11:30:10Z' })]
+    const wrapper = mount(FleetNodeGrid, {
+      props: { nodes, sortedNodes: nodes, sortBy: 'status', selectedNodeId: null, now: start },
+      global: { mocks: { $ver: (v: string) => v } },
+    })
+    expect(wrapper.text()).toContain('Seen 29m ago')
+    expect(wrapper.find('.fleet-dot-online').exists()).toBe(true)
+    await wrapper.setProps({ now: start + 15_000 })
+    expect(wrapper.text()).toContain('Not reporting · last seen 30m ago')
+    expect(wrapper.find('.fleet-dot-offline').exists()).toBe(true)
+    wrapper.unmount()
+  })
  it('renders empty averages without claiming zero load', () => {
    const wrapper = mount(FleetOverviewCards, {props: {
      nodeCount: 1, onlineCount: 0, offlineCount: 0, unknownCount: 1,
@@ -54,8 +54,7 @@ export function formatUptime(secs: number | null): string {
  return `${mins}m`
 }

-export function timeAgo(dateStr: string): string {
-  const now = Date.now()
+export function timeAgo(dateStr: string, now = Date.now()): string {
  const then = new Date(dateStr).getTime()
  if (!Number.isFinite(then) || then > now + 60_000) return 'Unknown'
  const diffMs = now - then
@@ -70,22 +69,22 @@ export function timeAgo(dateStr: string): string {
  return `${diffDays}d ago`
 }

-export function fleetStatus(reportedAt: string): 'online' | 'stale' | 'unknown' {
-  const age = Date.now() - new Date(reportedAt).getTime()
+export function fleetStatus(reportedAt: string, now = Date.now()): 'online' | 'stale' | 'unknown' {
+  const age = now - new Date(reportedAt).getTime()
  if (!Number.isFinite(age) || age < -60_000) return 'unknown'
  return age < 30 * 60 * 1000 ? 'online' : 'stale'
 }

-export function isOnline(reportedAt: string): boolean {
-  return fleetStatus(reportedAt) === 'online'
+export function isOnline(reportedAt: string, now = Date.now()): boolean {
+  return fleetStatus(reportedAt, now) === 'online'
 }

 export function formatMetric(value: number | null, digits = 0): string {
  return value === null ? '—' : `${value.toFixed(digits)}%`
 }

-export function averageMetric(nodes: FleetNode[], field: 'cpu_pct' | 'mem_pct' | 'disk_pct'): number | null {
-  const values = nodes.filter(n => isOnline(n.reported_at)).map(n => n[field]).filter((v): v is number => v !== null && Number.isFinite(v))
+export function averageMetric(nodes: FleetNode[], field: 'cpu_pct' | 'mem_pct' | 'disk_pct', now = Date.now()): number | null {
+  const values = nodes.filter(n => isOnline(n.reported_at, now)).map(n => n[field]).filter((v): v is number => v !== null && Number.isFinite(v))
  return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null
 }

@@ -150,21 +149,21 @@ export const SORT_OPTIONS: Array<{ label: string; value: SortOption }> = [
  { label: 'Name', value: 'name' },
 ]

-function reportTime(node: FleetNode): number {
-  return fleetStatus(node.reported_at) === 'unknown' ? 0 : new Date(node.reported_at).getTime()
+function reportTime(node: FleetNode, now: number): number {
+  return fleetStatus(node.reported_at, now) === 'unknown' ? 0 : new Date(node.reported_at).getTime()
 }

-export function sortFleetNodes(nodes: FleetNode[], sortBy: SortOption): FleetNode[] {
+export function sortFleetNodes(nodes: FleetNode[], sortBy: SortOption, now = Date.now()): FleetNode[] {
  const sorted = [...nodes]
  switch (sortBy) {
    case 'status':
      sorted.sort((a, b) => {
        const rank = { online: 0, unknown: 1, stale: 2 }
-        return rank[fleetStatus(a.reported_at)] - rank[fleetStatus(b.reported_at)]
+        return rank[fleetStatus(a.reported_at, now)] - rank[fleetStatus(b.reported_at, now)]
      })
      break
    case 'last-seen':
-      sorted.sort((a, b) => reportTime(b) - reportTime(a))
+      sorted.sort((a, b) => reportTime(b, now) - reportTime(a, now))
      break
    case 'name':
      sorted.sort((a, b) => fleetNodeDisplayName(a).localeCompare(fleetNodeDisplayName(b)))
@@ -256,6 +255,9 @@ function writeFleetCache(state: FleetCache) {
 // --- Composable ---

 export function useFleetData() {
+  const now = ref(Date.now())
+  let clockTimer: ReturnType<typeof setInterval> | null = null
+  let active = true
  const cached = readFleetCache()
  const loading = ref(!(cached.nodes?.length ?? 0))
  const errorMessage = ref('')
@@ -274,28 +276,28 @@ export function useFleetData() {

  // --- Computed ---

-  const onlineCount = computed(() => nodes.value.filter(n => isOnline(n.reported_at)).length)
-  const offlineCount = computed(() => nodes.value.filter(n => fleetStatus(n.reported_at) === 'stale').length)
-  const unknownCount = computed(() => nodes.value.filter(n => fleetStatus(n.reported_at) === 'unknown').length)
-  const healthyCount = computed(() => nodes.value.filter(n => isOnline(n.reported_at) && n.cpu_pct !== null && n.mem_pct !== null && n.disk_pct !== null && n.recent_alerts.length === 0).length)
+  const onlineCount = computed(() => nodes.value.filter(n => isOnline(n.reported_at, now.value)).length)
+  const offlineCount = computed(() => nodes.value.filter(n => fleetStatus(n.reported_at, now.value) === 'stale').length)
+  const unknownCount = computed(() => nodes.value.filter(n => fleetStatus(n.reported_at, now.value) === 'unknown').length)
+  const healthyCount = computed(() => nodes.value.filter(n => isOnline(n.reported_at, now.value) && n.cpu_pct !== null && n.mem_pct !== null && n.disk_pct !== null && n.recent_alerts.length === 0).length)

  const fleetHealthPct = computed(() => {
    if (!nodes.value.length) return 0
    return Math.round((healthyCount.value / nodes.value.length) * 100)
  })

-  const avgCpu = computed(() => averageMetric(nodes.value, 'cpu_pct'))
+  const avgCpu = computed(() => averageMetric(nodes.value, 'cpu_pct', now.value))

-  const avgMem = computed(() => averageMetric(nodes.value, 'mem_pct'))
+  const avgMem = computed(() => averageMetric(nodes.value, 'mem_pct', now.value))

-  const avgDisk = computed(() => averageMetric(nodes.value, 'disk_pct'))
+  const avgDisk = computed(() => averageMetric(nodes.value, 'disk_pct', now.value))

  const selectedNode = computed(() => {
    if (!selectedNodeId.value) return null
    return nodes.value.find(n => n.node_id === selectedNodeId.value) ?? null
  })

-  const sortedNodes = computed(() => sortFleetNodes(nodes.value, sortBy.value))
+  const sortedNodes = computed(() => sortFleetNodes(nodes.value, sortBy.value, now.value))

  const allAppIds = computed(() => {
    const appSet = new Set<string>()
@@ -426,7 +428,7 @@ export function useFleetData() {

  function toggleAutoRefresh() {
    autoRefresh.value = !autoRefresh.value
-    if (autoRefresh.value) {
+    if (autoRefresh.value && active) {
      startAutoRefresh()
    } else {
      stopAutoRefresh()
@@ -516,17 +518,24 @@ export function useFleetData() {
  // first, so calling it from both onMounted and onActivated on a fresh
  // KeepAlive-wrapped mount is safe and idempotent.
  function armFleetPoll() {
+    active = true
+    now.value = Date.now()
+    if (clockTimer) clearInterval(clockTimer)
+    clockTimer = setInterval(() => { now.value = Date.now() }, 15_000)
    if (autoRefresh.value) startAutoRefresh()
  }
  function disarmFleetPoll() {
+    active = false
+    if (clockTimer) clearInterval(clockTimer)
+    clockTimer = null
    stopAutoRefresh()
  }

  onMounted(async () => {
    updateChartWidth()
    window.addEventListener('resize', updateChartWidth)
-    await refreshAll()
    armFleetPoll()
+    await refreshAll()
  })

  onActivated(() => {
@@ -545,7 +554,7 @@ export function useFleetData() {
  return {
    loading, refreshing, errorMessage, nodes, fleetAlerts, alertsLoading,
    selectedNodeId, selectedNode, nodeHistory, nodeHistoryLoading,
-    autoRefresh, lastRefreshed, sortBy, chartWidth,
+    autoRefresh, lastRefreshed, sortBy, chartWidth, now,
    onlineCount, offlineCount, unknownCount, healthyCount, fleetHealthPct,
    avgCpu, avgMem, avgDisk, sortedNodes, allAppIds,
    nodeHistoryLabels, nodeHistoryCpuDatasets, nodeHistoryMemDatasets, nodeHistoryDiskDatasets,