Age Fleet status locally during stalled or paused telemetry refresh

This commit is contained in:
archipelago
2026-10-06 06:50:29 -04:00
parent ea3367ed45
commit c2d2b00c5c
5 changed files with 100 additions and 27 deletions
+2 -1
View File
@@ -99,6 +99,7 @@
</div> </div>
<FleetNodeGrid <FleetNodeGrid
:now="fleet.now.value"
:nodes="fleet.nodes.value" :nodes="fleet.nodes.value"
:sorted-nodes="fleet.sortedNodes.value" :sorted-nodes="fleet.sortedNodes.value"
:sort-by="fleet.sortBy.value" :sort-by="fleet.sortBy.value"
@@ -133,7 +134,7 @@
<p class="text-xs text-white/30 mt-4 text-center"> <p class="text-xs text-white/30 mt-4 text-center">
{{ fleet.autoRefresh.value ? 'Auto-refreshing every 60s' : 'Auto-refresh paused' }} {{ fleet.autoRefresh.value ? 'Auto-refreshing every 60s' : 'Auto-refresh paused' }}
&middot; Last updated {{ fleet.lastRefreshed.value ? timeAgo(fleet.lastRefreshed.value) : 'never' }} &middot; Last updated {{ fleet.lastRefreshed.value ? timeAgo(fleet.lastRefreshed.value, fleet.now.value) : 'never' }}
</p> </p>
</template> </template>
</div> </div>
+3 -2
View File
@@ -32,7 +32,7 @@
<div class="flex items-center gap-2"> <div class="flex items-center gap-2">
<span <span
class="fleet-status-dot" class="fleet-status-dot"
:class="fleetStatus(node.reported_at) === 'unknown' ? 'bg-white/30' : isOnline(node.reported_at) ? 'fleet-dot-online' : 'fleet-dot-offline'" :class="fleetStatus(node.reported_at, now) === 'unknown' ? 'bg-white/30' : isOnline(node.reported_at, now) ? 'fleet-dot-online' : 'fleet-dot-offline'"
></span> ></span>
<span class="text-sm font-semibold text-white truncate">{{ fleetNodeDisplayName(node) }}</span> <span class="text-sm font-semibold text-white truncate">{{ fleetNodeDisplayName(node) }}</span>
</div> </div>
@@ -84,7 +84,7 @@
</div> </div>
<div class="flex flex-wrap gap-x-3 gap-y-1 items-center justify-between text-xs text-white/40 mt-1"> <div class="flex flex-wrap gap-x-3 gap-y-1 items-center justify-between text-xs text-white/40 mt-1">
<span>{{ node.uptime_secs === null ? 'Uptime unavailable' : 'Up ' + formatUptime(node.uptime_secs) }}</span> <span>{{ node.uptime_secs === null ? 'Uptime unavailable' : 'Up ' + formatUptime(node.uptime_secs) }}</span>
<span>{{ fleetStatus(node.reported_at) === 'unknown' ? 'Status unknown' : fleetStatus(node.reported_at) === 'stale' ? 'Not reporting · last seen ' + timeAgo(node.reported_at) : 'Seen ' + timeAgo(node.reported_at) }}</span> <span>{{ fleetStatus(node.reported_at, now) === 'unknown' ? 'Status unknown' : fleetStatus(node.reported_at, now) === 'stale' ? 'Not reporting · last seen ' + timeAgo(node.reported_at, now) : 'Seen ' + timeAgo(node.reported_at, now) }}</span>
</div> </div>
</div> </div>
</div> </div>
@@ -98,6 +98,7 @@ import {
} from './useFleetData' } from './useFleetData'
defineProps<{ defineProps<{
now?: number
nodes: FleetNode[] nodes: FleetNode[]
sortedNodes: FleetNode[] sortedNodes: FleetNode[]
sortBy: SortOption sortBy: SortOption
@@ -0,0 +1,48 @@
import { afterEach, expect, it, vi } from 'vitest'
import { defineComponent } from 'vue'
import { flushPromises, mount } from '@vue/test-utils'
import { rpcClient } from '@/api/rpc-client'
import { useFleetData } from '../useFleetData'
vi.mock('@/api/rpc-client', () => ({ rpcClient: { call: vi.fn() } }))
afterEach(() => { vi.useRealTimers(); sessionStorage.clear(); vi.clearAllMocks() })
it('ages cached reports while refresh is paused and cleans up its clock', async () => {
vi.useFakeTimers()
vi.setSystemTime(new Date('2026-10-06T12:00:00Z'))
sessionStorage.setItem('archipelago.fleet.cache.v2', JSON.stringify({ nodes: [{
node_id: 'node', reported_at: '2026-10-06T11:30:10Z', cpu_pct: 20, mem_pct: 30, disk_pct: 40,
}] }))
vi.mocked(rpcClient.call).mockRejectedValue(new Error('Offline'))
let fleet!: ReturnType<typeof useFleetData>
const wrapper = mount(defineComponent({ setup() { fleet = useFleetData(); return () => null } }))
await flushPromises()
expect(fleet.onlineCount.value).toBe(1)
expect(fleet.avgCpu.value).toBe(20)
fleet.toggleAutoRefresh()
const calls = vi.mocked(rpcClient.call).mock.calls.length
await vi.advanceTimersByTimeAsync(15_000)
expect(fleet.onlineCount.value).toBe(0)
expect(fleet.offlineCount.value).toBe(1)
expect(fleet.avgCpu.value).toBeNull()
expect(rpcClient.call).toHaveBeenCalledTimes(calls)
wrapper.unmount()
expect(vi.getTimerCount()).toBe(0)
})
it('does not rearm timers when requests finish after unmount', async () => {
vi.useFakeTimers()
const intervals = vi.spyOn(globalThis, 'setInterval')
const clear = vi.spyOn(globalThis, 'clearInterval')
const completions: Array<(value: unknown) => void> = []
vi.mocked(rpcClient.call).mockImplementation(() => new Promise(done => { completions.push(done) }) as never)
const wrapper = mount(defineComponent({ setup() { useFleetData(); return () => null } }))
const started = intervals.mock.results.map(result => result.value)
wrapper.unmount()
for (const timer of started) expect(clear).toHaveBeenCalledWith(timer)
const intervalCalls = intervals.mock.calls.length
completions.forEach(done => done({ nodes: [], alerts: [] }))
await flushPromises()
expect(intervals).toHaveBeenCalledTimes(intervalCalls)
intervals.mockRestore(); clear.mockRestore()
})
@@ -25,6 +25,20 @@ describe('fleet unavailable readings', () => {
expect(wrapper.text()).not.toContain('NaN') expect(wrapper.text()).not.toContain('NaN')
wrapper.unmount() wrapper.unmount()
}) })
it('updates last-seen text and status when only the local clock changes', async () => {
const start = Date.parse('2026-10-06T12:00:00Z')
const nodes = [normalizeFleetNode({ node_id: 'node', reported_at: '2026-10-06T11:30:10Z' })]
const wrapper = mount(FleetNodeGrid, {
props: { nodes, sortedNodes: nodes, sortBy: 'status', selectedNodeId: null, now: start },
global: { mocks: { $ver: (v: string) => v } },
})
expect(wrapper.text()).toContain('Seen 29m ago')
expect(wrapper.find('.fleet-dot-online').exists()).toBe(true)
await wrapper.setProps({ now: start + 15_000 })
expect(wrapper.text()).toContain('Not reporting · last seen 30m ago')
expect(wrapper.find('.fleet-dot-offline').exists()).toBe(true)
wrapper.unmount()
})
it('renders empty averages without claiming zero load', () => { it('renders empty averages without claiming zero load', () => {
const wrapper = mount(FleetOverviewCards, {props: { const wrapper = mount(FleetOverviewCards, {props: {
nodeCount: 1, onlineCount: 0, offlineCount: 0, unknownCount: 1, nodeCount: 1, onlineCount: 0, offlineCount: 0, unknownCount: 1,
+33 -24
View File
@@ -54,8 +54,7 @@ export function formatUptime(secs: number | null): string {
return `${mins}m` return `${mins}m`
} }
export function timeAgo(dateStr: string): string { export function timeAgo(dateStr: string, now = Date.now()): string {
const now = Date.now()
const then = new Date(dateStr).getTime() const then = new Date(dateStr).getTime()
if (!Number.isFinite(then) || then > now + 60_000) return 'Unknown' if (!Number.isFinite(then) || then > now + 60_000) return 'Unknown'
const diffMs = now - then const diffMs = now - then
@@ -70,22 +69,22 @@ export function timeAgo(dateStr: string): string {
return `${diffDays}d ago` return `${diffDays}d ago`
} }
export function fleetStatus(reportedAt: string): 'online' | 'stale' | 'unknown' { export function fleetStatus(reportedAt: string, now = Date.now()): 'online' | 'stale' | 'unknown' {
const age = Date.now() - new Date(reportedAt).getTime() const age = now - new Date(reportedAt).getTime()
if (!Number.isFinite(age) || age < -60_000) return 'unknown' if (!Number.isFinite(age) || age < -60_000) return 'unknown'
return age < 30 * 60 * 1000 ? 'online' : 'stale' return age < 30 * 60 * 1000 ? 'online' : 'stale'
} }
export function isOnline(reportedAt: string): boolean { export function isOnline(reportedAt: string, now = Date.now()): boolean {
return fleetStatus(reportedAt) === 'online' return fleetStatus(reportedAt, now) === 'online'
} }
export function formatMetric(value: number | null, digits = 0): string { export function formatMetric(value: number | null, digits = 0): string {
return value === null ? '—' : `${value.toFixed(digits)}%` return value === null ? '—' : `${value.toFixed(digits)}%`
} }
export function averageMetric(nodes: FleetNode[], field: 'cpu_pct' | 'mem_pct' | 'disk_pct'): number | null { export function averageMetric(nodes: FleetNode[], field: 'cpu_pct' | 'mem_pct' | 'disk_pct', now = Date.now()): number | null {
const values = nodes.filter(n => isOnline(n.reported_at)).map(n => n[field]).filter((v): v is number => v !== null && Number.isFinite(v)) const values = nodes.filter(n => isOnline(n.reported_at, now)).map(n => n[field]).filter((v): v is number => v !== null && Number.isFinite(v))
return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null
} }
@@ -150,21 +149,21 @@ export const SORT_OPTIONS: Array<{ label: string; value: SortOption }> = [
{ label: 'Name', value: 'name' }, { label: 'Name', value: 'name' },
] ]
function reportTime(node: FleetNode): number { function reportTime(node: FleetNode, now: number): number {
return fleetStatus(node.reported_at) === 'unknown' ? 0 : new Date(node.reported_at).getTime() return fleetStatus(node.reported_at, now) === 'unknown' ? 0 : new Date(node.reported_at).getTime()
} }
export function sortFleetNodes(nodes: FleetNode[], sortBy: SortOption): FleetNode[] { export function sortFleetNodes(nodes: FleetNode[], sortBy: SortOption, now = Date.now()): FleetNode[] {
const sorted = [...nodes] const sorted = [...nodes]
switch (sortBy) { switch (sortBy) {
case 'status': case 'status':
sorted.sort((a, b) => { sorted.sort((a, b) => {
const rank = { online: 0, unknown: 1, stale: 2 } const rank = { online: 0, unknown: 1, stale: 2 }
return rank[fleetStatus(a.reported_at)] - rank[fleetStatus(b.reported_at)] return rank[fleetStatus(a.reported_at, now)] - rank[fleetStatus(b.reported_at, now)]
}) })
break break
case 'last-seen': case 'last-seen':
sorted.sort((a, b) => reportTime(b) - reportTime(a)) sorted.sort((a, b) => reportTime(b, now) - reportTime(a, now))
break break
case 'name': case 'name':
sorted.sort((a, b) => fleetNodeDisplayName(a).localeCompare(fleetNodeDisplayName(b))) sorted.sort((a, b) => fleetNodeDisplayName(a).localeCompare(fleetNodeDisplayName(b)))
@@ -256,6 +255,9 @@ function writeFleetCache(state: FleetCache) {
// --- Composable --- // --- Composable ---
export function useFleetData() { export function useFleetData() {
const now = ref(Date.now())
let clockTimer: ReturnType<typeof setInterval> | null = null
let active = true
const cached = readFleetCache() const cached = readFleetCache()
const loading = ref(!(cached.nodes?.length ?? 0)) const loading = ref(!(cached.nodes?.length ?? 0))
const errorMessage = ref('') const errorMessage = ref('')
@@ -274,28 +276,28 @@ export function useFleetData() {
// --- Computed --- // --- Computed ---
const onlineCount = computed(() => nodes.value.filter(n => isOnline(n.reported_at)).length) const onlineCount = computed(() => nodes.value.filter(n => isOnline(n.reported_at, now.value)).length)
const offlineCount = computed(() => nodes.value.filter(n => fleetStatus(n.reported_at) === 'stale').length) const offlineCount = computed(() => nodes.value.filter(n => fleetStatus(n.reported_at, now.value) === 'stale').length)
const unknownCount = computed(() => nodes.value.filter(n => fleetStatus(n.reported_at) === 'unknown').length) const unknownCount = computed(() => nodes.value.filter(n => fleetStatus(n.reported_at, now.value) === 'unknown').length)
const healthyCount = computed(() => nodes.value.filter(n => isOnline(n.reported_at) && n.cpu_pct !== null && n.mem_pct !== null && n.disk_pct !== null && n.recent_alerts.length === 0).length) const healthyCount = computed(() => nodes.value.filter(n => isOnline(n.reported_at, now.value) && n.cpu_pct !== null && n.mem_pct !== null && n.disk_pct !== null && n.recent_alerts.length === 0).length)
const fleetHealthPct = computed(() => { const fleetHealthPct = computed(() => {
if (!nodes.value.length) return 0 if (!nodes.value.length) return 0
return Math.round((healthyCount.value / nodes.value.length) * 100) return Math.round((healthyCount.value / nodes.value.length) * 100)
}) })
const avgCpu = computed(() => averageMetric(nodes.value, 'cpu_pct')) const avgCpu = computed(() => averageMetric(nodes.value, 'cpu_pct', now.value))
const avgMem = computed(() => averageMetric(nodes.value, 'mem_pct')) const avgMem = computed(() => averageMetric(nodes.value, 'mem_pct', now.value))
const avgDisk = computed(() => averageMetric(nodes.value, 'disk_pct')) const avgDisk = computed(() => averageMetric(nodes.value, 'disk_pct', now.value))
const selectedNode = computed(() => { const selectedNode = computed(() => {
if (!selectedNodeId.value) return null if (!selectedNodeId.value) return null
return nodes.value.find(n => n.node_id === selectedNodeId.value) ?? null return nodes.value.find(n => n.node_id === selectedNodeId.value) ?? null
}) })
const sortedNodes = computed(() => sortFleetNodes(nodes.value, sortBy.value)) const sortedNodes = computed(() => sortFleetNodes(nodes.value, sortBy.value, now.value))
const allAppIds = computed(() => { const allAppIds = computed(() => {
const appSet = new Set<string>() const appSet = new Set<string>()
@@ -426,7 +428,7 @@ export function useFleetData() {
function toggleAutoRefresh() { function toggleAutoRefresh() {
autoRefresh.value = !autoRefresh.value autoRefresh.value = !autoRefresh.value
if (autoRefresh.value) { if (autoRefresh.value && active) {
startAutoRefresh() startAutoRefresh()
} else { } else {
stopAutoRefresh() stopAutoRefresh()
@@ -516,17 +518,24 @@ export function useFleetData() {
// first, so calling it from both onMounted and onActivated on a fresh // first, so calling it from both onMounted and onActivated on a fresh
// KeepAlive-wrapped mount is safe and idempotent. // KeepAlive-wrapped mount is safe and idempotent.
function armFleetPoll() { function armFleetPoll() {
active = true
now.value = Date.now()
if (clockTimer) clearInterval(clockTimer)
clockTimer = setInterval(() => { now.value = Date.now() }, 15_000)
if (autoRefresh.value) startAutoRefresh() if (autoRefresh.value) startAutoRefresh()
} }
function disarmFleetPoll() { function disarmFleetPoll() {
active = false
if (clockTimer) clearInterval(clockTimer)
clockTimer = null
stopAutoRefresh() stopAutoRefresh()
} }
onMounted(async () => { onMounted(async () => {
updateChartWidth() updateChartWidth()
window.addEventListener('resize', updateChartWidth) window.addEventListener('resize', updateChartWidth)
await refreshAll()
armFleetPoll() armFleetPoll()
await refreshAll()
}) })
onActivated(() => { onActivated(() => {
@@ -545,7 +554,7 @@ export function useFleetData() {
return { return {
loading, refreshing, errorMessage, nodes, fleetAlerts, alertsLoading, loading, refreshing, errorMessage, nodes, fleetAlerts, alertsLoading,
selectedNodeId, selectedNode, nodeHistory, nodeHistoryLoading, selectedNodeId, selectedNode, nodeHistory, nodeHistoryLoading,
autoRefresh, lastRefreshed, sortBy, chartWidth, autoRefresh, lastRefreshed, sortBy, chartWidth, now,
onlineCount, offlineCount, unknownCount, healthyCount, fleetHealthPct, onlineCount, offlineCount, unknownCount, healthyCount, fleetHealthPct,
avgCpu, avgMem, avgDisk, sortedNodes, allAppIds, avgCpu, avgMem, avgDisk, sortedNodes, allAppIds,
nodeHistoryLabels, nodeHistoryCpuDatasets, nodeHistoryMemDatasets, nodeHistoryDiskDatasets, nodeHistoryLabels, nodeHistoryCpuDatasets, nodeHistoryMemDatasets, nodeHistoryDiskDatasets,