/* === CoreScope — perf.js === */ 'use strict'; var GH = 'https://github.com/Kpa-clawbot/corescope'; // detectPerfAnomalies — pure, testable. // Computes per-component write rates over a rolling time window and flags any // component whose current per-second rate exceeds `factor` × its rolling // baseline rate. Issue #1120 acceptance: 5-minute window, 10× threshold. // // Inputs: // history: ordered array of snapshots [{ sampleAt: ISO, sources: { name: cum } }] // current: the freshest snapshot, same shape // opts: // windowMs (default 5*60*1000) — rolling baseline window // factor (default 10) — rate-multiplier threshold // minHistorySec (default 30) — refuse to flag until baseline is stable // // Returns: { rates, baselineRates, flags } — all keyed by source name. function detectPerfAnomalies(history, current, opts) { opts = opts || {}; const windowMs = opts.windowMs || (5 * 60 * 1000); const factor = opts.factor || 10; const minHistorySec = opts.minHistorySec != null ? opts.minHistorySec : 30; const out = { rates: {}, baselineRates: {}, flags: {} }; if (!current || !current.sources || !history || history.length === 0) return out; const curT = Date.parse(current.sampleAt); if (!isFinite(curT)) return out; // Find the most recent prior sample (for the *current* per-second rate) // and the oldest sample within the window (for the baseline). const prior = history[history.length - 1]; const priorT = Date.parse(prior.sampleAt); const curDt = (curT - priorT) / 1000; if (!(curDt > 0)) return out; // Baseline: oldest sample within window vs. prior (the snapshot just before // `current`). Anything older than windowMs is excluded. const cutoff = curT - windowMs; let baseIdx = 0; for (let i = history.length - 1; i >= 0; i--) { if (Date.parse(history[i].sampleAt) < cutoff) { baseIdx = i + 1; break; } } if (baseIdx >= history.length) baseIdx = history.length - 1; const baseSnap = history[baseIdx]; const baseT = Date.parse(baseSnap.sampleAt); const baseDt = (priorT - baseT) / 1000; // Compute rates for every source seen in current. for (const k of Object.keys(current.sources)) { const cur = current.sources[k] || 0; const prev = (prior.sources && prior.sources[k]) || 0; const rate = (cur - prev) / curDt; out.rates[k] = rate; if (baseDt <= 0 || baseDt < minHistorySec) { out.baselineRates[k] = null; continue; } const baseStart = (baseSnap.sources && baseSnap.sources[k]) || 0; const baseEnd = prev; // baseline window = [baseSnap .. prior] const baseRate = (baseEnd - baseStart) / baseDt; out.baselineRates[k] = baseRate; // Guard floor to avoid 0-baseline → infinite ratio false positives. const floor = 0.05; // 1 event per 20s minimum baseline if (rate > factor * Math.max(baseRate, floor) && rate > factor * floor) { out.flags[k] = true; } } return out; } if (typeof window !== 'undefined') { window.detectPerfAnomalies = detectPerfAnomalies; } function renderVersionCard(health) { if (!health || (!health.version && !health.commit)) return ''; var ver = health.version && health.version !== 'unknown' ? health.version : null; var sha = health.commit && health.commit !== 'unknown' ? health.commit : null; if (!ver && !sha) return ''; var vTag = ver ? (ver.charAt(0) === 'v' ? ver : 'v' + ver) : null; var parts = []; if (vTag) parts.push('' + vTag + ''); if (sha) parts.push('' + sha.slice(0, 7) + ''); return '
' + parts.join(' · ') + '
Version
'; } (function () { let interval = null; async function render(app) { app.innerHTML = '

Performance Dashboard

Loading...
'; await refresh(); } async function refresh() { const el = document.getElementById('perfContent'); if (!el) return; try { // #1258: /api/health was awaited AFTER Promise.all, adding a full RTT // (~50-200ms) on every 5s refresh. Issue it in parallel with the rest. const [server, client, ioStats, sqliteStats, writeSources, health] = await Promise.all([ fetch('/api/perf').then(r => r.json()), Promise.resolve(window.apiPerf ? window.apiPerf() : null), fetch('/api/perf/io').then(r => r.json()).catch(() => null), fetch('/api/perf/sqlite').then(r => r.json()).catch(() => null), fetch('/api/perf/write-sources').then(r => r.json()).catch(() => null), fetch('/api/health').then(r => r.json()).catch(() => null) ]); let html = ''; // Server overview html += `
${server.totalRequests}
Total Requests
${server.avgMs}ms
Avg Response
${health ? health.uptimeHuman : Math.round(server.uptime / 60) + 'm'}
Uptime
${server.slowQueries.length}
Slow (>100ms)
${renderVersionCard(health)}
`; // System health (memory, event loop / go runtime, WS) if (health) { const isGo = health.engine === 'go'; if (isGo && server.goRuntime) { const gr = server.goRuntime; const gcColor = gr.lastPauseMs > 5 ? 'var(--status-red)' : gr.lastPauseMs > 1 ? 'var(--status-yellow)' : 'var(--status-green)'; html += `

Go Runtime

${gr.goroutines}
Goroutines
${gr.numGC}
GC Collections
${(+gr.pauseTotalMs).toFixed(1)}ms
GC Pause Total
${(+gr.lastPauseMs).toFixed(1)}ms
Last GC Pause
${(+gr.heapAllocMB).toFixed(1)}MB
Heap Alloc
${(+gr.heapSysMB).toFixed(1)}MB
Heap Sys
${(+gr.heapInuseMB).toFixed(1)}MB
Heap Inuse
${(+gr.heapIdleMB).toFixed(1)}MB
Heap Idle
${gr.numCPU}
CPUs
${health.websocket.clients}
WS Clients
`; } else { const m = health.memory, el = health.eventLoop; const elColor = el.p95Ms > 500 ? 'var(--status-red)' : el.p95Ms > 100 ? 'var(--status-yellow)' : 'var(--status-green)'; const memColor = m.heapUsed > m.heapTotal * 0.85 ? 'var(--status-red)' : m.heapUsed > m.heapTotal * 0.7 ? 'var(--status-yellow)' : 'var(--status-green)'; html += `

System Health

${m.heapUsed}MB
Heap Used / ${m.heapTotal}MB
${m.rss}MB
RSS
${el.p95Ms}ms
Event Loop p95
${el.maxLagMs}ms
EL Max Lag
${el.currentLagMs}ms
EL Current
${health.websocket.clients}
WS Clients
`; } } // Disk I/O (#1120) if (ioStats) { const fmtRate = (bps) => { if (bps >= 1048576) return (bps / 1048576).toFixed(1) + ' MB/s'; if (bps >= 1024) return (bps / 1024).toFixed(1) + ' KB/s'; return Math.round(bps) + ' B/s'; }; const writeWarn = ioStats.writeBytesPerSec > 10 * 1048576 ? ' ' : ''; const cancelled = ioStats.cancelledWriteBytesPerSec || 0; // Cancelled writes warn at >1 MB/s — sustained cancellation usually // means truncate/unlink racing with active writers (#1119-shaped bug). const cancelledWarn = cancelled > 1048576 ? ' ' : ''; html += `

Disk I/O (server process)

${fmtRate(ioStats.readBytesPerSec || 0)}
Read
${fmtRate(ioStats.writeBytesPerSec || 0)}${writeWarn}
Write
${fmtRate(cancelled)}${cancelledWarn}
Cancelled Write
${Math.round(ioStats.syscallsRead || 0)}/s
Syscalls Read
${Math.round(ioStats.syscallsWrite || 0)}/s
Syscalls Write
`; // Ingestor row — sourced from ingestor's own /proc/self/io snapshot // surfaced via the stats file (#1120: "Both ingestor and server"). if (ioStats.ingestor) { const ing = ioStats.ingestor; const ingWriteWarn = (ing.writeBytesPerSec || 0) > 10 * 1048576 ? ' ' : ''; const ingCancelled = ing.cancelledWriteBytesPerSec || 0; const ingCancelledWarn = ingCancelled > 1048576 ? ' ' : ''; html += `

Disk I/O (Ingestor process)

${fmtRate(ing.readBytesPerSec || 0)}
Read
${fmtRate(ing.writeBytesPerSec || 0)}${ingWriteWarn}
Write
${fmtRate(ingCancelled)}${ingCancelledWarn}
Cancelled Write
${Math.round(ing.syscallsRead || 0)}/s
Syscalls Read
${Math.round(ing.syscallsWrite || 0)}/s
Syscalls Write
`; } } // Write Sources (#1120) — per-component counters from ingestor if (writeSources && writeSources.sources) { const src = writeSources.sources; const keys = Object.keys(src).sort((a, b) => (src[b] || 0) - (src[a] || 0)); html += '

Write Sources

'; if (keys.length === 0) { html += '

No ingestor stats yet (waiting for /tmp/corescope-ingestor-stats.json)

'; } else { // Anomaly detection (#1120 acceptance): flag any component whose // per-second write rate exceeds 10× its 5-minute rolling baseline. // History is stashed on window so the detector has multi-sample // context across the 5s refresh tick. if (!window._perfWriteSourcesHistory) window._perfWriteSourcesHistory = []; const history = window._perfWriteSourcesHistory; const current = { sampleAt: writeSources.sampleAt || new Date().toISOString(), sources: { ...src } }; const anom = detectPerfAnomalies(history, current, { windowMs: 5 * 60 * 1000, factor: 10 }); // Append current and prune anything older than 6 minutes (keeps a // little headroom past the 5-min window, bounded memory). history.push(current); const cutoff = Date.parse(current.sampleAt) - (6 * 60 * 1000); while (history.length > 1 && Date.parse(history[0].sampleAt) < cutoff) history.shift(); html += '
'; for (const k of keys) { const v = src[k] || 0; const rate = anom.rates[k]; const base = anom.baselineRates[k]; const flag = anom.flags[k] ? ' ' : ''; const rateStr = (rate != null && isFinite(rate)) ? rate.toFixed(2) : '—'; const baseStr = (base != null && isFinite(base)) ? base.toFixed(2) : '—'; html += ``; } html += '
SourceTotalRate/sBaseline/sAnomaly
${k}${v.toLocaleString()}${rateStr}${baseStr}${flag}
'; if (writeSources.sampleAt) { html += `
Sampled: ${writeSources.sampleAt} · baseline window: 5 min · threshold: 10×
`; } } } // SQLite perf (separate from existing SQLite block — focused on WAL + cache hit) (#1120) if (sqliteStats) { const walMB = sqliteStats.walSizeMB || 0; const walFlag = walMB > 100 ? ' ' : ''; const hitRate = (sqliteStats.cacheHitRate || 0) * 100; const hitFlag = hitRate > 0 && hitRate < 90 ? ' ' : ''; html += `

SQLite (WAL + Cache Hit)

${walMB.toFixed(1)}MB${walFlag}
WAL Size
${(sqliteStats.pageCount || 0).toLocaleString()}
Page Count
${sqliteStats.pageSize || 0}
Page Size
${hitRate.toFixed(1)}%${hitFlag}
Cache Hit Rate
`; } // Cache stats if (server.cache) { const c = server.cache; const clientCache = _apiCache ? _apiCache.size : 0; html += `

Cache

${c.size}
Server Entries
${c.hits}
Server Hits
${c.misses}
Server Misses
${c.hitRate}%
Server Hit Rate
${c.staleHits || 0}
Stale Hits (SWR)
${c.recomputes || 0}
Recomputes
${clientCache}
Client Entries
`; if (client) { html += `
${client.cacheHits || 0}
Client Hits
${client.cacheMisses || 0}
Client Misses
${client.cacheHitRate || 0}%
Client Hit Rate
`; } } // Packet Store stats if (server.packetStore) { const ps = server.packetStore; html += `

In-Memory Packet Store

${ps.inMemory.toLocaleString()}
Packets in RAM
${ps.trackedMB}MB
Tracked Memory
${ps.maxMB}MB
Memory Limit
${ps.estimatedMB}MB
Heap (debug)
${ps.queries.toLocaleString()}
Queries Served
${ps.inserts.toLocaleString()}
Live Inserts
${ps.evicted.toLocaleString()}
Evicted
${ps.indexes.byHash.toLocaleString()}
Unique Hashes
${ps.indexes.byObserver}
Observers
${ps.indexes.byNode.toLocaleString()}
Indexed Nodes
`; } // SQLite stats if (server.sqlite && !server.sqlite.error) { const sq = server.sqlite; const walColor = sq.walSizeMB > 50 ? 'var(--status-red)' : sq.walSizeMB > 10 ? 'var(--status-yellow)' : 'var(--status-green)'; const freelistColor = sq.freelistMB > 10 ? 'var(--status-yellow)' : 'var(--status-green)'; html += `

SQLite

${sq.dbSizeMB}MB
DB Size
${sq.walSizeMB}MB
WAL Size
${sq.freelistMB}MB
Freelist
${(sq.rows.transmissions || 0).toLocaleString()}
Transmissions
${(sq.rows.observations || 0).toLocaleString()}
Observations
${sq.rows.nodes || 0}
Nodes
${sq.rows.observers || 0}
Observers
`; if (sq.walPages) { html += `
${sq.walPages.busy}
WAL Busy Pages
`; } html += `
`; } // Server endpoints table — sort by total time (count * avg) DESC. // #1258: header claimed "sorted by total time" but JSON map order is // undefined and the frontend was not sorting. Slow endpoints could // appear anywhere in the table, defeating the section's whole purpose. const eps = Object.entries(server.endpoints).sort((a, b) => { const ta = (a[1].count || 0) * (a[1].avgMs || 0); const tb = (b[1].count || 0) * (b[1].avgMs || 0); return tb - ta; }); if (eps.length) { html += '

Server Endpoints (sorted by total time)

'; html += '
'; for (const [path, s] of eps) { const total = Math.round(s.count * s.avgMs); const cls = s.p95Ms > 200 ? ' class="perf-slow"' : s.p95Ms > 50 ? ' class="perf-warn"' : ''; html += ``; } html += '
EndpointCountAvgP50P95MaxTotal
${path}${s.count}${s.avgMs}ms${s.p50Ms}ms${s.p95Ms}ms${s.maxMs}ms${total}ms
'; } // Client API calls if (client && client.endpoints.length) { html += '

Client API Calls (this session)

'; html += '
'; for (const s of client.endpoints) { const cls = s.maxMs > 500 ? ' class="perf-slow"' : s.avgMs > 200 ? ' class="perf-warn"' : ''; html += ``; } html += '
EndpointCountAvgMaxTotal
${s.path}${s.count}${s.avgMs}ms${s.maxMs}ms${s.totalMs}ms
'; } // Slow queries if (server.slowQueries.length) { html += '

Recent Slow Queries (>100ms)

'; html += '
'; for (const q of server.slowQueries.slice().reverse()) { html += ``; } html += '
TimePathDurationStatus
${new Date(q.time).toLocaleTimeString()}${q.path}${q.ms}ms${q.status}
'; } html += `
`; el.innerHTML = html; document.getElementById('perfReset')?.addEventListener('click', async () => { await fetch('/api/perf/reset', { method: 'POST' }); if (window._apiPerf) { window._apiPerf = { calls: 0, totalMs: 0, log: [] }; } refresh(); }); document.getElementById('perfRefresh')?.addEventListener('click', refresh); } catch (err) { el.innerHTML = `

Error: ${err.message}

`; } } registerPage('perf', { init(app) { render(app); // #1258: don't burn CPU/network rebuilding the page (and its many cards // + 3 large tables) every 5s while the tab is hidden. Pause polling on // visibilitychange and resume on focus. Reduces background fetch traffic // to zero and prevents a returning user from seeing a 100+ms thrash as // a backlog of refreshes flush. const tick = () => { if (document.hidden) return; refresh(); }; interval = setInterval(tick, 5000); const onVis = () => { if (!document.hidden) refresh(); }; document.addEventListener('visibilitychange', onVis); this._onVis = onVis; }, destroy() { if (interval) { clearInterval(interval); interval = null; } if (this._onVis) { document.removeEventListener('visibilitychange', this._onVis); this._onVis = null; } } }); })();