Compare commits

..

6 Commits

Author SHA1 Message Date
you
fb70b42425 perf: add TTL cache layer + rewrite bulk-health to single-query
- Add TTLCache class with hit/miss tracking
- Cache all expensive endpoints:
  - analytics/* endpoints: 60s TTL
  - channels: 30s TTL
  - channels/:hash/messages: 15s TTL
  - nodes/:pubkey: 30s TTL
  - nodes/:pubkey/health: 30s TTL
  - observers: 30s TTL
  - bulk-health: 60s TTL
- Invalidate all caches on new packet ingestion (POST + MQTT)
- Rewrite bulk-health from N×5 queries to 1 query + JS matching
- Add cache stats (size, hits, misses, hitRate) to /api/perf
2026-03-20 02:06:23 +00:00
you
e7651549ea feat: add frontend API response caching with TTL, in-flight dedup, and WebSocket invalidation
- Replace api() with caching version supporting TTL and request deduplication
- Add appropriate TTLs to all api() call sites across all frontend JS files:
  - /stats: 5s TTL (was called 962 times in 3 min)
  - /nodes/:pubkey: 15s, /health: 30s, /observers: 30s
  - /channels: 15s, messages: 10s
  - /analytics/*: 60s, /bulk-health: 60s, /network-status: 60s
  - /nodes?*: 10s
- Skip caching for real-time endpoints (/packets, /resolve-hops, /perf)
- Invalidate /stats, /nodes, /channels caches on WebSocket messages
- Deduplicate in-flight requests (same path returns same promise)
- Add cache hit rate to window.apiPerf() console debugging
- Update all cache busters in index.html
2026-03-20 02:03:25 +00:00
you
6bf5beafcb merge: performance instrumentation 2026-03-20 01:49:19 +00:00
you
36bf6eac82 feat: performance instrumentation — server timing middleware, client API tracking, /api/perf endpoint, #/perf dashboard 2026-03-20 01:34:25 +00:00
you
af94065399 release: v2.0.1 — mobile packets UX 2026-03-20 01:19:19 +00:00
you
95db662c5a merge: mobile packets improvements (v2.0.1) 2026-03-20 01:19:19 +00:00
14 changed files with 438 additions and 91 deletions

View File

@@ -1,6 +1,6 @@
{
"name": "meshcore-analyzer",
"version": "2.0.0",
"version": "2.0.1",
"description": "Community-run alternative to the closed-source `analyzer.letsmesh.net`. MQTT packet collection + open-source web analyzer for the Bay Area MeshCore mesh.",
"main": "index.js",
"scripts": {

View File

@@ -101,10 +101,10 @@
try {
_analyticsData = {};
const [hashData, rfData, topoData, chanData] = await Promise.all([
api('/analytics/hash-sizes'),
api('/analytics/rf'),
api('/analytics/topology'),
api('/analytics/channels'),
api('/analytics/hash-sizes', { ttl: 60000 }),
api('/analytics/rf', { ttl: 60000 }),
api('/analytics/topology', { ttl: 60000 }),
api('/analytics/channels', { ttl: 60000 }),
]);
_analyticsData = { hashData, rfData, topoData, chanData };
renderTab('overview');
@@ -747,7 +747,7 @@
</div>
`;
let allNodes = [];
try { const nd = await api('/nodes?limit=2000'); allNodes = nd.nodes || []; } catch {}
try { const nd = await api('/nodes?limit=2000', { ttl: 10000 }); allNodes = nd.nodes || []; } catch {}
renderHashMatrix(data.topHops, allNodes);
renderCollisions(data.topHops, allNodes);
}
@@ -938,10 +938,10 @@
el.innerHTML = '<div class="text-center text-muted" style="padding:40px">Analyzing route patterns…</div>';
try {
const [d2, d3, d4, d5] = await Promise.all([
api('/analytics/subpaths?minLen=2&maxLen=2&limit=50'),
api('/analytics/subpaths?minLen=3&maxLen=3&limit=30'),
api('/analytics/subpaths?minLen=4&maxLen=4&limit=20'),
api('/analytics/subpaths?minLen=5&maxLen=8&limit=15')
api('/analytics/subpaths?minLen=2&maxLen=2&limit=50', { ttl: 60000 }),
api('/analytics/subpaths?minLen=3&maxLen=3&limit=30', { ttl: 60000 }),
api('/analytics/subpaths?minLen=4&maxLen=4&limit=20', { ttl: 60000 }),
api('/analytics/subpaths?minLen=5&maxLen=8&limit=15', { ttl: 60000 })
]);
function renderTable(data, title) {
@@ -1032,7 +1032,7 @@
panel.classList.remove('collapsed');
panel.innerHTML = '<div class="text-center text-muted" style="padding:40px">Loading…</div>';
try {
const data = await api('/analytics/subpath-detail?hops=' + encodeURIComponent(hopsStr));
const data = await api('/analytics/subpath-detail?hops=' + encodeURIComponent(hopsStr), { ttl: 60000 });
renderSubpathDetail(panel, data);
} catch (e) {
panel.innerHTML = `<div class="text-muted">Error: ${e.message}</div>`;
@@ -1141,9 +1141,9 @@
el.innerHTML = '<div style="padding:40px;text-align:center;color:var(--text-muted)">Loading node analytics…</div>';
try {
const [nodesResp, bulkHealth, netStatus] = await Promise.all([
api('/nodes?limit=200&sortBy=lastSeen'),
api('/nodes/bulk-health?limit=50'),
api('/nodes/network-status')
api('/nodes?limit=200&sortBy=lastSeen', { ttl: 10000 }),
api('/nodes/bulk-health?limit=50', { ttl: 60000 }),
api('/nodes/network-status', { ttl: 60000 })
]);
const nodes = nodesResp.nodes || nodesResp;
const myNodes = JSON.parse(localStorage.getItem('meshcore-my-nodes') || '[]');

View File

@@ -11,12 +11,65 @@ function payloadTypeName(n) { return PAYLOAD_TYPES[n] || 'UNKNOWN'; }
function payloadTypeColor(n) { return PAYLOAD_COLORS[n] || 'unknown'; }
// --- Utilities ---
async function api(path) {
const res = await fetch('/api' + path);
if (!res.ok) throw new Error(`API ${res.status}: ${path}`);
return res.json();
const _apiPerf = { calls: 0, totalMs: 0, log: [], cacheHits: 0 };
const _apiCache = new Map();
const _inflight = new Map();
async function api(path, { ttl = 0, bust = false } = {}) {
const t0 = performance.now();
if (!bust && ttl > 0) {
const cached = _apiCache.get(path);
if (cached && Date.now() < cached.expires) {
_apiPerf.calls++;
_apiPerf.cacheHits++;
_apiPerf.log.push({ path, ms: 0, time: Date.now(), cached: true });
if (_apiPerf.log.length > 200) _apiPerf.log.shift();
return cached.data;
}
}
// Deduplicate in-flight requests
if (_inflight.has(path)) return _inflight.get(path);
const promise = (async () => {
const res = await fetch('/api' + path);
if (!res.ok) throw new Error(`API ${res.status}: ${path}`);
const data = await res.json();
const ms = performance.now() - t0;
_apiPerf.calls++;
_apiPerf.totalMs += ms;
_apiPerf.log.push({ path, ms: Math.round(ms), time: Date.now() });
if (_apiPerf.log.length > 200) _apiPerf.log.shift();
if (ms > 500) console.warn(`[SLOW API] ${path} took ${Math.round(ms)}ms`);
if (ttl > 0) _apiCache.set(path, { data, expires: Date.now() + ttl });
return data;
})();
_inflight.set(path, promise);
promise.finally(() => _inflight.delete(path));
return promise;
}
function invalidateApiCache(prefix) {
for (const key of _apiCache.keys()) {
if (key.startsWith(prefix || '')) _apiCache.delete(key);
}
}
// Expose for console debugging: apiPerf()
window.apiPerf = function() {
const byPath = {};
_apiPerf.log.forEach(e => {
if (!byPath[e.path]) byPath[e.path] = { count: 0, totalMs: 0, maxMs: 0 };
byPath[e.path].count++;
byPath[e.path].totalMs += e.ms;
if (e.ms > byPath[e.path].maxMs) byPath[e.path].maxMs = e.ms;
});
const rows = Object.entries(byPath).map(([p, s]) => ({
path: p, count: s.count, avgMs: Math.round(s.totalMs / s.count), maxMs: s.maxMs,
totalMs: Math.round(s.totalMs)
})).sort((a, b) => b.totalMs - a.totalMs);
console.table(rows);
const hitRate = _apiPerf.calls ? Math.round(_apiPerf.cacheHits / _apiPerf.calls * 100) : 0;
console.log(`Cache: ${_apiPerf.cacheHits} hits / ${_apiPerf.calls} calls (${hitRate}% hit rate)`);
return { calls: _apiPerf.calls, avgMs: Math.round(_apiPerf.totalMs / (_apiPerf.calls - _apiPerf.cacheHits || 1)), cacheHits: _apiPerf.cacheHits, hitRate: hitRate + '%', endpoints: rows };
};
function timeAgo(iso) {
if (!iso) return '—';
const s = Math.floor((Date.now() - new Date(iso).getTime()) / 1000);
@@ -140,6 +193,10 @@ function connectWS() {
ws.onmessage = (e) => {
try {
const msg = JSON.parse(e.data);
// Invalidate caches when new data arrives
invalidateApiCache('/stats');
invalidateApiCache('/nodes');
invalidateApiCache('/channels');
wsListeners.forEach(fn => fn(msg));
} catch {}
};
@@ -217,7 +274,10 @@ function navigate() {
const app = document.getElementById('app');
if (pages[basePage]?.init) {
const t0 = performance.now();
pages[basePage].init(app, routeParam);
const ms = performance.now() - t0;
if (ms > 100) console.warn(`[SLOW PAGE] ${basePage} init took ${Math.round(ms)}ms`);
app.classList.remove('page-enter'); void app.offsetWidth; app.classList.add('page-enter');
} else {
app.innerHTML = `<div style="padding:40px;text-align:center;color:#6b7280"><h2>${route}</h2><p>Page not yet implemented.</p></div>`;
@@ -290,7 +350,7 @@ window.addEventListener('DOMContentLoaded', () => {
favDropdown.innerHTML = '<div class="fav-dd-loading">Loading...</div>';
const items = await Promise.all(favs.map(async (pk) => {
try {
const h = await api('/nodes/' + pk + '/health');
const h = await api('/nodes/' + pk + '/health', { ttl: 30000 });
const age = h.stats.lastHeard ? Date.now() - new Date(h.stats.lastHeard).getTime() : null;
const status = age === null ? '🔴' : age < 3600000 ? '🟢' : age < 86400000 ? '🟡' : '🔴';
return '<a href="#/nodes/' + pk + '" class="fav-dd-item" data-key="' + pk + '">'
@@ -394,7 +454,7 @@ window.addEventListener('DOMContentLoaded', () => {
// --- Nav Stats ---
async function updateNavStats() {
try {
const stats = await api('/stats');
const stats = await api('/stats', { ttl: 5000 });
const el = document.getElementById('navStats');
if (el) {
el.innerHTML = `<span class="stat-val">${stats.totalPackets}</span> pkts · <span class="stat-val">${stats.totalNodes}</span> nodes · <span class="stat-val">${stats.totalObservers}</span> obs`;

View File

@@ -18,7 +18,7 @@
if (cached && !cached.fetchedAt) return cached; // legacy null entries
}
try {
const data = await api('/nodes/search?q=' + encodeURIComponent(name));
const data = await api('/nodes/search?q=' + encodeURIComponent(name), { ttl: 10000 });
// Try exact match first, then case-insensitive, then contains
const nodes = data.nodes || [];
const match = nodes.find(n => n.name === name)
@@ -110,7 +110,7 @@
}
try {
const detail = await api('/nodes/' + encodeURIComponent(node.public_key));
const detail = await api('/nodes/' + encodeURIComponent(node.public_key), { ttl: 15000 });
const n = detail.node;
const adverts = detail.recentAdverts || [];
const role = n.is_repeater ? '📡 Repeater' : n.is_room ? '🏠 Room' : n.is_sensor ? '🌡 Sensor' : '📻 Companion';
@@ -389,7 +389,7 @@
async function loadChannels(silent) {
try {
const data = await api('/channels');
const data = await api('/channels', { ttl: 15000 });
channels = (data.channels || []).sort((a, b) => (b.lastActivity || '').localeCompare(a.lastActivity || ''));
renderChannelList();
} catch (e) {
@@ -451,7 +451,7 @@
msgEl.innerHTML = '<div class="ch-loading">Loading messages…</div>';
try {
const data = await api(`/channels/${hash}/messages?limit=200`);
const data = await api(`/channels/${hash}/messages?limit=200`, { ttl: 10000 });
messages = data.messages || [];
renderMessages();
scrollToBottom();
@@ -466,7 +466,7 @@
if (!msgEl) return;
const wasAtBottom = msgEl.scrollHeight - msgEl.scrollTop - msgEl.clientHeight < 60;
try {
const data = await api(`/channels/${selectedHash}/messages?limit=200`);
const data = await api(`/channels/${selectedHash}/messages?limit=200`, { ttl: 10000 });
const newMsgs = data.messages || [];
// #92: Use message ID/hash for change detection instead of count + timestamp
var _getLastId = function (arr) { var m = arr.length ? arr[arr.length - 1] : null; return m ? (m.id || m.packetId || m.timestamp || '') : ''; };

View File

@@ -146,7 +146,7 @@
if (!q) { suggest.classList.remove('open'); input.setAttribute('aria-expanded', 'false'); input.setAttribute('aria-activedescendant', ''); return; }
searchTimeout = setTimeout(async () => {
try {
const data = await api('/nodes/search?q=' + encodeURIComponent(q));
const data = await api('/nodes/search?q=' + encodeURIComponent(q), { ttl: 10000 });
const nodes = data.nodes || [];
if (!nodes.length) {
suggest.innerHTML = '<div class="suggest-empty">No nodes found</div>';
@@ -247,7 +247,7 @@
const cards = await Promise.all(myNodes.map(async (mn) => {
try {
const h = await api('/nodes/' + encodeURIComponent(mn.pubkey) + '/health');
const h = await api('/nodes/' + encodeURIComponent(mn.pubkey) + '/health', { ttl: 30000 });
const node = h.node || {};
const stats = h.stats || {};
const obs = h.observers || [];
@@ -369,7 +369,7 @@
// ==================== STATS ====================
async function loadStats() {
try {
const s = await api('/stats');
const s = await api('/stats', { ttl: 5000 });
const el = document.getElementById('homeStats');
if (!el) return;
el.innerHTML = `
@@ -391,7 +391,7 @@
if (journey) journey.classList.remove('visible');
try {
const h = await api('/nodes/' + encodeURIComponent(pubkey) + '/health');
const h = await api('/nodes/' + encodeURIComponent(pubkey) + '/health', { ttl: 30000 });
const node = h.node || {};
const stats = h.stats || {};
const packets = h.recentPackets || [];

View File

@@ -20,7 +20,7 @@
<meta name="twitter:title" content="MeshCore Analyzer">
<meta name="twitter:description" content="Real-time MeshCore LoRa mesh network analyzer — live packet visualization, node tracking, channel decryption, and route analysis.">
<meta name="twitter:image" content="https://raw.githubusercontent.com/Kpa-clawbot/meshcore-analyzer/master/public/og-image.png">
<link rel="stylesheet" href="style.css?v=1773969261">
<link rel="stylesheet" href="style.css?v=1773970465">
<link rel="stylesheet" href="home.css">
<link rel="stylesheet" href="live.css?v=1773966856">
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css"
@@ -76,16 +76,17 @@
<main id="app" role="main"></main>
<script src="vendor/qrcode.js"></script>
<script src="app.js?v=1774079160"></script>
<script src="home.js?v=1774079160"></script>
<script src="packets.js?v=1773969349"></script>
<script src="map.js?v=1774079160" onerror="console.error('Failed to load:', this.src)"></script>
<script src="channels.js?v=1774079160" onerror="console.error('Failed to load:', this.src)"></script>
<script src="nodes.js?v=1773961950" onerror="console.error('Failed to load:', this.src)"></script>
<script src="traces.js?v=1774079160" onerror="console.error('Failed to load:', this.src)"></script>
<script src="analytics.js?v=1773961035" onerror="console.error('Failed to load:', this.src)"></script>
<script src="app.js?v=1773972187"></script>
<script src="home.js?v=1773972187"></script>
<script src="packets.js?v=1773972187"></script>
<script src="map.js?v=1773972187" onerror="console.error('Failed to load:', this.src)"></script>
<script src="channels.js?v=1773972187" onerror="console.error('Failed to load:', this.src)"></script>
<script src="nodes.js?v=1773972187" onerror="console.error('Failed to load:', this.src)"></script>
<script src="traces.js?v=1773972187" onerror="console.error('Failed to load:', this.src)"></script>
<script src="analytics.js?v=1773972187" onerror="console.error('Failed to load:', this.src)"></script>
<script src="live.js?v=1773964458" onerror="console.error('Failed to load:', this.src)"></script>
<script src="observers.js?v=1774079160" onerror="console.error('Failed to load:', this.src)"></script>
<script src="node-analytics.js?v=1773961276" onerror="console.error('Failed to load:', this.src)"></script>
<script src="observers.js?v=1773972187" onerror="console.error('Failed to load:', this.src)"></script>
<script src="node-analytics.js?v=1773972187" onerror="console.error('Failed to load:', this.src)"></script>
<script src="perf.js?v=1" onerror="console.error('Failed to load:', this.src)"></script>
</body>
</html>

View File

@@ -245,12 +245,12 @@
async function loadNodes() {
try {
const data = await api(`/nodes?limit=10000&lastHeard=${filters.lastHeard}`);
const data = await api(`/nodes?limit=10000&lastHeard=${filters.lastHeard}`, { ttl: 10000 });
nodes = data.nodes || [];
buildRoleChecks(data.counts || {});
// Load observers for jump buttons
const obsData = await api('/observers');
const obsData = await api('/observers', { ttl: 30000 });
observers = obsData.observers || [];
buildJumpButtons();

View File

@@ -40,7 +40,7 @@
let data;
try {
data = await api('/nodes/' + encodeURIComponent(pubkey) + '/analytics?days=' + days);
data = await api('/nodes/' + encodeURIComponent(pubkey) + '/analytics?days=' + days, { ttl: 60000 });
} catch (e) {
container.innerHTML = '<div style="padding:40px;text-align:center;color:#ff6b6b">Failed to load analytics: ' + escapeHtml(e.message) + '</div>';
return;

View File

@@ -85,8 +85,8 @@
const body = document.getElementById('nodeFullBody');
try {
const [nodeData, healthData] = await Promise.all([
api('/nodes/' + encodeURIComponent(pubkey)),
api('/nodes/' + encodeURIComponent(pubkey) + '/health').catch(() => null)
api('/nodes/' + encodeURIComponent(pubkey), { ttl: 15000 }),
api('/nodes/' + encodeURIComponent(pubkey) + '/health', { ttl: 30000 }).catch(() => null)
]);
const n = nodeData.node;
const adverts = nodeData.recentAdverts || [];
@@ -228,7 +228,7 @@
if (activeTab !== 'all') params.set('role', activeTab);
if (search) params.set('search', search);
if (lastHeard) params.set('lastHeard', lastHeard);
const data = await api('/nodes?' + params);
const data = await api('/nodes?' + params, { ttl: 10000 });
nodes = data.nodes || [];
counts = data.counts || {};
@@ -238,7 +238,7 @@
const missing = myNodes.filter(mn => !existingKeys.has(mn.pubkey));
if (missing.length) {
const fetched = await Promise.allSettled(
missing.map(mn => api('/nodes/' + encodeURIComponent(mn.pubkey)))
missing.map(mn => api('/nodes/' + encodeURIComponent(mn.pubkey), { ttl: 15000 }))
);
fetched.forEach(r => {
if (r.status === 'fulfilled' && r.value && r.value.public_key) nodes.push(r.value);
@@ -401,8 +401,8 @@
try {
const [data, healthData] = await Promise.all([
api('/nodes/' + encodeURIComponent(pubkey)),
api('/nodes/' + encodeURIComponent(pubkey) + '/health').catch(() => null)
api('/nodes/' + encodeURIComponent(pubkey), { ttl: 15000 }),
api('/nodes/' + encodeURIComponent(pubkey) + '/health', { ttl: 30000 }).catch(() => null)
]);
data.healthData = healthData;
renderDetail(panel, data);

View File

@@ -38,7 +38,7 @@
async function loadObservers() {
try {
const data = await api('/observers');
const data = await api('/observers', { ttl: 30000 });
observers = data.observers || [];
render();
} catch (e) {

View File

@@ -207,7 +207,7 @@
async function loadObservers() {
try {
const data = await api('/observers');
const data = await api('/observers', { ttl: 30000 });
observers = data.observers || [];
} catch {}
}

88
public/perf.js Normal file
View File

@@ -0,0 +1,88 @@
/* === MeshCore Analyzer — perf.js === */
'use strict';
(function () {
let interval = null;
async function render(app) {
app.innerHTML = '<div style="height:100%;overflow-y:auto;padding:16px 24px;"><h2>⚡ Performance Dashboard</h2><div id="perfContent">Loading...</div></div>';
await refresh();
}
async function refresh() {
const el = document.getElementById('perfContent');
if (!el) return;
try {
const [server, client] = await Promise.all([
fetch('/api/perf').then(r => r.json()),
Promise.resolve(window.apiPerf ? window.apiPerf() : null)
]);
let html = '';
// Server overview
html += `<div style="display:flex;gap:16px;flex-wrap:wrap;margin:16px 0;">
<div class="perf-card"><div class="perf-num">${server.totalRequests}</div><div class="perf-label">Total Requests</div></div>
<div class="perf-card"><div class="perf-num">${server.avgMs}ms</div><div class="perf-label">Avg Response</div></div>
<div class="perf-card"><div class="perf-num">${Math.round(server.uptime / 60)}m</div><div class="perf-label">Uptime</div></div>
<div class="perf-card"><div class="perf-num">${server.slowQueries.length}</div><div class="perf-label">Slow (&gt;100ms)</div></div>
</div>`;
// Server endpoints table
const eps = Object.entries(server.endpoints);
if (eps.length) {
html += '<h3>Server Endpoints (sorted by total time)</h3>';
html += '<div style="overflow-x:auto"><table class="perf-table"><thead><tr><th>Endpoint</th><th>Count</th><th>Avg</th><th>P50</th><th>P95</th><th>Max</th><th>Total</th></tr></thead><tbody>';
for (const [path, s] of eps) {
const total = Math.round(s.count * s.avgMs);
const cls = s.p95Ms > 200 ? ' class="perf-slow"' : s.p95Ms > 50 ? ' class="perf-warn"' : '';
html += `<tr${cls}><td><code>${path}</code></td><td>${s.count}</td><td>${s.avgMs}ms</td><td>${s.p50Ms}ms</td><td>${s.p95Ms}ms</td><td>${s.maxMs}ms</td><td>${total}ms</td></tr>`;
}
html += '</tbody></table></div>';
}
// Client API calls
if (client && client.endpoints.length) {
html += '<h3>Client API Calls (this session)</h3>';
html += '<div style="overflow-x:auto"><table class="perf-table"><thead><tr><th>Endpoint</th><th>Count</th><th>Avg</th><th>Max</th><th>Total</th></tr></thead><tbody>';
for (const s of client.endpoints) {
const cls = s.maxMs > 500 ? ' class="perf-slow"' : s.avgMs > 200 ? ' class="perf-warn"' : '';
html += `<tr${cls}><td><code>${s.path}</code></td><td>${s.count}</td><td>${s.avgMs}ms</td><td>${s.maxMs}ms</td><td>${s.totalMs}ms</td></tr>`;
}
html += '</tbody></table></div>';
}
// Slow queries
if (server.slowQueries.length) {
html += '<h3>Recent Slow Queries (&gt;100ms)</h3>';
html += '<div style="overflow-x:auto"><table class="perf-table"><thead><tr><th>Time</th><th>Path</th><th>Duration</th><th>Status</th></tr></thead><tbody>';
for (const q of server.slowQueries.slice().reverse()) {
html += `<tr class="perf-slow"><td>${new Date(q.time).toLocaleTimeString()}</td><td><code>${q.path}</code></td><td>${q.ms}ms</td><td>${q.status}</td></tr>`;
}
html += '</tbody></table></div>';
}
html += `<div style="margin-top:16px"><button id="perfReset" style="padding:8px 16px;cursor:pointer">Reset Stats</button> <button id="perfRefresh" style="padding:8px 16px;cursor:pointer">Refresh</button></div>`;
el.innerHTML = html;
document.getElementById('perfReset')?.addEventListener('click', async () => {
await fetch('/api/perf/reset', { method: 'POST' });
if (window._apiPerf) { window._apiPerf = { calls: 0, totalMs: 0, log: [] }; }
refresh();
});
document.getElementById('perfRefresh')?.addEventListener('click', refresh);
} catch (err) {
el.innerHTML = `<p style="color:red">Error: ${err.message}</p>`;
}
}
registerPage('perf', {
init(app) {
render(app);
interval = setInterval(refresh, 5000);
},
destroy() {
if (interval) { clearInterval(interval); interval = null; }
}
});
})();

View File

@@ -1413,3 +1413,16 @@ tr[data-hops]:hover { background: rgba(59,130,246,0.1); }
}
.mobile-sheet-close:hover { color: var(--text); }
.mobile-sheet-content { padding-top: 4px; }
/* Perf dashboard */
.perf-card { background: var(--surface-1); border: 1px solid var(--border); border-radius: 8px; padding: 12px 20px; min-width: 120px; text-align: center; }
.perf-num { font-size: 24px; font-weight: 800; color: var(--text); font-variant-numeric: tabular-nums; }
.perf-label { font-size: 11px; color: var(--text-muted); text-transform: uppercase; letter-spacing: 0.5px; margin-top: 4px; }
.perf-table { width: 100%; border-collapse: collapse; font-size: 13px; }
.perf-table th { text-align: left; padding: 6px 10px; border-bottom: 2px solid var(--border); color: var(--text-muted); font-size: 11px; text-transform: uppercase; }
.perf-table td { padding: 5px 10px; border-bottom: 1px solid var(--border); font-variant-numeric: tabular-nums; }
.perf-table code { font-size: 12px; color: var(--text); }
.perf-table .perf-slow { background: rgba(239, 68, 68, 0.08); }
.perf-table .perf-slow td { color: #ef4444; }
.perf-table .perf-warn { background: rgba(251, 191, 36, 0.06); }
.perf-table .perf-warn td { color: #f59e0b; }

265
server.js
View File

@@ -28,12 +28,102 @@ function computeContentHash(rawHex) {
const db = require('./db');
const channelKeys = require("./config.json").channelKeys || {};
// --- TTL Cache ---
class TTLCache {
constructor() { this.store = new Map(); this.hits = 0; this.misses = 0; }
get(key) {
const entry = this.store.get(key);
if (!entry) { this.misses++; return undefined; }
if (Date.now() > entry.expires) { this.store.delete(key); this.misses++; return undefined; }
this.hits++;
return entry.value;
}
set(key, value, ttlMs) {
this.store.set(key, { value, expires: Date.now() + ttlMs });
}
invalidate(prefix) {
for (const key of this.store.keys()) {
if (key.startsWith(prefix)) this.store.delete(key);
}
}
clear() { this.store.clear(); }
get size() { return this.store.size; }
}
const cache = new TTLCache();
// Seed DB if empty
db.seed();
const app = express();
const server = http.createServer(app);
// --- Performance Instrumentation ---
const perfStats = {
requests: 0,
totalMs: 0,
endpoints: {}, // { path: { count, totalMs, maxMs, avgMs, p95: [], lastSlow } }
slowQueries: [], // last 50 requests > 100ms
startedAt: Date.now(),
reset() {
this.requests = 0; this.totalMs = 0; this.endpoints = {}; this.slowQueries = []; this.startedAt = Date.now();
}
};
app.use((req, res, next) => {
if (!req.path.startsWith('/api/')) return next();
const start = process.hrtime.bigint();
const origEnd = res.end;
res.end = function(...args) {
const ms = Number(process.hrtime.bigint() - start) / 1e6;
perfStats.requests++;
perfStats.totalMs += ms;
// Normalize parameterized routes
const key = req.route ? req.route.path : req.path.replace(/[0-9a-f]{8,}/gi, ':id');
if (!perfStats.endpoints[key]) perfStats.endpoints[key] = { count: 0, totalMs: 0, maxMs: 0, recent: [] };
const ep = perfStats.endpoints[key];
ep.count++;
ep.totalMs += ms;
if (ms > ep.maxMs) ep.maxMs = ms;
ep.recent.push(ms);
if (ep.recent.length > 100) ep.recent.shift();
if (ms > 100) {
perfStats.slowQueries.push({ path: req.path, ms: Math.round(ms * 10) / 10, time: new Date().toISOString(), status: res.statusCode });
if (perfStats.slowQueries.length > 50) perfStats.slowQueries.shift();
}
origEnd.apply(res, args);
};
next();
});
app.get('/api/perf', (req, res) => {
const summary = {};
for (const [path, ep] of Object.entries(perfStats.endpoints)) {
const sorted = [...ep.recent].sort((a, b) => a - b);
const p95 = sorted[Math.floor(sorted.length * 0.95)] || 0;
const p50 = sorted[Math.floor(sorted.length * 0.5)] || 0;
summary[path] = {
count: ep.count,
avgMs: Math.round(ep.totalMs / ep.count * 10) / 10,
p50Ms: Math.round(p50 * 10) / 10,
p95Ms: Math.round(p95 * 10) / 10,
maxMs: Math.round(ep.maxMs * 10) / 10,
};
}
// Sort by total time spent (count * avg) descending
const sorted = Object.entries(summary).sort((a, b) => (b[1].count * b[1].avgMs) - (a[1].count * a[1].avgMs));
res.json({
uptime: Math.round((Date.now() - perfStats.startedAt) / 1000),
totalRequests: perfStats.requests,
avgMs: perfStats.requests ? Math.round(perfStats.totalMs / perfStats.requests * 10) / 10 : 0,
endpoints: Object.fromEntries(sorted),
slowQueries: perfStats.slowQueries.slice(-20),
cache: { size: cache.size, hits: cache.hits, misses: cache.misses, hitRate: cache.hits + cache.misses > 0 ? Math.round(cache.hits / (cache.hits + cache.misses) * 1000) / 10 : 0 },
});
});
app.post('/api/perf/reset', (req, res) => { perfStats.reset(); res.json({ ok: true }); });
// --- WebSocket ---
const wss = new WebSocketServer({ server });
@@ -198,6 +288,15 @@ try {
db.upsertObserver({ id: observerId, iata: region });
}
// Invalidate caches on new data
cache.invalidate('analytics:');
cache.invalidate('channels');
cache.invalidate('node:');
cache.invalidate('health:');
cache.invalidate('observers');
cache.invalidate('bulk-health');
const broadcastData = { id: packetId, raw: msg.raw, decoded, snr: msg.SNR, rssi: msg.RSSI, hash: msg.hash, observer: observerId };
broadcast({ type: 'packet', data: broadcastData });
@@ -533,6 +632,15 @@ app.post('/api/packets', (req, res) => {
db.upsertObserver({ id: observer, iata: region || null });
}
// Invalidate caches on new data
cache.invalidate('analytics:');
cache.invalidate('channels');
cache.invalidate('node:');
cache.invalidate('health:');
cache.invalidate('observers');
cache.invalidate('bulk-health');
broadcast({ type: 'packet', data: { id: packetId, decoded } });
res.json({ id: packetId, decoded });
@@ -581,41 +689,85 @@ app.get('/api/nodes/search', (req, res) => {
// Bulk health summary for analytics — single query approach (MUST be before :pubkey routes)
app.get('/api/nodes/bulk-health', (req, res) => {
const limit = Math.min(Number(req.query.limit) || 50, 200);
const _ck = 'bulk-health:' + limit;
const _c = cache.get(_ck); if (_c) return res.json(_c);
const nodes = db.db.prepare(`SELECT * FROM nodes ORDER BY last_seen DESC LIMIT ?`).all(limit);
const todayStart = new Date();
todayStart.setUTCHours(0, 0, 0, 0);
const todayISO = todayStart.toISOString();
const results = nodes.map(node => {
const pk = node.public_key;
const keyPattern = `%${pk}%`;
const namePattern = node.name ? `%${node.name.replace(/[%_]/g, '')}%` : null;
const where = namePattern
? `(decoded_json LIKE @k OR decoded_json LIKE @n)`
: `decoded_json LIKE @k`;
const p = namePattern ? { k: keyPattern, n: namePattern } : { k: keyPattern };
if (nodes.length === 0) { cache.set(_ck, [], 60000); return res.json([]); }
const observerRows = db.db.prepare(`
SELECT observer_id, observer_name, AVG(snr) as avgSnr, AVG(rssi) as avgRssi, COUNT(*) as packetCount
FROM packets WHERE ${where} AND observer_id IS NOT NULL GROUP BY observer_id ORDER BY packetCount DESC
`).all(p);
const totalPackets = db.db.prepare(`SELECT COUNT(*) as c FROM packets WHERE ${where}`).get(p).c;
const packetsToday = db.db.prepare(`SELECT COUNT(*) as c FROM packets WHERE ${where} AND timestamp > @s`).get({ ...p, s: todayISO }).c;
const avgSnr = db.db.prepare(`SELECT AVG(snr) as v FROM packets WHERE ${where}`).get(p).v;
const lastHeard = db.db.prepare(`SELECT MAX(timestamp) as v FROM packets WHERE ${where}`).get(p).v;
return {
public_key: pk,
name: node.name,
role: node.role,
lat: node.lat,
lon: node.lon,
stats: { totalPackets, packetsToday, avgSnr, lastHeard },
observers: observerRows
};
// Build OR conditions for all nodes to fetch matching packets in ONE query
const likeConditions = [];
const params = {};
nodes.forEach((node, i) => {
params['k' + i] = '%' + node.public_key + '%';
likeConditions.push('decoded_json LIKE @k' + i);
if (node.name) {
params['n' + i] = '%' + node.name.replace(/[%_]/g, '') + '%';
likeConditions.push('decoded_json LIKE @n' + i);
}
});
// Single query to get ALL matching packets
const allPackets = db.db.prepare(
'SELECT decoded_json, snr, rssi, timestamp, observer_id, observer_name FROM packets WHERE ' + likeConditions.join(' OR ')
).all(params);
// Match packets to nodes in JS
const nodeMap = new Map();
for (const node of nodes) {
nodeMap.set(node.public_key, {
node, totalPackets: 0, packetsToday: 0, snrSum: 0, snrCount: 0, lastHeard: null,
observers: {}
});
}
for (const pkt of allPackets) {
const dj = pkt.decoded_json || '';
for (const [pk, data] of nodeMap) {
const nd = data.node;
if (!dj.includes(pk) && !(nd.name && dj.includes(nd.name))) continue;
data.totalPackets++;
if (pkt.timestamp > todayISO) data.packetsToday++;
if (pkt.snr != null) { data.snrSum += pkt.snr; data.snrCount++; }
if (!data.lastHeard || pkt.timestamp > data.lastHeard) data.lastHeard = pkt.timestamp;
if (pkt.observer_id) {
if (!data.observers[pkt.observer_id]) {
data.observers[pkt.observer_id] = { name: pkt.observer_name, snrSum: 0, snrCount: 0, rssiSum: 0, rssiCount: 0, count: 0 };
}
const obs = data.observers[pkt.observer_id];
obs.count++;
if (pkt.snr != null) { obs.snrSum += pkt.snr; obs.snrCount++; }
if (pkt.rssi != null) { obs.rssiSum += pkt.rssi; obs.rssiCount++; }
}
}
}
const results = [];
for (const [pk, data] of nodeMap) {
const observerRows = Object.entries(data.observers)
.map(([id, o]) => ({
observer_id: id, observer_name: o.name,
avgSnr: o.snrCount ? o.snrSum / o.snrCount : null,
avgRssi: o.rssiCount ? o.rssiSum / o.rssiCount : null,
packetCount: o.count
}))
.sort((a, b) => b.packetCount - a.packetCount);
results.push({
public_key: pk, name: data.node.name, role: data.node.role,
lat: data.node.lat, lon: data.node.lon,
stats: {
totalPackets: data.totalPackets, packetsToday: data.packetsToday,
avgSnr: data.snrCount ? data.snrSum / data.snrCount : null, lastHeard: data.lastHeard
},
observers: observerRows
});
}
cache.set(_ck, results, 60000);
res.json(results);
});
@@ -640,16 +792,21 @@ app.get('/api/nodes/network-status', (req, res) => {
});
app.get('/api/nodes/:pubkey', (req, res) => {
const _ck = 'node:' + req.params.pubkey;
const _c = cache.get(_ck); if (_c) return res.json(_c);
const node = db.getNode(req.params.pubkey);
if (!node) return res.status(404).json({ error: 'Not found' });
const recentAdverts = node.recentPackets || [];
delete node.recentPackets;
res.json({ node, recentAdverts });
const _nResult = { node, recentAdverts };
cache.set(_ck, _nResult, 30000);
res.json(_nResult);
});
// --- Analytics API ---
// --- RF Analytics ---
app.get('/api/analytics/rf', (req, res) => {
const _c = cache.get('analytics:rf'); if (_c) return res.json(_c);
const PTYPES = { 0:'REQ',1:'RESPONSE',2:'TXT_MSG',3:'ACK',4:'ADVERT',5:'GRP_TXT',7:'ANON_REQ',8:'PATH',9:'TRACE',11:'CONTROL' };
const packets = db.db.prepare(`SELECT snr, rssi, payload_type, timestamp, raw_hex FROM packets WHERE snr IS NOT NULL`).all();
@@ -710,7 +867,7 @@ app.get('/api/analytics/rf', (req, res) => {
const times = packets.map(p => new Date(p.timestamp).getTime());
const timeSpanHours = times.length ? (Math.max(...times) - Math.min(...times)) / 3600000 : 0;
res.json({
const _rfResult = {
totalPackets: packets.length,
snr: { min: Math.min(...snrVals), max: Math.max(...snrVals), avg: snrAvg, median: median(snrVals), stddev: stddev(snrVals, snrAvg) },
rssi: { min: Math.min(...rssiVals), max: Math.max(...rssiVals), avg: rssiAvg, median: median(rssiVals), stddev: stddev(rssiVals, rssiAvg) },
@@ -719,11 +876,14 @@ app.get('/api/analytics/rf', (req, res) => {
maxPacketSize: packetSizes.length ? Math.max(...packetSizes) : 0,
avgPacketSize: packetSizes.length ? Math.round(packetSizes.reduce((a, b) => a + b, 0) / packetSizes.length) : 0,
packetsPerHour, payloadTypes, snrByType: snrByTypeArr, signalOverTime, scatterData, timeSpanHours
});
};
cache.set('analytics:rf', _rfResult, 60000);
res.json(_rfResult);
});
// --- Topology Analytics ---
app.get('/api/analytics/topology', (req, res) => {
const _c = cache.get('analytics:topology'); if (_c) return res.json(_c);
const packets = db.db.prepare(`SELECT path_json, snr, decoded_json, observer_id FROM packets WHERE path_json IS NOT NULL AND path_json != '[]'`).all();
const allNodes = db.db.prepare('SELECT public_key, name, lat, lon FROM nodes WHERE name IS NOT NULL').all();
const resolveHop = (hop, contextPositions) => {
@@ -874,7 +1034,7 @@ app.get('/api/analytics/topology', (req, res) => {
.sort((a, b) => a.minDist - b.minDist)
.slice(0, 50);
res.json({
const _topoResult = {
uniqueNodes: new Set(Object.keys(hopFreq)).size,
avgHops, medianHops, maxHops,
hopDistribution, topRepeaters, topPairs, hopsVsSnr,
@@ -882,11 +1042,14 @@ app.get('/api/analytics/topology', (req, res) => {
perObserverReach,
multiObsNodes,
bestPathList
});
};
cache.set('analytics:topology', _topoResult, 60000);
res.json(_topoResult);
});
// --- Channel Analytics ---
app.get('/api/analytics/channels', (req, res) => {
const _c = cache.get('analytics:channels'); if (_c) return res.json(_c);
const packets = db.db.prepare(`SELECT decoded_json, timestamp FROM packets WHERE payload_type = 5 AND decoded_json IS NOT NULL`).all();
const channels = {};
@@ -935,17 +1098,20 @@ app.get('/api/analytics/channels', (req, res) => {
})
.sort((a, b) => a.hour.localeCompare(b.hour));
res.json({
const _chanResult = {
activeChannels: channelList.length,
decryptable: channelList.filter(c => !c.encrypted).length,
channels: channelList,
topSenders,
channelTimeline,
msgLengths
});
};
cache.set('analytics:channels', _chanResult, 60000);
res.json(_chanResult);
});
app.get('/api/analytics/hash-sizes', (req, res) => {
const _c = cache.get('analytics:hash-sizes'); if (_c) return res.json(_c);
// Get all packets with raw_hex and non-empty paths, extract hash_size from path_length byte
const packets = db.db.prepare(`
SELECT raw_hex, path_json, timestamp, payload_type, decoded_json
@@ -1025,13 +1191,15 @@ app.get('/api/analytics/hash-sizes', (req, res) => {
.sort(([, a], [, b]) => b.packets - a.packets)
.map(([name, data]) => ({ name, ...data }));
res.json({
const _hsResult = {
total: packets.length,
distribution,
hourly,
topHops,
multiByteNodes
});
};
cache.set('analytics:hash-sizes', _hsResult, 60000);
res.json(_hsResult);
});
// Resolve path hop hex prefixes to node names
@@ -1181,6 +1349,7 @@ const channelHashNames = {};
}
app.get('/api/channels', (req, res) => {
const _c = cache.get('channels'); if (_c) return res.json(_c);
const packets = db.db.prepare(`SELECT * FROM packets WHERE payload_type = 5 ORDER BY timestamp DESC`).all();
const channelMap = {};
@@ -1239,10 +1408,14 @@ app.get('/api/channels', (req, res) => {
// Don't double-count if already counted above
}
res.json({ channels: Object.values(channelMap) });
const _chResult = { channels: Object.values(channelMap) };
cache.set('channels', _chResult, 30000);
res.json(_chResult);
});
app.get('/api/channels/:hash/messages', (req, res) => {
const _ck = 'channels:' + req.params.hash + ':' + (req.query.limit||100) + ':' + (req.query.offset||0);
const _c = cache.get(_ck); if (_c) return res.json(_c);
const { limit = 100, offset = 0 } = req.query;
const channelHash = req.params.hash;
const packets = db.db.prepare(`SELECT * FROM packets WHERE payload_type = 5 ORDER BY timestamp ASC`).all();
@@ -1302,17 +1475,22 @@ app.get('/api/channels/:hash/messages', (req, res) => {
const start = Math.max(0, total - Number(limit) - Number(offset));
const end = total - Number(offset);
const messages = allMessages.slice(Math.max(0, start), Math.max(0, end));
res.json({ messages, total });
const _msgResult = { messages, total };
cache.set(_ck, _msgResult, 15000);
res.json(_msgResult);
});
app.get('/api/observers', (req, res) => {
const _c = cache.get('observers'); if (_c) return res.json(_c);
const observers = db.getObservers();
const oneHourAgo = new Date(Date.now() - 3600000).toISOString();
const result = observers.map(o => {
const lastHour = db.db.prepare(`SELECT COUNT(*) as count FROM packets WHERE observer_id = ? AND timestamp > ?`).get(o.id, oneHourAgo);
return { ...o, packetsLastHour: lastHour.count };
});
res.json({ observers: result, server_time: new Date().toISOString() });
const _oResult = { observers: result, server_time: new Date().toISOString() };
cache.set('observers', _oResult, 30000);
res.json(_oResult);
});
app.get('/api/traces/:hash', (req, res) => {
@@ -1322,8 +1500,11 @@ app.get('/api/traces/:hash', (req, res) => {
});
app.get('/api/nodes/:pubkey/health', (req, res) => {
const _ck = 'health:' + req.params.pubkey;
const _c = cache.get(_ck); if (_c) return res.json(_c);
const health = db.getNodeHealth(req.params.pubkey);
if (!health) return res.status(404).json({ error: 'Not found' });
cache.set(_ck, health, 30000);
res.json(health);
});
@@ -1336,6 +1517,8 @@ app.get('/api/nodes/:pubkey/analytics', (req, res) => {
// Subpath frequency analysis
app.get('/api/analytics/subpaths', (req, res) => {
const _ck = 'analytics:subpaths:' + (req.query.minLen||2) + ':' + (req.query.maxLen||8) + ':' + (req.query.limit||100);
const _c = cache.get(_ck); if (_c) return res.json(_c);
const minLen = Math.max(2, Number(req.query.minLen) || 2);
const maxLen = Number(req.query.maxLen) || 8;
const packets = db.db.prepare(`SELECT path_json FROM packets WHERE path_json IS NOT NULL AND path_json != '[]'`).all();
@@ -1387,7 +1570,9 @@ app.get('/api/analytics/subpaths', (req, res) => {
.sort((a, b) => b.count - a.count)
.slice(0, limit);
res.json({ subpaths: ranked, totalPaths });
const _spResult = { subpaths: ranked, totalPaths };
cache.set(_ck, _spResult, 60000);
res.json(_spResult);
});
// Subpath detail — stats for a specific subpath (by raw hop prefixes)