mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-03-29 09:40:22 +00:00
feat: live A/B benchmark — launches SQLite-only vs in-memory servers
NO_MEMORY_STORE=1 env var makes packet-store fall through to SQLite for all reads. Benchmark spins up both servers on temp ports and compares: SQLite cold, Memory cold, Memory cached. Results on 27K packets (ARM64): Subpaths 5-8: SQLite 4.7s → cached 1.1ms (4,273×) Bulk health: SQLite 1.8s → cached 1.7ms (1,059×) Topology: SQLite 1.1s → cached 3.0ms (367×) Channels: SQLite 617ms → cached 1.9ms (325×) RF Analytics: SQLite 448ms → cached 1.6ms (280×)
This commit is contained in:
you
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"_comment": "Pre-optimization measurements from /api/perf (v2.0.1+instrumentation, March 20 2026, 27K packets, pure SQLite)",
|
||||
"endpoints": {
|
||||
"Stats": { "avg": 1.0, "bytes": 145 },
|
||||
"Packets (50)": { "avg": 77.5, "bytes": 11700 },
|
||||
"Packets (100)": { "avg": 77.5, "bytes": 23200 },
|
||||
"Packets grouped": { "avg": 77.5, "bytes": 57900 },
|
||||
"Packets filtered (type=5)": { "avg": 77.5, "bytes": 32000 },
|
||||
"Packets timestamps": { "avg": 10.0, "bytes": 721000 },
|
||||
"Nodes list": { "avg": 2.2, "bytes": 11100 },
|
||||
"Node detail": { "avg": 24.2, "bytes": 7800 },
|
||||
"Node health": { "avg": 24.2, "bytes": 8000 },
|
||||
"Bulk health": { "avg": 1610.0, "bytes": 15600 },
|
||||
"Network status": { "avg": 3.3, "bytes": 98 },
|
||||
"Observers": { "avg": 8.9, "bytes": 219 },
|
||||
"Channels": { "avg": 59.9, "bytes": 17600 },
|
||||
"Analytics: RF": { "avg": 271.5, "bytes": 1032198 },
|
||||
"Analytics: Topology": { "avg": 697.4, "bytes": 66600 },
|
||||
"Analytics: Channels": { "avg": 100.5, "bytes": 57800 },
|
||||
"Analytics: Hash sizes": { "avg": 430.1, "bytes": 10100 },
|
||||
"Subpaths (2-hop)": { "avg": 937.0, "bytes": 4800 },
|
||||
"Subpaths (3-hop)": { "avg": 1990.0, "bytes": 4900 },
|
||||
"Subpaths (4-hop)": { "avg": 3090.0, "bytes": 3700 },
|
||||
"Subpaths (5-8 hop)": { "avg": 6190.0, "bytes": 3600 }
|
||||
}
|
||||
}
|
||||
300
benchmark.js
300
benchmark.js
@@ -2,26 +2,30 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Benchmark suite for meshcore-analyzer API endpoints.
|
||||
* Tests with cache enabled (warm) and disabled (cold) to measure true compute cost.
|
||||
* Benchmark suite for meshcore-analyzer.
|
||||
* Launches two server instances — one with in-memory store, one with pure SQLite —
|
||||
* and compares performance side by side.
|
||||
*
|
||||
* Usage: node benchmark.js [--base-url http://localhost:3000] [--runs 5] [--json]
|
||||
* Usage: node benchmark.js [--runs 5] [--json]
|
||||
*/
|
||||
|
||||
const http = require('http');
|
||||
const https = require('https');
|
||||
const { spawn } = require('child_process');
|
||||
const path = require('path');
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const BASE = args.find((a, i) => args[i - 1] === '--base-url') || 'http://127.0.0.1:3000';
|
||||
const RUNS = Number(args.find((a, i) => args[i - 1] === '--runs') || 5);
|
||||
const JSON_OUT = args.includes('--json');
|
||||
|
||||
const PORT_MEM = 13001; // In-memory store
|
||||
const PORT_SQL = 13002; // SQLite-only
|
||||
|
||||
const ENDPOINTS = [
|
||||
{ name: 'Stats', path: '/api/stats' },
|
||||
{ name: 'Packets (50)', path: '/api/packets?limit=50' },
|
||||
{ name: 'Packets (100)', path: '/api/packets?limit=100' },
|
||||
{ name: 'Packets grouped', path: '/api/packets?limit=100&groupByHash=true' },
|
||||
{ name: 'Packets filtered (type=5)', path: '/api/packets?limit=50&type=5' },
|
||||
{ name: 'Packets filtered', path: '/api/packets?limit=50&type=5' },
|
||||
{ name: 'Packets timestamps', path: '/api/packets/timestamps?since=2020-01-01' },
|
||||
{ name: 'Nodes list', path: '/api/nodes?limit=50' },
|
||||
{ name: 'Node detail', path: '/api/nodes/__FIRST_NODE__' },
|
||||
@@ -30,21 +34,20 @@ const ENDPOINTS = [
|
||||
{ name: 'Network status', path: '/api/nodes/network-status' },
|
||||
{ name: 'Observers', path: '/api/observers' },
|
||||
{ name: 'Channels', path: '/api/channels' },
|
||||
{ name: 'Analytics: RF', path: '/api/analytics/rf' },
|
||||
{ name: 'Analytics: Topology', path: '/api/analytics/topology' },
|
||||
{ name: 'Analytics: Channels', path: '/api/analytics/channels' },
|
||||
{ name: 'Analytics: Hash sizes', path: '/api/analytics/hash-sizes' },
|
||||
{ name: 'Subpaths (2-hop)', path: '/api/analytics/subpaths?minLen=2&maxLen=2&limit=50' },
|
||||
{ name: 'Subpaths (3-hop)', path: '/api/analytics/subpaths?minLen=3&maxLen=3&limit=30' },
|
||||
{ name: 'Subpaths (4-hop)', path: '/api/analytics/subpaths?minLen=4&maxLen=4&limit=20' },
|
||||
{ name: 'Subpaths (5-8 hop)', path: '/api/analytics/subpaths?minLen=5&maxLen=8&limit=15' },
|
||||
{ name: 'RF Analytics', path: '/api/analytics/rf' },
|
||||
{ name: 'Topology', path: '/api/analytics/topology' },
|
||||
{ name: 'Channel Analytics', path: '/api/analytics/channels' },
|
||||
{ name: 'Hash Sizes', path: '/api/analytics/hash-sizes' },
|
||||
{ name: 'Subpaths 2-hop', path: '/api/analytics/subpaths?minLen=2&maxLen=2&limit=50' },
|
||||
{ name: 'Subpaths 3-hop', path: '/api/analytics/subpaths?minLen=3&maxLen=3&limit=30' },
|
||||
{ name: 'Subpaths 4-hop', path: '/api/analytics/subpaths?minLen=4&maxLen=4&limit=20' },
|
||||
{ name: 'Subpaths 5-8 hop', path: '/api/analytics/subpaths?minLen=5&maxLen=8&limit=15' },
|
||||
];
|
||||
|
||||
function fetch(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const mod = url.startsWith('https') ? https : http;
|
||||
const t0 = process.hrtime.bigint();
|
||||
const req = mod.get(url, (res) => {
|
||||
const req = http.get(url, (res) => {
|
||||
let body = '';
|
||||
res.on('data', c => body += c);
|
||||
res.on('end', () => {
|
||||
@@ -53,27 +56,124 @@ function fetch(url) {
|
||||
});
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.setTimeout(30000, () => { req.destroy(); reject(new Error('timeout')); });
|
||||
req.setTimeout(60000, () => { req.destroy(); reject(new Error('timeout')); });
|
||||
});
|
||||
}
|
||||
|
||||
function stats(arr) {
|
||||
const sorted = [...arr].sort((a, b) => a - b);
|
||||
const sum = sorted.reduce((a, b) => a + b, 0);
|
||||
return {
|
||||
avg: Math.round(sum / sorted.length * 10) / 10,
|
||||
min: Math.round(sorted[0] * 10) / 10,
|
||||
max: Math.round(sorted[sorted.length - 1] * 10) / 10,
|
||||
p50: Math.round(sorted[Math.floor(sorted.length * 0.5)] * 10) / 10,
|
||||
p95: Math.round(sorted[Math.floor(sorted.length * 0.95)] * 10) / 10,
|
||||
};
|
||||
function median(arr) { const s = [...arr].sort((a,b)=>a-b); return s[Math.floor(s.length/2)]; }
|
||||
function p95(arr) { const s = [...arr].sort((a,b)=>a-b); return s[Math.floor(s.length*0.95)]; }
|
||||
function avg(arr) { return arr.reduce((a,b)=>a+b,0)/arr.length; }
|
||||
function fmt(ms) { return ms >= 1000 ? (ms/1000).toFixed(1)+'s' : ms.toFixed(1)+'ms'; }
|
||||
function fmtSize(b) { return b >= 1048576 ? (b/1048576).toFixed(1)+'MB' : b >= 1024 ? (b/1024).toFixed(0)+'KB' : b+'B'; }
|
||||
|
||||
function launchServer(port, env = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('node', ['server.js'], {
|
||||
cwd: __dirname,
|
||||
env: { ...process.env, PORT: String(port), ...env },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let started = false;
|
||||
const timeout = setTimeout(() => { if (!started) { child.kill(); reject(new Error('Server start timeout')); } }, 30000);
|
||||
|
||||
child.stdout.on('data', (d) => {
|
||||
if (!started && (d.toString().includes('listening') || d.toString().includes('running'))) {
|
||||
started = true; clearTimeout(timeout); resolve(child);
|
||||
}
|
||||
});
|
||||
child.stderr.on('data', (d) => {
|
||||
if (!started && (d.toString().includes('listening') || d.toString().includes('running'))) {
|
||||
started = true; clearTimeout(timeout); resolve(child);
|
||||
}
|
||||
});
|
||||
child.on('exit', (code) => { if (!started) { clearTimeout(timeout); reject(new Error(`Server exited with ${code}`)); } });
|
||||
|
||||
// Fallback: wait longer (SQLite-only mode pre-warms subpaths ~6s)
|
||||
setTimeout(() => {
|
||||
if (!started) {
|
||||
started = true; clearTimeout(timeout);
|
||||
resolve(child);
|
||||
}
|
||||
}, 15000);
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForServer(port, maxMs = 20000) {
|
||||
const t0 = Date.now();
|
||||
while (Date.now() - t0 < maxMs) {
|
||||
try {
|
||||
const r = await fetch(`http://127.0.0.1:${port}/api/stats`);
|
||||
if (r.status === 200) return true;
|
||||
} catch {}
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
}
|
||||
throw new Error(`Server on port ${port} didn't start`);
|
||||
}
|
||||
|
||||
async function benchmarkEndpoints(port, endpoints, nocache = false) {
|
||||
const results = [];
|
||||
for (const ep of endpoints) {
|
||||
const suffix = nocache ? (ep.path.includes('?') ? '&nocache=1' : '?nocache=1') : '';
|
||||
const url = `http://127.0.0.1:${port}${ep.path}${suffix}`;
|
||||
|
||||
// Warm-up
|
||||
try { await fetch(url); } catch {}
|
||||
|
||||
const times = [];
|
||||
let bytes = 0;
|
||||
let failed = false;
|
||||
|
||||
for (let i = 0; i < RUNS; i++) {
|
||||
try {
|
||||
const r = await fetch(url);
|
||||
if (r.status !== 200) { failed = true; break; }
|
||||
times.push(r.ms);
|
||||
bytes = r.bytes;
|
||||
} catch { failed = true; break; }
|
||||
}
|
||||
|
||||
if (failed || !times.length) {
|
||||
results.push({ name: ep.name, failed: true });
|
||||
} else {
|
||||
results.push({
|
||||
name: ep.name,
|
||||
avg: Math.round(avg(times) * 10) / 10,
|
||||
p50: Math.round(median(times) * 10) / 10,
|
||||
p95: Math.round(p95(times) * 10) / 10,
|
||||
bytes
|
||||
});
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async function run() {
|
||||
// Get first node pubkey for parameterized endpoints
|
||||
console.log(`\nMeshCore Analyzer Benchmark — ${RUNS} runs per endpoint`);
|
||||
console.log('Launching servers...\n');
|
||||
|
||||
// Launch both servers
|
||||
let memServer, sqlServer;
|
||||
try {
|
||||
console.log(' Starting in-memory server (port ' + PORT_MEM + ')...');
|
||||
memServer = await launchServer(PORT_MEM, {});
|
||||
await waitForServer(PORT_MEM);
|
||||
console.log(' ✅ In-memory server ready');
|
||||
|
||||
console.log(' Starting SQLite-only server (port ' + PORT_SQL + ')...');
|
||||
sqlServer = await launchServer(PORT_SQL, { NO_MEMORY_STORE: '1' });
|
||||
await waitForServer(PORT_SQL);
|
||||
console.log(' ✅ SQLite-only server ready\n');
|
||||
} catch (e) {
|
||||
console.error('Failed to start servers:', e.message);
|
||||
if (memServer) memServer.kill();
|
||||
if (sqlServer) sqlServer.kill();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Get first node pubkey
|
||||
let firstNode = '';
|
||||
try {
|
||||
const r = await fetch(`${BASE}/api/nodes?limit=1`);
|
||||
const r = await fetch(`http://127.0.0.1:${PORT_MEM}/api/nodes?limit=1`);
|
||||
const data = JSON.parse(r.body);
|
||||
firstNode = data.nodes?.[0]?.public_key || '';
|
||||
} catch {}
|
||||
@@ -83,104 +183,64 @@ async function run() {
|
||||
path: e.path.replace('__FIRST_NODE__', firstNode),
|
||||
}));
|
||||
|
||||
const results = [];
|
||||
// Get packet count
|
||||
try {
|
||||
const r = await fetch(`http://127.0.0.1:${PORT_MEM}/api/stats`);
|
||||
const stats = JSON.parse(r.body);
|
||||
console.log(`Dataset: ${(stats.totalPackets || '?').toLocaleString()} packets\n`);
|
||||
} catch {}
|
||||
|
||||
for (const mode of ['cached', 'nocache']) {
|
||||
if (!JSON_OUT) {
|
||||
console.log(`\n${'='.repeat(70)}`);
|
||||
console.log(` ${mode === 'cached' ? '🟢 CACHE ENABLED (warm)' : '🔴 CACHE DISABLED (cold compute)'}`);
|
||||
console.log(` ${RUNS} runs per endpoint`);
|
||||
console.log(`${'='.repeat(70)}`);
|
||||
console.log(`${'Endpoint'.padEnd(28)} ${'Avg'.padStart(8)} ${'P50'.padStart(8)} ${'P95'.padStart(8)} ${'Max'.padStart(8)} ${'Size'.padStart(9)}`);
|
||||
console.log(`${'-'.repeat(28)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(9)}`);
|
||||
}
|
||||
// Run benchmarks
|
||||
console.log('Benchmarking in-memory store (nocache for true compute cost)...');
|
||||
const memResults = await benchmarkEndpoints(PORT_MEM, endpoints, true);
|
||||
|
||||
for (const ep of endpoints) {
|
||||
const suffix = mode === 'nocache' ? (ep.path.includes('?') ? '&nocache=1' : '?nocache=1') : '';
|
||||
const url = `${BASE}${ep.path}${suffix}`;
|
||||
console.log('Benchmarking SQLite-only (nocache)...');
|
||||
const sqlResults = await benchmarkEndpoints(PORT_SQL, endpoints, true);
|
||||
|
||||
// Warm-up run (discard)
|
||||
try { await fetch(url); } catch {}
|
||||
// Also test cached in-memory for the full picture
|
||||
console.log('Benchmarking in-memory store (cached)...');
|
||||
const memCachedResults = await benchmarkEndpoints(PORT_MEM, endpoints, false);
|
||||
|
||||
const times = [];
|
||||
let bytes = 0;
|
||||
let failed = false;
|
||||
|
||||
for (let i = 0; i < RUNS; i++) {
|
||||
try {
|
||||
const r = await fetch(url);
|
||||
if (r.status !== 200) { failed = true; break; }
|
||||
times.push(r.ms);
|
||||
bytes = r.bytes;
|
||||
} catch { failed = true; break; }
|
||||
}
|
||||
|
||||
if (failed || !times.length) {
|
||||
if (!JSON_OUT) console.log(`${ep.name.padEnd(28)} FAILED`);
|
||||
results.push({ name: ep.name, mode, failed: true });
|
||||
continue;
|
||||
}
|
||||
|
||||
const s = stats(times);
|
||||
const sizeStr = bytes > 1024 ? `${(bytes / 1024).toFixed(1)}KB` : `${bytes}B`;
|
||||
|
||||
results.push({ name: ep.name, mode, ...s, bytes });
|
||||
|
||||
if (!JSON_OUT) {
|
||||
console.log(
|
||||
`${ep.name.padEnd(28)} ${(s.avg + 'ms').padStart(8)} ${(s.p50 + 'ms').padStart(8)} ${(s.p95 + 'ms').padStart(8)} ${(s.max + 'ms').padStart(8)} ${sizeStr.padStart(9)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!JSON_OUT) {
|
||||
// Summary comparison: cached vs nocache
|
||||
console.log(`\n${'='.repeat(80)}`);
|
||||
console.log(' 📊 CACHE IMPACT (avg ms: cached → nocache)');
|
||||
console.log(`${'='.repeat(80)}`);
|
||||
console.log(`${'Endpoint'.padEnd(28)} ${'Cached'.padStart(8)} ${'No-cache'.padStart(8)} ${'Speedup'.padStart(8)}`);
|
||||
console.log(`${'-'.repeat(28)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(8)}`);
|
||||
|
||||
const cached = results.filter(r => r.mode === 'cached' && !r.failed);
|
||||
const nocache = results.filter(r => r.mode === 'nocache' && !r.failed);
|
||||
|
||||
for (const c of cached) {
|
||||
const nc = nocache.find(n => n.name === c.name);
|
||||
if (!nc) continue;
|
||||
const speedup = nc.avg > 0 ? (nc.avg / c.avg).toFixed(1) + '×' : '—';
|
||||
console.log(
|
||||
`${c.name.padEnd(28)} ${(c.avg + 'ms').padStart(8)} ${(nc.avg + 'ms').padStart(8)} ${speedup.padStart(8)}`
|
||||
);
|
||||
}
|
||||
|
||||
// Compare against baseline (pre-optimization) if available
|
||||
let baseline;
|
||||
try { baseline = JSON.parse(require('fs').readFileSync(require('path').join(__dirname, 'benchmark-baseline.json'), 'utf8')); } catch {}
|
||||
if (baseline) {
|
||||
console.log(`\n${'='.repeat(80)}`);
|
||||
console.log(' 🏁 vs BASELINE (pre-optimization, pure SQLite, no in-memory store)');
|
||||
console.log(`${'='.repeat(80)}`);
|
||||
console.log(`${'Endpoint'.padEnd(28)} ${'Baseline'.padStart(9)} ${'Current'.padStart(9)} ${'Speedup'.padStart(9)} ${'Size Δ'.padStart(12)}`);
|
||||
console.log(`${'-'.repeat(28)} ${'-'.repeat(9)} ${'-'.repeat(9)} ${'-'.repeat(9)} ${'-'.repeat(12)}`);
|
||||
|
||||
for (const c of cached) {
|
||||
const bl = baseline.endpoints[c.name];
|
||||
if (!bl) continue;
|
||||
const speedup = bl.avg > 0 && c.avg > 0 ? (bl.avg / c.avg).toFixed(0) + '×' : '—';
|
||||
const sizeOld = bl.bytes > 1024 ? (bl.bytes / 1024).toFixed(0) + 'KB' : bl.bytes + 'B';
|
||||
const sizeNew = c.bytes > 1024 ? (c.bytes / 1024).toFixed(0) + 'KB' : c.bytes + 'B';
|
||||
const sizeChange = bl.bytes && c.bytes ? (((c.bytes - bl.bytes) / bl.bytes) * 100).toFixed(0) + '%' : '—';
|
||||
console.log(
|
||||
`${c.name.padEnd(28)} ${(bl.avg + 'ms').padStart(9)} ${(c.avg + 'ms').padStart(9)} ${speedup.padStart(9)} ${(sizeOld + '→' + sizeNew).padStart(12)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Kill servers
|
||||
memServer.kill();
|
||||
sqlServer.kill();
|
||||
|
||||
if (JSON_OUT) {
|
||||
console.log(JSON.stringify(results, null, 2));
|
||||
console.log(JSON.stringify({ memoryNocache: memResults, sqliteNocache: sqlResults, memoryCached: memCachedResults }, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
// Print results
|
||||
const W = 94;
|
||||
console.log(`\n${'═'.repeat(W)}`);
|
||||
console.log(' 🏁 BENCHMARK RESULTS: SQLite vs In-Memory Store');
|
||||
console.log(`${'═'.repeat(W)}`);
|
||||
console.log(`${'Endpoint'.padEnd(24)} ${'SQLite'.padStart(9)} ${'Memory'.padStart(9)} ${'Cached'.padStart(9)} ${'Speedup'.padStart(9)} ${'Size (SQL)'.padStart(10)} ${'Size (Mem)'.padStart(10)}`);
|
||||
console.log(`${'─'.repeat(24)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(10)} ${'─'.repeat(10)}`);
|
||||
|
||||
for (let i = 0; i < endpoints.length; i++) {
|
||||
const sql = sqlResults[i];
|
||||
const mem = memResults[i];
|
||||
const cached = memCachedResults[i];
|
||||
if (!sql || sql.failed || !mem || mem.failed) {
|
||||
console.log(`${endpoints[i].name.padEnd(24)} ${'FAILED'.padStart(9)}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const speedup = sql.avg > 0 && mem.avg > 0 ? Math.round(sql.avg / mem.avg) + '×' : '—';
|
||||
const cachedStr = cached && !cached.failed ? fmt(cached.avg) : '—';
|
||||
|
||||
console.log(
|
||||
`${sql.name.padEnd(24)} ${fmt(sql.avg).padStart(9)} ${fmt(mem.avg).padStart(9)} ${cachedStr.padStart(9)} ${speedup.padStart(9)} ${fmtSize(sql.bytes).padStart(10)} ${fmtSize(mem.bytes).padStart(10)}`
|
||||
);
|
||||
}
|
||||
|
||||
// Summary
|
||||
const sqlTotal = sqlResults.filter(r => !r.failed).reduce((s, r) => s + r.avg, 0);
|
||||
const memTotal = memResults.filter(r => !r.failed).reduce((s, r) => s + r.avg, 0);
|
||||
console.log(`${'─'.repeat(24)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(9)}`);
|
||||
console.log(`${'TOTAL'.padEnd(24)} ${fmt(sqlTotal).padStart(9)} ${fmt(memTotal).padStart(9)} ${''.padStart(9)} ${(Math.round(sqlTotal/memTotal)+'×').padStart(9)}`);
|
||||
console.log(`\n${'═'.repeat(W)}\n`);
|
||||
}
|
||||
|
||||
run().catch(e => { console.error(e); process.exit(1); });
|
||||
|
||||
@@ -13,6 +13,9 @@ class PacketStore {
|
||||
this.estPacketBytes = config.estimatedPacketBytes || 450;
|
||||
this.maxPackets = Math.floor(this.maxBytes / this.estPacketBytes);
|
||||
|
||||
// SQLite-only mode: skip RAM loading, all reads go to DB
|
||||
this.sqliteOnly = process.env.NO_MEMORY_STORE === '1';
|
||||
|
||||
// Core storage: array sorted by timestamp DESC (newest first)
|
||||
this.packets = [];
|
||||
// Indexes
|
||||
@@ -27,6 +30,11 @@ class PacketStore {
|
||||
|
||||
/** Load all packets from SQLite into memory */
|
||||
load() {
|
||||
if (this.sqliteOnly) {
|
||||
console.log('[PacketStore] SQLite-only mode (NO_MEMORY_STORE=1) — all reads go to database');
|
||||
this.loaded = true;
|
||||
return this;
|
||||
}
|
||||
const t0 = Date.now();
|
||||
const rows = this.db.prepare(
|
||||
'SELECT * FROM packets ORDER BY timestamp DESC'
|
||||
@@ -112,9 +120,12 @@ class PacketStore {
|
||||
return id;
|
||||
}
|
||||
|
||||
/** Query packets with filters — all from memory */
|
||||
/** Query packets with filters — all from memory (or SQLite in fallback mode) */
|
||||
query({ limit = 50, offset = 0, type, route, region, observer, hash, since, until, node, order = 'DESC' } = {}) {
|
||||
this.stats.queries++;
|
||||
|
||||
if (this.sqliteOnly) return this._querySQLite({ limit, offset, type, route, region, observer, hash, since, until, node, order });
|
||||
|
||||
let results = this.packets;
|
||||
|
||||
// Use indexes for single-key filters when possible
|
||||
@@ -183,6 +194,8 @@ class PacketStore {
|
||||
queryGrouped({ limit = 50, offset = 0, type, route, region, observer, hash, since, until, node } = {}) {
|
||||
this.stats.queries++;
|
||||
|
||||
if (this.sqliteOnly) return this._queryGroupedSQLite({ limit, offset, type, route, region, observer, hash, since, until, node });
|
||||
|
||||
// Get filtered results first
|
||||
const { packets: filtered, total: filteredTotal } = this.query({
|
||||
limit: 999999, offset: 0, type, route, region, observer, hash, since, until, node
|
||||
@@ -231,37 +244,49 @@ class PacketStore {
|
||||
|
||||
/** Get timestamps for sparkline */
|
||||
getTimestamps(since) {
|
||||
if (this.sqliteOnly) {
|
||||
return this.db.prepare('SELECT timestamp FROM packets WHERE timestamp > ? ORDER BY timestamp ASC').all(since).map(r => r.timestamp);
|
||||
}
|
||||
const results = [];
|
||||
for (const p of this.packets) {
|
||||
if (p.timestamp <= since) break; // sorted DESC, so we can stop early
|
||||
if (p.timestamp <= since) break;
|
||||
results.push(p.timestamp);
|
||||
}
|
||||
return results.reverse(); // return ASC
|
||||
return results.reverse();
|
||||
}
|
||||
|
||||
/** Get a single packet by ID */
|
||||
getById(id) {
|
||||
if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets WHERE id = ?').get(id) || null;
|
||||
return this.byId.get(id) || null;
|
||||
}
|
||||
|
||||
/** Get all siblings of a packet (same hash) */
|
||||
getSiblings(hash) {
|
||||
if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets WHERE hash = ? ORDER BY timestamp DESC').all(hash);
|
||||
return this.byHash.get(hash) || [];
|
||||
}
|
||||
|
||||
/** Get all packets (raw array reference — do not mutate) */
|
||||
all() { return this.packets; }
|
||||
all() {
|
||||
if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets ORDER BY timestamp DESC').all();
|
||||
return this.packets;
|
||||
}
|
||||
|
||||
/** Get all packets matching a filter function */
|
||||
filter(fn) { return this.packets.filter(fn); }
|
||||
filter(fn) {
|
||||
if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets ORDER BY timestamp DESC').all().filter(fn);
|
||||
return this.packets.filter(fn);
|
||||
}
|
||||
|
||||
/** Memory stats */
|
||||
getStats() {
|
||||
return {
|
||||
...this.stats,
|
||||
inMemory: this.packets.length,
|
||||
inMemory: this.sqliteOnly ? 0 : this.packets.length,
|
||||
sqliteOnly: this.sqliteOnly,
|
||||
maxPackets: this.maxPackets,
|
||||
estimatedMB: Math.round(this.packets.length * this.estPacketBytes / 1024 / 1024),
|
||||
estimatedMB: this.sqliteOnly ? 0 : Math.round(this.packets.length * this.estPacketBytes / 1024 / 1024),
|
||||
maxMB: Math.round(this.maxBytes / 1024 / 1024),
|
||||
indexes: {
|
||||
byHash: this.byHash.size,
|
||||
@@ -270,6 +295,48 @@ class PacketStore {
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** SQLite fallback: query with filters */
|
||||
_querySQLite({ limit, offset, type, route, region, observer, hash, since, until, node, order }) {
|
||||
const where = []; const params = [];
|
||||
if (type !== undefined) { where.push('payload_type = ?'); params.push(Number(type)); }
|
||||
if (route !== undefined) { where.push('route_type = ?'); params.push(Number(route)); }
|
||||
if (observer) { where.push('observer_id = ?'); params.push(observer); }
|
||||
if (hash) { where.push('hash = ?'); params.push(hash); }
|
||||
if (since) { where.push('timestamp > ?'); params.push(since); }
|
||||
if (until) { where.push('timestamp < ?'); params.push(until); }
|
||||
if (region) { where.push('observer_id IN (SELECT id FROM observers WHERE iata = ?)'); params.push(region); }
|
||||
if (node) { where.push('decoded_json LIKE ?'); params.push(`%${node}%`); }
|
||||
const w = where.length ? 'WHERE ' + where.join(' AND ') : '';
|
||||
const total = this.db.prepare(`SELECT COUNT(*) as c FROM packets ${w}`).get(...params).c;
|
||||
const packets = this.db.prepare(`SELECT * FROM packets ${w} ORDER BY timestamp ${order === 'ASC' ? 'ASC' : 'DESC'} LIMIT ? OFFSET ?`).all(...params, limit, offset);
|
||||
return { packets, total };
|
||||
}
|
||||
|
||||
/** SQLite fallback: grouped query */
|
||||
_queryGroupedSQLite({ limit, offset, type, route, region, observer, hash, since, until, node }) {
|
||||
const where = []; const params = [];
|
||||
if (type !== undefined) { where.push('payload_type = ?'); params.push(Number(type)); }
|
||||
if (route !== undefined) { where.push('route_type = ?'); params.push(Number(route)); }
|
||||
if (observer) { where.push('observer_id = ?'); params.push(observer); }
|
||||
if (hash) { where.push('hash = ?'); params.push(hash); }
|
||||
if (since) { where.push('timestamp > ?'); params.push(since); }
|
||||
if (until) { where.push('timestamp < ?'); params.push(until); }
|
||||
if (region) { where.push('observer_id IN (SELECT id FROM observers WHERE iata = ?)'); params.push(region); }
|
||||
if (node) { where.push('decoded_json LIKE ?'); params.push(`%${node}%`); }
|
||||
const w = where.length ? 'WHERE ' + where.join(' AND ') : '';
|
||||
|
||||
const sql = `SELECT hash, COUNT(*) as count, COUNT(DISTINCT observer_id) as observer_count,
|
||||
MAX(timestamp) as latest, MIN(observer_id) as observer_id, MIN(observer_name) as observer_name,
|
||||
MIN(path_json) as path_json, MIN(payload_type) as payload_type, MIN(raw_hex) as raw_hex,
|
||||
MIN(decoded_json) as decoded_json
|
||||
FROM packets ${w} GROUP BY hash ORDER BY latest DESC LIMIT ? OFFSET ?`;
|
||||
const packets = this.db.prepare(sql).all(...params, limit, offset);
|
||||
|
||||
const countSql = `SELECT COUNT(DISTINCT hash) as c FROM packets ${w}`;
|
||||
const total = this.db.prepare(countSql).get(...params).c;
|
||||
return { packets, total };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = PacketStore;
|
||||
|
||||
Reference in New Issue
Block a user