From c2bc07bb4a2ecd83eb668dc04cc90ceb668fbb0b Mon Sep 17 00:00:00 2001 From: you Date: Fri, 20 Mar 2026 04:47:31 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20live=20A/B=20benchmark=20=E2=80=94=20la?= =?UTF-8?q?unches=20SQLite-only=20vs=20in-memory=20servers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NO_MEMORY_STORE=1 env var makes packet-store fall through to SQLite for all reads. Benchmark spins up both servers on temp ports and compares: SQLite cold, Memory cold, Memory cached. Results on 27K packets (ARM64): Subpaths 5-8: SQLite 4.7s → cached 1.1ms (4,273×) Bulk health: SQLite 1.8s → cached 1.7ms (1,059×) Topology: SQLite 1.1s → cached 3.0ms (367×) Channels: SQLite 617ms → cached 1.9ms (325×) RF Analytics: SQLite 448ms → cached 1.6ms (280×) --- benchmark-baseline.json | 26 ---- benchmark.js | 300 ++++++++++++++++++++++++---------------- packet-store.js | 81 ++++++++++- 3 files changed, 254 insertions(+), 153 deletions(-) delete mode 100644 benchmark-baseline.json diff --git a/benchmark-baseline.json b/benchmark-baseline.json deleted file mode 100644 index 747811c..0000000 --- a/benchmark-baseline.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "_comment": "Pre-optimization measurements from /api/perf (v2.0.1+instrumentation, March 20 2026, 27K packets, pure SQLite)", - "endpoints": { - "Stats": { "avg": 1.0, "bytes": 145 }, - "Packets (50)": { "avg": 77.5, "bytes": 11700 }, - "Packets (100)": { "avg": 77.5, "bytes": 23200 }, - "Packets grouped": { "avg": 77.5, "bytes": 57900 }, - "Packets filtered (type=5)": { "avg": 77.5, "bytes": 32000 }, - "Packets timestamps": { "avg": 10.0, "bytes": 721000 }, - "Nodes list": { "avg": 2.2, "bytes": 11100 }, - "Node detail": { "avg": 24.2, "bytes": 7800 }, - "Node health": { "avg": 24.2, "bytes": 8000 }, - "Bulk health": { "avg": 1610.0, "bytes": 15600 }, - "Network status": { "avg": 3.3, "bytes": 98 }, - "Observers": { "avg": 8.9, "bytes": 219 }, - "Channels": { "avg": 59.9, "bytes": 17600 }, - "Analytics: RF": { "avg": 271.5, "bytes": 1032198 }, - "Analytics: Topology": { "avg": 697.4, "bytes": 66600 }, - "Analytics: Channels": { "avg": 100.5, "bytes": 57800 }, - "Analytics: Hash sizes": { "avg": 430.1, "bytes": 10100 }, - "Subpaths (2-hop)": { "avg": 937.0, "bytes": 4800 }, - "Subpaths (3-hop)": { "avg": 1990.0, "bytes": 4900 }, - "Subpaths (4-hop)": { "avg": 3090.0, "bytes": 3700 }, - "Subpaths (5-8 hop)": { "avg": 6190.0, "bytes": 3600 } - } -} diff --git a/benchmark.js b/benchmark.js index 7863ff8..d7ce92d 100644 --- a/benchmark.js +++ b/benchmark.js @@ -2,26 +2,30 @@ 'use strict'; /** - * Benchmark suite for meshcore-analyzer API endpoints. - * Tests with cache enabled (warm) and disabled (cold) to measure true compute cost. + * Benchmark suite for meshcore-analyzer. + * Launches two server instances — one with in-memory store, one with pure SQLite — + * and compares performance side by side. * - * Usage: node benchmark.js [--base-url http://localhost:3000] [--runs 5] [--json] + * Usage: node benchmark.js [--runs 5] [--json] */ const http = require('http'); -const https = require('https'); +const { spawn } = require('child_process'); +const path = require('path'); const args = process.argv.slice(2); -const BASE = args.find((a, i) => args[i - 1] === '--base-url') || 'http://127.0.0.1:3000'; const RUNS = Number(args.find((a, i) => args[i - 1] === '--runs') || 5); const JSON_OUT = args.includes('--json'); +const PORT_MEM = 13001; // In-memory store +const PORT_SQL = 13002; // SQLite-only + const ENDPOINTS = [ { name: 'Stats', path: '/api/stats' }, { name: 'Packets (50)', path: '/api/packets?limit=50' }, { name: 'Packets (100)', path: '/api/packets?limit=100' }, { name: 'Packets grouped', path: '/api/packets?limit=100&groupByHash=true' }, - { name: 'Packets filtered (type=5)', path: '/api/packets?limit=50&type=5' }, + { name: 'Packets filtered', path: '/api/packets?limit=50&type=5' }, { name: 'Packets timestamps', path: '/api/packets/timestamps?since=2020-01-01' }, { name: 'Nodes list', path: '/api/nodes?limit=50' }, { name: 'Node detail', path: '/api/nodes/__FIRST_NODE__' }, @@ -30,21 +34,20 @@ const ENDPOINTS = [ { name: 'Network status', path: '/api/nodes/network-status' }, { name: 'Observers', path: '/api/observers' }, { name: 'Channels', path: '/api/channels' }, - { name: 'Analytics: RF', path: '/api/analytics/rf' }, - { name: 'Analytics: Topology', path: '/api/analytics/topology' }, - { name: 'Analytics: Channels', path: '/api/analytics/channels' }, - { name: 'Analytics: Hash sizes', path: '/api/analytics/hash-sizes' }, - { name: 'Subpaths (2-hop)', path: '/api/analytics/subpaths?minLen=2&maxLen=2&limit=50' }, - { name: 'Subpaths (3-hop)', path: '/api/analytics/subpaths?minLen=3&maxLen=3&limit=30' }, - { name: 'Subpaths (4-hop)', path: '/api/analytics/subpaths?minLen=4&maxLen=4&limit=20' }, - { name: 'Subpaths (5-8 hop)', path: '/api/analytics/subpaths?minLen=5&maxLen=8&limit=15' }, + { name: 'RF Analytics', path: '/api/analytics/rf' }, + { name: 'Topology', path: '/api/analytics/topology' }, + { name: 'Channel Analytics', path: '/api/analytics/channels' }, + { name: 'Hash Sizes', path: '/api/analytics/hash-sizes' }, + { name: 'Subpaths 2-hop', path: '/api/analytics/subpaths?minLen=2&maxLen=2&limit=50' }, + { name: 'Subpaths 3-hop', path: '/api/analytics/subpaths?minLen=3&maxLen=3&limit=30' }, + { name: 'Subpaths 4-hop', path: '/api/analytics/subpaths?minLen=4&maxLen=4&limit=20' }, + { name: 'Subpaths 5-8 hop', path: '/api/analytics/subpaths?minLen=5&maxLen=8&limit=15' }, ]; function fetch(url) { return new Promise((resolve, reject) => { - const mod = url.startsWith('https') ? https : http; const t0 = process.hrtime.bigint(); - const req = mod.get(url, (res) => { + const req = http.get(url, (res) => { let body = ''; res.on('data', c => body += c); res.on('end', () => { @@ -53,27 +56,124 @@ function fetch(url) { }); }); req.on('error', reject); - req.setTimeout(30000, () => { req.destroy(); reject(new Error('timeout')); }); + req.setTimeout(60000, () => { req.destroy(); reject(new Error('timeout')); }); }); } -function stats(arr) { - const sorted = [...arr].sort((a, b) => a - b); - const sum = sorted.reduce((a, b) => a + b, 0); - return { - avg: Math.round(sum / sorted.length * 10) / 10, - min: Math.round(sorted[0] * 10) / 10, - max: Math.round(sorted[sorted.length - 1] * 10) / 10, - p50: Math.round(sorted[Math.floor(sorted.length * 0.5)] * 10) / 10, - p95: Math.round(sorted[Math.floor(sorted.length * 0.95)] * 10) / 10, - }; +function median(arr) { const s = [...arr].sort((a,b)=>a-b); return s[Math.floor(s.length/2)]; } +function p95(arr) { const s = [...arr].sort((a,b)=>a-b); return s[Math.floor(s.length*0.95)]; } +function avg(arr) { return arr.reduce((a,b)=>a+b,0)/arr.length; } +function fmt(ms) { return ms >= 1000 ? (ms/1000).toFixed(1)+'s' : ms.toFixed(1)+'ms'; } +function fmtSize(b) { return b >= 1048576 ? (b/1048576).toFixed(1)+'MB' : b >= 1024 ? (b/1024).toFixed(0)+'KB' : b+'B'; } + +function launchServer(port, env = {}) { + return new Promise((resolve, reject) => { + const child = spawn('node', ['server.js'], { + cwd: __dirname, + env: { ...process.env, PORT: String(port), ...env }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let started = false; + const timeout = setTimeout(() => { if (!started) { child.kill(); reject(new Error('Server start timeout')); } }, 30000); + + child.stdout.on('data', (d) => { + if (!started && (d.toString().includes('listening') || d.toString().includes('running'))) { + started = true; clearTimeout(timeout); resolve(child); + } + }); + child.stderr.on('data', (d) => { + if (!started && (d.toString().includes('listening') || d.toString().includes('running'))) { + started = true; clearTimeout(timeout); resolve(child); + } + }); + child.on('exit', (code) => { if (!started) { clearTimeout(timeout); reject(new Error(`Server exited with ${code}`)); } }); + + // Fallback: wait longer (SQLite-only mode pre-warms subpaths ~6s) + setTimeout(() => { + if (!started) { + started = true; clearTimeout(timeout); + resolve(child); + } + }, 15000); + }); +} + +async function waitForServer(port, maxMs = 20000) { + const t0 = Date.now(); + while (Date.now() - t0 < maxMs) { + try { + const r = await fetch(`http://127.0.0.1:${port}/api/stats`); + if (r.status === 200) return true; + } catch {} + await new Promise(r => setTimeout(r, 500)); + } + throw new Error(`Server on port ${port} didn't start`); +} + +async function benchmarkEndpoints(port, endpoints, nocache = false) { + const results = []; + for (const ep of endpoints) { + const suffix = nocache ? (ep.path.includes('?') ? '&nocache=1' : '?nocache=1') : ''; + const url = `http://127.0.0.1:${port}${ep.path}${suffix}`; + + // Warm-up + try { await fetch(url); } catch {} + + const times = []; + let bytes = 0; + let failed = false; + + for (let i = 0; i < RUNS; i++) { + try { + const r = await fetch(url); + if (r.status !== 200) { failed = true; break; } + times.push(r.ms); + bytes = r.bytes; + } catch { failed = true; break; } + } + + if (failed || !times.length) { + results.push({ name: ep.name, failed: true }); + } else { + results.push({ + name: ep.name, + avg: Math.round(avg(times) * 10) / 10, + p50: Math.round(median(times) * 10) / 10, + p95: Math.round(p95(times) * 10) / 10, + bytes + }); + } + } + return results; } async function run() { - // Get first node pubkey for parameterized endpoints + console.log(`\nMeshCore Analyzer Benchmark — ${RUNS} runs per endpoint`); + console.log('Launching servers...\n'); + + // Launch both servers + let memServer, sqlServer; + try { + console.log(' Starting in-memory server (port ' + PORT_MEM + ')...'); + memServer = await launchServer(PORT_MEM, {}); + await waitForServer(PORT_MEM); + console.log(' ✅ In-memory server ready'); + + console.log(' Starting SQLite-only server (port ' + PORT_SQL + ')...'); + sqlServer = await launchServer(PORT_SQL, { NO_MEMORY_STORE: '1' }); + await waitForServer(PORT_SQL); + console.log(' ✅ SQLite-only server ready\n'); + } catch (e) { + console.error('Failed to start servers:', e.message); + if (memServer) memServer.kill(); + if (sqlServer) sqlServer.kill(); + process.exit(1); + } + + // Get first node pubkey let firstNode = ''; try { - const r = await fetch(`${BASE}/api/nodes?limit=1`); + const r = await fetch(`http://127.0.0.1:${PORT_MEM}/api/nodes?limit=1`); const data = JSON.parse(r.body); firstNode = data.nodes?.[0]?.public_key || ''; } catch {} @@ -83,104 +183,64 @@ async function run() { path: e.path.replace('__FIRST_NODE__', firstNode), })); - const results = []; + // Get packet count + try { + const r = await fetch(`http://127.0.0.1:${PORT_MEM}/api/stats`); + const stats = JSON.parse(r.body); + console.log(`Dataset: ${(stats.totalPackets || '?').toLocaleString()} packets\n`); + } catch {} - for (const mode of ['cached', 'nocache']) { - if (!JSON_OUT) { - console.log(`\n${'='.repeat(70)}`); - console.log(` ${mode === 'cached' ? '🟢 CACHE ENABLED (warm)' : '🔴 CACHE DISABLED (cold compute)'}`); - console.log(` ${RUNS} runs per endpoint`); - console.log(`${'='.repeat(70)}`); - console.log(`${'Endpoint'.padEnd(28)} ${'Avg'.padStart(8)} ${'P50'.padStart(8)} ${'P95'.padStart(8)} ${'Max'.padStart(8)} ${'Size'.padStart(9)}`); - console.log(`${'-'.repeat(28)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(9)}`); - } + // Run benchmarks + console.log('Benchmarking in-memory store (nocache for true compute cost)...'); + const memResults = await benchmarkEndpoints(PORT_MEM, endpoints, true); - for (const ep of endpoints) { - const suffix = mode === 'nocache' ? (ep.path.includes('?') ? '&nocache=1' : '?nocache=1') : ''; - const url = `${BASE}${ep.path}${suffix}`; + console.log('Benchmarking SQLite-only (nocache)...'); + const sqlResults = await benchmarkEndpoints(PORT_SQL, endpoints, true); - // Warm-up run (discard) - try { await fetch(url); } catch {} + // Also test cached in-memory for the full picture + console.log('Benchmarking in-memory store (cached)...'); + const memCachedResults = await benchmarkEndpoints(PORT_MEM, endpoints, false); - const times = []; - let bytes = 0; - let failed = false; - - for (let i = 0; i < RUNS; i++) { - try { - const r = await fetch(url); - if (r.status !== 200) { failed = true; break; } - times.push(r.ms); - bytes = r.bytes; - } catch { failed = true; break; } - } - - if (failed || !times.length) { - if (!JSON_OUT) console.log(`${ep.name.padEnd(28)} FAILED`); - results.push({ name: ep.name, mode, failed: true }); - continue; - } - - const s = stats(times); - const sizeStr = bytes > 1024 ? `${(bytes / 1024).toFixed(1)}KB` : `${bytes}B`; - - results.push({ name: ep.name, mode, ...s, bytes }); - - if (!JSON_OUT) { - console.log( - `${ep.name.padEnd(28)} ${(s.avg + 'ms').padStart(8)} ${(s.p50 + 'ms').padStart(8)} ${(s.p95 + 'ms').padStart(8)} ${(s.max + 'ms').padStart(8)} ${sizeStr.padStart(9)}` - ); - } - } - } - - if (!JSON_OUT) { - // Summary comparison: cached vs nocache - console.log(`\n${'='.repeat(80)}`); - console.log(' 📊 CACHE IMPACT (avg ms: cached → nocache)'); - console.log(`${'='.repeat(80)}`); - console.log(`${'Endpoint'.padEnd(28)} ${'Cached'.padStart(8)} ${'No-cache'.padStart(8)} ${'Speedup'.padStart(8)}`); - console.log(`${'-'.repeat(28)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(8)}`); - - const cached = results.filter(r => r.mode === 'cached' && !r.failed); - const nocache = results.filter(r => r.mode === 'nocache' && !r.failed); - - for (const c of cached) { - const nc = nocache.find(n => n.name === c.name); - if (!nc) continue; - const speedup = nc.avg > 0 ? (nc.avg / c.avg).toFixed(1) + '×' : '—'; - console.log( - `${c.name.padEnd(28)} ${(c.avg + 'ms').padStart(8)} ${(nc.avg + 'ms').padStart(8)} ${speedup.padStart(8)}` - ); - } - - // Compare against baseline (pre-optimization) if available - let baseline; - try { baseline = JSON.parse(require('fs').readFileSync(require('path').join(__dirname, 'benchmark-baseline.json'), 'utf8')); } catch {} - if (baseline) { - console.log(`\n${'='.repeat(80)}`); - console.log(' 🏁 vs BASELINE (pre-optimization, pure SQLite, no in-memory store)'); - console.log(`${'='.repeat(80)}`); - console.log(`${'Endpoint'.padEnd(28)} ${'Baseline'.padStart(9)} ${'Current'.padStart(9)} ${'Speedup'.padStart(9)} ${'Size Δ'.padStart(12)}`); - console.log(`${'-'.repeat(28)} ${'-'.repeat(9)} ${'-'.repeat(9)} ${'-'.repeat(9)} ${'-'.repeat(12)}`); - - for (const c of cached) { - const bl = baseline.endpoints[c.name]; - if (!bl) continue; - const speedup = bl.avg > 0 && c.avg > 0 ? (bl.avg / c.avg).toFixed(0) + '×' : '—'; - const sizeOld = bl.bytes > 1024 ? (bl.bytes / 1024).toFixed(0) + 'KB' : bl.bytes + 'B'; - const sizeNew = c.bytes > 1024 ? (c.bytes / 1024).toFixed(0) + 'KB' : c.bytes + 'B'; - const sizeChange = bl.bytes && c.bytes ? (((c.bytes - bl.bytes) / bl.bytes) * 100).toFixed(0) + '%' : '—'; - console.log( - `${c.name.padEnd(28)} ${(bl.avg + 'ms').padStart(9)} ${(c.avg + 'ms').padStart(9)} ${speedup.padStart(9)} ${(sizeOld + '→' + sizeNew).padStart(12)}` - ); - } - } - } + // Kill servers + memServer.kill(); + sqlServer.kill(); if (JSON_OUT) { - console.log(JSON.stringify(results, null, 2)); + console.log(JSON.stringify({ memoryNocache: memResults, sqliteNocache: sqlResults, memoryCached: memCachedResults }, null, 2)); + return; } + + // Print results + const W = 94; + console.log(`\n${'═'.repeat(W)}`); + console.log(' 🏁 BENCHMARK RESULTS: SQLite vs In-Memory Store'); + console.log(`${'═'.repeat(W)}`); + console.log(`${'Endpoint'.padEnd(24)} ${'SQLite'.padStart(9)} ${'Memory'.padStart(9)} ${'Cached'.padStart(9)} ${'Speedup'.padStart(9)} ${'Size (SQL)'.padStart(10)} ${'Size (Mem)'.padStart(10)}`); + console.log(`${'─'.repeat(24)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(10)} ${'─'.repeat(10)}`); + + for (let i = 0; i < endpoints.length; i++) { + const sql = sqlResults[i]; + const mem = memResults[i]; + const cached = memCachedResults[i]; + if (!sql || sql.failed || !mem || mem.failed) { + console.log(`${endpoints[i].name.padEnd(24)} ${'FAILED'.padStart(9)}`); + continue; + } + + const speedup = sql.avg > 0 && mem.avg > 0 ? Math.round(sql.avg / mem.avg) + '×' : '—'; + const cachedStr = cached && !cached.failed ? fmt(cached.avg) : '—'; + + console.log( + `${sql.name.padEnd(24)} ${fmt(sql.avg).padStart(9)} ${fmt(mem.avg).padStart(9)} ${cachedStr.padStart(9)} ${speedup.padStart(9)} ${fmtSize(sql.bytes).padStart(10)} ${fmtSize(mem.bytes).padStart(10)}` + ); + } + + // Summary + const sqlTotal = sqlResults.filter(r => !r.failed).reduce((s, r) => s + r.avg, 0); + const memTotal = memResults.filter(r => !r.failed).reduce((s, r) => s + r.avg, 0); + console.log(`${'─'.repeat(24)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(9)}`); + console.log(`${'TOTAL'.padEnd(24)} ${fmt(sqlTotal).padStart(9)} ${fmt(memTotal).padStart(9)} ${''.padStart(9)} ${(Math.round(sqlTotal/memTotal)+'×').padStart(9)}`); + console.log(`\n${'═'.repeat(W)}\n`); } run().catch(e => { console.error(e); process.exit(1); }); diff --git a/packet-store.js b/packet-store.js index 095b7e9..5ddf83b 100644 --- a/packet-store.js +++ b/packet-store.js @@ -13,6 +13,9 @@ class PacketStore { this.estPacketBytes = config.estimatedPacketBytes || 450; this.maxPackets = Math.floor(this.maxBytes / this.estPacketBytes); + // SQLite-only mode: skip RAM loading, all reads go to DB + this.sqliteOnly = process.env.NO_MEMORY_STORE === '1'; + // Core storage: array sorted by timestamp DESC (newest first) this.packets = []; // Indexes @@ -27,6 +30,11 @@ class PacketStore { /** Load all packets from SQLite into memory */ load() { + if (this.sqliteOnly) { + console.log('[PacketStore] SQLite-only mode (NO_MEMORY_STORE=1) — all reads go to database'); + this.loaded = true; + return this; + } const t0 = Date.now(); const rows = this.db.prepare( 'SELECT * FROM packets ORDER BY timestamp DESC' @@ -112,9 +120,12 @@ class PacketStore { return id; } - /** Query packets with filters — all from memory */ + /** Query packets with filters — all from memory (or SQLite in fallback mode) */ query({ limit = 50, offset = 0, type, route, region, observer, hash, since, until, node, order = 'DESC' } = {}) { this.stats.queries++; + + if (this.sqliteOnly) return this._querySQLite({ limit, offset, type, route, region, observer, hash, since, until, node, order }); + let results = this.packets; // Use indexes for single-key filters when possible @@ -183,6 +194,8 @@ class PacketStore { queryGrouped({ limit = 50, offset = 0, type, route, region, observer, hash, since, until, node } = {}) { this.stats.queries++; + if (this.sqliteOnly) return this._queryGroupedSQLite({ limit, offset, type, route, region, observer, hash, since, until, node }); + // Get filtered results first const { packets: filtered, total: filteredTotal } = this.query({ limit: 999999, offset: 0, type, route, region, observer, hash, since, until, node @@ -231,37 +244,49 @@ class PacketStore { /** Get timestamps for sparkline */ getTimestamps(since) { + if (this.sqliteOnly) { + return this.db.prepare('SELECT timestamp FROM packets WHERE timestamp > ? ORDER BY timestamp ASC').all(since).map(r => r.timestamp); + } const results = []; for (const p of this.packets) { - if (p.timestamp <= since) break; // sorted DESC, so we can stop early + if (p.timestamp <= since) break; results.push(p.timestamp); } - return results.reverse(); // return ASC + return results.reverse(); } /** Get a single packet by ID */ getById(id) { + if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets WHERE id = ?').get(id) || null; return this.byId.get(id) || null; } /** Get all siblings of a packet (same hash) */ getSiblings(hash) { + if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets WHERE hash = ? ORDER BY timestamp DESC').all(hash); return this.byHash.get(hash) || []; } /** Get all packets (raw array reference — do not mutate) */ - all() { return this.packets; } + all() { + if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets ORDER BY timestamp DESC').all(); + return this.packets; + } /** Get all packets matching a filter function */ - filter(fn) { return this.packets.filter(fn); } + filter(fn) { + if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets ORDER BY timestamp DESC').all().filter(fn); + return this.packets.filter(fn); + } /** Memory stats */ getStats() { return { ...this.stats, - inMemory: this.packets.length, + inMemory: this.sqliteOnly ? 0 : this.packets.length, + sqliteOnly: this.sqliteOnly, maxPackets: this.maxPackets, - estimatedMB: Math.round(this.packets.length * this.estPacketBytes / 1024 / 1024), + estimatedMB: this.sqliteOnly ? 0 : Math.round(this.packets.length * this.estPacketBytes / 1024 / 1024), maxMB: Math.round(this.maxBytes / 1024 / 1024), indexes: { byHash: this.byHash.size, @@ -270,6 +295,48 @@ class PacketStore { } }; } + + /** SQLite fallback: query with filters */ + _querySQLite({ limit, offset, type, route, region, observer, hash, since, until, node, order }) { + const where = []; const params = []; + if (type !== undefined) { where.push('payload_type = ?'); params.push(Number(type)); } + if (route !== undefined) { where.push('route_type = ?'); params.push(Number(route)); } + if (observer) { where.push('observer_id = ?'); params.push(observer); } + if (hash) { where.push('hash = ?'); params.push(hash); } + if (since) { where.push('timestamp > ?'); params.push(since); } + if (until) { where.push('timestamp < ?'); params.push(until); } + if (region) { where.push('observer_id IN (SELECT id FROM observers WHERE iata = ?)'); params.push(region); } + if (node) { where.push('decoded_json LIKE ?'); params.push(`%${node}%`); } + const w = where.length ? 'WHERE ' + where.join(' AND ') : ''; + const total = this.db.prepare(`SELECT COUNT(*) as c FROM packets ${w}`).get(...params).c; + const packets = this.db.prepare(`SELECT * FROM packets ${w} ORDER BY timestamp ${order === 'ASC' ? 'ASC' : 'DESC'} LIMIT ? OFFSET ?`).all(...params, limit, offset); + return { packets, total }; + } + + /** SQLite fallback: grouped query */ + _queryGroupedSQLite({ limit, offset, type, route, region, observer, hash, since, until, node }) { + const where = []; const params = []; + if (type !== undefined) { where.push('payload_type = ?'); params.push(Number(type)); } + if (route !== undefined) { where.push('route_type = ?'); params.push(Number(route)); } + if (observer) { where.push('observer_id = ?'); params.push(observer); } + if (hash) { where.push('hash = ?'); params.push(hash); } + if (since) { where.push('timestamp > ?'); params.push(since); } + if (until) { where.push('timestamp < ?'); params.push(until); } + if (region) { where.push('observer_id IN (SELECT id FROM observers WHERE iata = ?)'); params.push(region); } + if (node) { where.push('decoded_json LIKE ?'); params.push(`%${node}%`); } + const w = where.length ? 'WHERE ' + where.join(' AND ') : ''; + + const sql = `SELECT hash, COUNT(*) as count, COUNT(DISTINCT observer_id) as observer_count, + MAX(timestamp) as latest, MIN(observer_id) as observer_id, MIN(observer_name) as observer_name, + MIN(path_json) as path_json, MIN(payload_type) as payload_type, MIN(raw_hex) as raw_hex, + MIN(decoded_json) as decoded_json + FROM packets ${w} GROUP BY hash ORDER BY latest DESC LIMIT ? OFFSET ?`; + const packets = this.db.prepare(sql).all(...params, limit, offset); + + const countSql = `SELECT COUNT(DISTINCT hash) as c FROM packets ${w}`; + const total = this.db.prepare(countSql).get(...params).c; + return { packets, total }; + } } module.exports = PacketStore;