From 1be6b4f4adcf05079bf092d9ebe7f0d873aa5163 Mon Sep 17 00:00:00 2001 From: you Date: Fri, 20 Mar 2026 03:43:23 +0000 Subject: [PATCH] =?UTF-8?q?perf:=20ALL=20packet=20reads=20from=20RAM=20?= =?UTF-8?q?=E2=80=94=20analytics,=20channels,=20topology,=20subpaths,=20RF?= =?UTF-8?q?,=20observers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zero SQLite reads from packets table. Every endpoint that previously scanned packets now reads from the in-memory PacketStore. Expected: subpaths from 1.6s to <100ms, topology from 700ms to <50ms, RF from 270ms to <30ms on cold calls. --- packet-store.js | 6 ++++++ server.js | 23 ++++++++++++----------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/packet-store.js b/packet-store.js index edc82c54..095b7e9d 100644 --- a/packet-store.js +++ b/packet-store.js @@ -249,6 +249,12 @@ class PacketStore { return this.byHash.get(hash) || []; } + /** Get all packets (raw array reference — do not mutate) */ + all() { return this.packets; } + + /** Get all packets matching a filter function */ + filter(fn) { return this.packets.filter(fn); } + /** Memory stats */ getStats() { return { diff --git a/server.js b/server.js index d4387167..883692bd 100644 --- a/server.js +++ b/server.js @@ -815,7 +815,7 @@ app.get('/api/nodes/:pubkey', (req, res) => { app.get('/api/analytics/rf', (req, res) => { const _c = cache.get('analytics:rf'); if (_c) return res.json(_c); const PTYPES = { 0:'REQ',1:'RESPONSE',2:'TXT_MSG',3:'ACK',4:'ADVERT',5:'GRP_TXT',7:'ANON_REQ',8:'PATH',9:'TRACE',11:'CONTROL' }; - const packets = db.db.prepare(`SELECT snr, rssi, payload_type, timestamp, raw_hex FROM packets WHERE snr IS NOT NULL`).all(); + const packets = pktStore.filter(p => p.snr != null); const snrVals = packets.map(p => p.snr).filter(v => v != null); const rssiVals = packets.map(p => p.rssi).filter(v => v != null); @@ -891,7 +891,7 @@ app.get('/api/analytics/rf', (req, res) => { // --- Topology Analytics --- app.get('/api/analytics/topology', (req, res) => { const _c = cache.get('analytics:topology'); if (_c) return res.json(_c); - const packets = db.db.prepare(`SELECT path_json, snr, decoded_json, observer_id FROM packets WHERE path_json IS NOT NULL AND path_json != '[]'`).all(); + const packets = pktStore.filter(p => p.path_json && p.path_json !== '[]'); const allNodes = db.db.prepare('SELECT public_key, name, lat, lon FROM nodes WHERE name IS NOT NULL').all(); const resolveHop = (hop, contextPositions) => { const h = hop.toLowerCase(); @@ -969,7 +969,7 @@ app.get('/api/analytics/topology', (req, res) => { .sort((a, b) => a.hops - b.hops); // Reachability: per-observer hop distances + cross-observer comparison + best path - const observers = db.db.prepare(`SELECT DISTINCT observer_id, observer_name FROM packets WHERE path_json IS NOT NULL AND path_json != '[]'`).all(); + const observerMap = new Map(); pktStore.filter(p => p.path_json && p.path_json !== '[]' && p.observer_id).forEach(p => observerMap.set(p.observer_id, p.observer_name)); const observers = [...observerMap].map(([observer_id, observer_name]) => ({ observer_id, observer_name })); // Per-observer: node → min hop distance seen from that observer const perObserver = {}; // observer_id → { hop_hex → { minDist, maxDist, count } } @@ -1057,7 +1057,7 @@ app.get('/api/analytics/topology', (req, res) => { // --- Channel Analytics --- app.get('/api/analytics/channels', (req, res) => { const _c = cache.get('analytics:channels'); if (_c) return res.json(_c); - const packets = db.db.prepare(`SELECT decoded_json, timestamp FROM packets WHERE payload_type = 5 AND decoded_json IS NOT NULL`).all(); + const packets = pktStore.filter(p => p.payload_type === 5 && p.decoded_json); const channels = {}; const senderCounts = {}; @@ -1357,7 +1357,7 @@ const channelHashNames = {}; app.get('/api/channels', (req, res) => { const _c = cache.get('channels'); if (_c) return res.json(_c); - const packets = db.db.prepare(`SELECT * FROM packets WHERE payload_type = 5 ORDER BY timestamp DESC`).all(); + const packets = pktStore.filter(p => p.payload_type === 5).sort((a,b) => b.timestamp > a.timestamp ? 1 : -1); const channelMap = {}; for (const pkt of packets) { @@ -1395,7 +1395,7 @@ app.get('/api/channels', (req, res) => { } // Also include companion bridge messages (no raw_hex, have text directly) - const companionPkts = db.db.prepare(`SELECT * FROM packets WHERE payload_type = 5 AND raw_hex IS NULL ORDER BY timestamp DESC`).all(); + const companionPkts = pktStore.filter(p => p.payload_type === 5 && !p.raw_hex).sort((a,b) => b.timestamp > a.timestamp ? 1 : -1); for (const pkt of companionPkts) { let decoded; try { decoded = JSON.parse(pkt.decoded_json); } catch { continue; } @@ -1425,7 +1425,7 @@ app.get('/api/channels/:hash/messages', (req, res) => { const _c = cache.get(_ck); if (_c) return res.json(_c); const { limit = 100, offset = 0 } = req.query; const channelHash = req.params.hash; - const packets = db.db.prepare(`SELECT * FROM packets WHERE payload_type = 5 ORDER BY timestamp ASC`).all(); + const packets = pktStore.filter(p => p.payload_type === 5).sort((a,b) => a.timestamp > b.timestamp ? 1 : -1); // Group by message content + timestamp to deduplicate repeats const msgMap = new Map(); @@ -1492,7 +1492,8 @@ app.get('/api/observers', (req, res) => { const observers = db.getObservers(); const oneHourAgo = new Date(Date.now() - 3600000).toISOString(); const result = observers.map(o => { - const lastHour = db.db.prepare(`SELECT COUNT(*) as count FROM packets WHERE observer_id = ? AND timestamp > ?`).get(o.id, oneHourAgo); + const obsPackets = pktStore.byObserver.get(o.id) || []; + const lastHour = { count: obsPackets.filter(p => p.timestamp > oneHourAgo).length }; return { ...o, packetsLastHour: lastHour.count }; }); const _oResult = { observers: result, server_time: new Date().toISOString() }; @@ -1501,7 +1502,7 @@ app.get('/api/observers', (req, res) => { }); app.get('/api/traces/:hash', (req, res) => { - const packets = db.db.prepare(`SELECT observer_id, timestamp, snr, rssi FROM packets WHERE hash = ? ORDER BY timestamp`).all(req.params.hash); + const packets = (pktStore.getSiblings(req.params.hash) || []).sort((a,b) => a.timestamp > b.timestamp ? 1 : -1); const traces = packets.map(p => ({ observer: p.observer_id, time: p.timestamp, snr: p.snr, rssi: p.rssi })); res.json({ traces }); }); @@ -1528,7 +1529,7 @@ app.get('/api/analytics/subpaths', (req, res) => { const _c = cache.get(_ck); if (_c) return res.json(_c); const minLen = Math.max(2, Number(req.query.minLen) || 2); const maxLen = Number(req.query.maxLen) || 8; - const packets = db.db.prepare(`SELECT path_json FROM packets WHERE path_json IS NOT NULL AND path_json != '[]'`).all(); + const packets = pktStore.filter(p => p.path_json && p.path_json !== '[]'); const allNodes = db.db.prepare('SELECT public_key, name, lat, lon FROM nodes WHERE name IS NOT NULL').all(); // Disambiguate per path with caching (same hop sequence = same result) @@ -1589,7 +1590,7 @@ app.get('/api/analytics/subpath-detail', (req, res) => { const rawHops = (req.query.hops || '').split(',').filter(Boolean); if (rawHops.length < 2) return res.json({ error: 'Need at least 2 hops' }); - const packets = db.db.prepare(`SELECT path_json, snr, rssi, timestamp, decoded_json, observer_name FROM packets WHERE path_json IS NOT NULL AND path_json != '[]'`).all(); + const packets = pktStore.filter(p => p.path_json && p.path_json !== '[]'); const allNodes = db.db.prepare('SELECT public_key, name, lat, lon FROM nodes WHERE name IS NOT NULL').all(); // Disambiguate the requested hops