diff --git a/db.js b/db.js index 66a1d13f..0ba06128 100644 --- a/db.js +++ b/db.js @@ -12,27 +12,19 @@ db.pragma('journal_mode = WAL'); db.pragma('foreign_keys = ON'); db.pragma('wal_autocheckpoint = 0'); // Disable auto-checkpoint — manual checkpoint on timer to avoid random event loop spikes +// --- Migration: drop legacy tables (replaced by transmissions + observations in v2.3.0) --- +// Drop paths first (has FK to packets) +const legacyTables = ['paths', 'packets']; +for (const t of legacyTables) { + const exists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(t); + if (exists) { + console.log(`[migration] Dropping legacy table: ${t}`); + db.exec(`DROP TABLE IF EXISTS ${t}`); + } +} + // --- Schema --- db.exec(` - CREATE TABLE IF NOT EXISTS packets ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - timestamp TEXT NOT NULL, - observer_id TEXT, - observer_name TEXT, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - hash TEXT, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - path_json TEXT, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')) - ); - CREATE TABLE IF NOT EXISTS nodes ( public_key TEXT PRIMARY KEY, name TEXT, @@ -60,16 +52,6 @@ db.exec(` noise_floor INTEGER ); - CREATE TABLE IF NOT EXISTS paths ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - packet_id INTEGER REFERENCES packets(id), - hop_index INTEGER, - node_hash TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_packets_timestamp ON packets(timestamp); - CREATE INDEX IF NOT EXISTS idx_packets_hash ON packets(hash); - CREATE INDEX IF NOT EXISTS idx_packets_payload_type ON packets(payload_type); CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen); CREATE INDEX IF NOT EXISTS idx_observers_last_seen ON observers(last_seen); @@ -149,11 +131,6 @@ for (const col of ['model', 'firmware', 'client_version', 'radio', 'battery_mv', // --- Prepared statements --- const stmts = { - insertPacket: db.prepare(` - INSERT INTO packets (raw_hex, timestamp, observer_id, observer_name, direction, snr, rssi, score, hash, route_type, payload_type, payload_version, path_json, decoded_json) - VALUES (@raw_hex, @timestamp, @observer_id, @observer_name, @direction, @snr, @rssi, @score, @hash, @route_type, @payload_type, @payload_version, @path_json, @decoded_json) - `), - insertPath: db.prepare(`INSERT INTO paths (packet_id, hop_index, node_hash) VALUES (?, ?, ?)`), upsertNode: db.prepare(` INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES (@public_key, @name, @role, @lat, @lon, @last_seen, @first_seen, 1) @@ -197,7 +174,6 @@ const stmts = { noise_floor = COALESCE(@noise_floor, noise_floor) `), getPacket: db.prepare(`SELECT * FROM packets_v WHERE id = ?`), - getPathsForPacket: db.prepare(`SELECT * FROM paths WHERE packet_id = ? ORDER BY hop_index`), getNode: db.prepare(`SELECT * FROM nodes WHERE public_key = ?`), getRecentPacketsForNode: db.prepare(` SELECT * FROM packets_v WHERE decoded_json LIKE ? OR decoded_json LIKE ? OR decoded_json LIKE ? OR decoded_json LIKE ? @@ -222,26 +198,6 @@ const stmts = { // --- Helper functions --- -function insertPacket(data) { - const d = { - raw_hex: data.raw_hex, - timestamp: data.timestamp || new Date().toISOString(), - observer_id: data.observer_id || null, - observer_name: data.observer_name || null, - direction: data.direction || null, - snr: data.snr ?? null, - rssi: data.rssi ?? null, - score: data.score ?? null, - hash: data.hash || null, - route_type: data.route_type ?? null, - payload_type: data.payload_type ?? null, - payload_version: data.payload_version ?? null, - path_json: data.path_json || null, - decoded_json: data.decoded_json || null, - }; - return stmts.insertPacket.run(d).lastInsertRowid; -} - function insertTransmission(data) { const hash = data.hash; if (!hash) return null; // Can't deduplicate without a hash @@ -285,15 +241,6 @@ function insertTransmission(data) { return { transmissionId, observationId: obsResult.lastInsertRowid }; } -function insertPath(packetId, hops) { - const tx = db.transaction((hops) => { - for (let i = 0; i < hops.length; i++) { - stmts.insertPath.run(packetId, i, hops[i]); - } - }); - tx(hops); -} - function upsertNode(data) { const now = new Date().toISOString(); stmts.upsertNode.run({ @@ -365,7 +312,6 @@ function getTransmission(id) { function getPacket(id) { const packet = stmts.getPacket.get(id); if (!packet) return null; - packet.paths = stmts.getPathsForPacket.all(id); return packet; } @@ -685,4 +631,4 @@ function getNodeAnalytics(pubkey, days) { }; } -module.exports = { db, insertPacket, insertTransmission, insertPath, upsertNode, upsertObserver, updateObserverStatus, getPackets, getPacket, getTransmission, getNodes, getNode, getObservers, getStats, seed, searchNodes, getNodeHealth, getNodeAnalytics }; +module.exports = { db, insertTransmission, upsertNode, upsertObserver, updateObserverStatus, getPackets, getPacket, getTransmission, getNodes, getNode, getObservers, getStats, seed, searchNodes, getNodeHealth, getNodeAnalytics }; diff --git a/packet-store.js b/packet-store.js index ffb300f8..47a52d42 100644 --- a/packet-store.js +++ b/packet-store.js @@ -8,7 +8,7 @@ */ class PacketStore { constructor(dbModule, config = {}) { - this.dbModule = dbModule; // The full db module (has .db, .insertPacket, .getPacket) + this.dbModule = dbModule; // The full db module (has .db, .insertTransmission, .getPacket) this.db = dbModule.db; // Raw better-sqlite3 instance for queries this.maxBytes = (config.maxMemoryMB || 1024) * 1024 * 1024; this.estPacketBytes = config.estimatedPacketBytes || 450; @@ -327,11 +327,10 @@ class PacketStore { /** Insert a new packet (to both memory and SQLite) */ insert(packetData) { - const id = this.dbModule.insertPacket(packetData); - // Also write to normalized tables and get the transmission ID + // Write to normalized tables and get the transmission ID const txResult = this.dbModule.insertTransmission ? this.dbModule.insertTransmission(packetData) : null; const transmissionId = txResult ? txResult.transmissionId : null; - const observationId = txResult ? txResult.observationId : id; + const observationId = txResult ? txResult.observationId : null; // Build row directly from packetData — avoids view ID mismatch issues const row = { @@ -675,7 +674,7 @@ class PacketStore { if (since) { where.push('timestamp > ?'); params.push(since); } if (until) { where.push('timestamp < ?'); params.push(until); } if (region) { where.push('observer_id IN (SELECT id FROM observers WHERE iata = ?)'); params.push(region); } - if (node) { try { const nr = this.db.prepare('SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1').get(node, node); const pk = nr ? nr.public_key : node; where.push('(decoded_json LIKE ? OR id IN (SELECT packet_id FROM paths WHERE node_hash = ?))'); params.push('%' + pk + '%', pk.substring(0, 8)); } catch(e) { where.push('decoded_json LIKE ?'); params.push('%' + node + '%'); } } + if (node) { try { const nr = this.db.prepare('SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1').get(node, node); const pk = nr ? nr.public_key : node; where.push('decoded_json LIKE ?'); params.push('%' + pk + '%'); } catch(e) { where.push('decoded_json LIKE ?'); params.push('%' + node + '%'); } } const w = where.length ? 'WHERE ' + where.join(' AND ') : ''; const total = this.db.prepare(`SELECT COUNT(*) as c FROM packets_v ${w}`).get(...params).c; const packets = this.db.prepare(`SELECT * FROM packets_v ${w} ORDER BY timestamp ${order === 'ASC' ? 'ASC' : 'DESC'} LIMIT ? OFFSET ?`).all(...params, limit, offset); @@ -692,7 +691,7 @@ class PacketStore { if (since) { where.push('timestamp > ?'); params.push(since); } if (until) { where.push('timestamp < ?'); params.push(until); } if (region) { where.push('observer_id IN (SELECT id FROM observers WHERE iata = ?)'); params.push(region); } - if (node) { try { const nr = this.db.prepare('SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1').get(node, node); const pk = nr ? nr.public_key : node; where.push('(decoded_json LIKE ? OR id IN (SELECT packet_id FROM paths WHERE node_hash = ?))'); params.push('%' + pk + '%', pk.substring(0, 8)); } catch(e) { where.push('decoded_json LIKE ?'); params.push('%' + node + '%'); } } + if (node) { try { const nr = this.db.prepare('SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1').get(node, node); const pk = nr ? nr.public_key : node; where.push('decoded_json LIKE ?'); params.push('%' + pk + '%'); } catch(e) { where.push('decoded_json LIKE ?'); params.push('%' + node + '%'); } } const w = where.length ? 'WHERE ' + where.join(' AND ') : ''; const sql = `SELECT hash, COUNT(*) as count, COUNT(DISTINCT observer_id) as observer_count, diff --git a/server.js b/server.js index 1357640b..642c4391 100644 --- a/server.js +++ b/server.js @@ -697,7 +697,6 @@ for (const source of mqttSources) { try { db.insertTransmission(pktData); } catch (e) { console.error('[dual-write] transmission insert error:', e.message); } if (decoded.path.hops.length > 0) { - db.insertPath(packetId, decoded.path.hops); // Auto-create stub nodes from 2+ byte path hops autoLearnHopNodes(decoded.path.hops, now); } @@ -920,9 +919,7 @@ app.get('/api/packets', (req, res) => { for (const p of found) allPackets.set(p.id, p); } let results = [...allPackets.values()].sort((a, b) => order === 'DESC' ? b.timestamp.localeCompare(a.timestamp) : a.timestamp.localeCompare(b.timestamp)); - // Apply additional filters - if (type !== undefined) { const types = String(type).split(','); results = results.filter(p => types.includes(String(p.payload_type))); } - if (observer) { const obsIds = observer.split(','); results = results.filter(p => obsIds.includes(p.observer_id)); } + // Apply additional filters (type/observer filtering done client-side; server only filters for nodes query path) if (region) results = results.filter(p => (p.observer_id || '').includes(region) || (p.decoded_json || '').includes(region)); if (since) results = results.filter(p => p.timestamp >= since); if (until) results = results.filter(p => p.timestamp <= until); @@ -1093,7 +1090,6 @@ app.post('/api/packets', requireApiKey, (req, res) => { try { db.insertTransmission(apiPktData); } catch (e) { console.error('[dual-write] transmission insert error:', e.message); } if (decoded.path.hops.length > 0) { - db.insertPath(packetId, decoded.path.hops); autoLearnHopNodes(decoded.path.hops, new Date().toISOString()); }