diff --git a/db.js b/db.js index 402ad5f2..4f4468b2 100644 --- a/db.js +++ b/db.js @@ -108,6 +108,14 @@ db.exec(` CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id); CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp); CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(hash, observer_id, path_json); + + CREATE VIEW IF NOT EXISTS packets_v AS + SELECT o.id, t.raw_hex, o.timestamp, o.observer_id, o.observer_name, + o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type, + t.payload_type, t.payload_version, o.path_json, t.decoded_json, + t.created_at + FROM observations o + JOIN transmissions t ON t.id = o.transmission_id; `); // --- Migrations for existing DBs --- @@ -182,18 +190,18 @@ const stmts = { uptime_secs = COALESCE(@uptime_secs, uptime_secs), noise_floor = COALESCE(@noise_floor, noise_floor) `), - getPacket: db.prepare(`SELECT * FROM packets WHERE id = ?`), + getPacket: db.prepare(`SELECT * FROM packets_v WHERE id = ?`), getPathsForPacket: db.prepare(`SELECT * FROM paths WHERE packet_id = ? ORDER BY hop_index`), getNode: db.prepare(`SELECT * FROM nodes WHERE public_key = ?`), getRecentPacketsForNode: db.prepare(` - SELECT * FROM packets WHERE decoded_json LIKE ? OR decoded_json LIKE ? OR decoded_json LIKE ? OR decoded_json LIKE ? + SELECT * FROM packets_v WHERE decoded_json LIKE ? OR decoded_json LIKE ? OR decoded_json LIKE ? OR decoded_json LIKE ? ORDER BY timestamp DESC LIMIT 20 `), getObservers: db.prepare(`SELECT * FROM observers ORDER BY last_seen DESC`), - countPackets: db.prepare(`SELECT COUNT(*) as count FROM packets`), + countPackets: db.prepare(`SELECT COUNT(*) as count FROM observations`), countNodes: db.prepare(`SELECT COUNT(*) as count FROM nodes`), countObservers: db.prepare(`SELECT COUNT(*) as count FROM observers`), - countRecentPackets: db.prepare(`SELECT COUNT(*) as count FROM packets WHERE timestamp > ?`), + countRecentPackets: db.prepare(`SELECT COUNT(*) as count FROM observations WHERE timestamp > ?`), getTransmissionByHash: db.prepare(`SELECT id, first_seen FROM transmissions WHERE hash = ?`), insertTransmission: db.prepare(` INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) @@ -337,8 +345,8 @@ function getPackets({ limit = 50, offset = 0, type, route, hash, since } = {}) { if (hash) { where.push('hash = @hash'); params.hash = hash; } if (since) { where.push('timestamp > @since'); params.since = since; } const clause = where.length ? 'WHERE ' + where.join(' AND ') : ''; - const rows = db.prepare(`SELECT * FROM packets ${clause} ORDER BY timestamp DESC LIMIT @limit OFFSET @offset`).all({ ...params, limit, offset }); - const total = db.prepare(`SELECT COUNT(*) as count FROM packets ${clause}`).get(params).count; + const rows = db.prepare(`SELECT * FROM packets_v ${clause} ORDER BY timestamp DESC LIMIT @limit OFFSET @offset`).all({ ...params, limit, offset }); + const total = db.prepare(`SELECT COUNT(*) as count FROM packets_v ${clause}`).get(params).count; return { rows, total }; } @@ -391,9 +399,9 @@ function getStats() { totalTransmissions = db.prepare('SELECT COUNT(*) as count FROM transmissions').get().count; } catch {} return { - totalPackets: stmts.countPackets.get().count, + totalPackets: totalTransmissions || stmts.countPackets.get().count, totalTransmissions, - totalObservations: stmts.countPackets.get().count, // legacy packets = observations + totalObservations: stmts.countPackets.get().count, totalNodes: stmts.countNodes.get().count, totalObservers: stmts.countObservers.get().count, packetsLastHour: stmts.countRecentPackets.get(oneHourAgo).count, @@ -407,7 +415,7 @@ function seed() { upsertObserver({ id: 'obs-seed-001', name: 'Seed Observer', iata: 'UNK', last_seen: now, first_seen: now }); - const pktId = insertPacket({ + insertTransmission({ raw_hex: rawHex, timestamp: now, observer_id: 'obs-seed-001', @@ -424,8 +432,6 @@ function seed() { decoded_json: JSON.stringify({ type: 'ADVERT', name: 'Test Repeater', role: 'repeater', lat: 0, lon: 0 }), }); - insertPath(pktId, ['A1B2', 'C3D4']); - upsertNode({ public_key: 'seed-test-pubkey', name: 'Test Repeater', @@ -475,7 +481,7 @@ function getNodeHealth(pubkey) { const observers = db.prepare(` SELECT observer_id, observer_name, AVG(snr) as avgSnr, AVG(rssi) as avgRssi, COUNT(*) as packetCount - FROM packets + FROM packets_v WHERE ${whereClause} AND observer_id IS NOT NULL GROUP BY observer_id ORDER BY packetCount DESC @@ -483,20 +489,20 @@ function getNodeHealth(pubkey) { // Stats const packetsToday = db.prepare(` - SELECT COUNT(*) as count FROM packets WHERE ${whereClause} AND timestamp > @since + SELECT COUNT(*) as count FROM packets_v WHERE ${whereClause} AND timestamp > @since `).get({ ...params, since: todayISO }).count; const avgStats = db.prepare(` - SELECT AVG(snr) as avgSnr FROM packets WHERE ${whereClause} + SELECT AVG(snr) as avgSnr FROM packets_v WHERE ${whereClause} `).get(params); const lastHeard = db.prepare(` - SELECT MAX(timestamp) as lastHeard FROM packets WHERE ${whereClause} + SELECT MAX(timestamp) as lastHeard FROM packets_v WHERE ${whereClause} `).get(params).lastHeard; // Avg hops from path_json const pathRows = db.prepare(` - SELECT path_json FROM packets WHERE ${whereClause} AND path_json IS NOT NULL + SELECT path_json FROM packets_v WHERE ${whereClause} AND path_json IS NOT NULL `).all(params); let totalHops = 0, hopCount = 0; @@ -509,12 +515,12 @@ function getNodeHealth(pubkey) { const avgHops = hopCount > 0 ? Math.round(totalHops / hopCount) : 0; const totalPackets = db.prepare(` - SELECT COUNT(*) as count FROM packets WHERE ${whereClause} + SELECT COUNT(*) as count FROM packets_v WHERE ${whereClause} `).get(params).count; // Recent 10 packets const recentPackets = db.prepare(` - SELECT * FROM packets WHERE ${whereClause} ORDER BY timestamp DESC LIMIT 10 + SELECT * FROM packets_v WHERE ${whereClause} ORDER BY timestamp DESC LIMIT 10 `).all(params); return { @@ -545,31 +551,31 @@ function getNodeAnalytics(pubkey, days) { // Activity timeline const activityTimeline = db.prepare(` SELECT strftime('%Y-%m-%dT%H:00:00Z', timestamp) as bucket, COUNT(*) as count - FROM packets WHERE ${timeWhere} GROUP BY bucket ORDER BY bucket + FROM packets_v WHERE ${timeWhere} GROUP BY bucket ORDER BY bucket `).all(params); // SNR trend const snrTrend = db.prepare(` SELECT timestamp, snr, rssi, observer_id, observer_name - FROM packets WHERE ${timeWhere} AND snr IS NOT NULL ORDER BY timestamp + FROM packets_v WHERE ${timeWhere} AND snr IS NOT NULL ORDER BY timestamp `).all(params); // Packet type breakdown const packetTypeBreakdown = db.prepare(` - SELECT payload_type, COUNT(*) as count FROM packets WHERE ${timeWhere} GROUP BY payload_type + SELECT payload_type, COUNT(*) as count FROM packets_v WHERE ${timeWhere} GROUP BY payload_type `).all(params); // Observer coverage const observerCoverage = db.prepare(` SELECT observer_id, observer_name, COUNT(*) as packetCount, AVG(snr) as avgSnr, AVG(rssi) as avgRssi, MIN(timestamp) as firstSeen, MAX(timestamp) as lastSeen - FROM packets WHERE ${timeWhere} AND observer_id IS NOT NULL + FROM packets_v WHERE ${timeWhere} AND observer_id IS NOT NULL GROUP BY observer_id ORDER BY packetCount DESC `).all(params); // Hop distribution const pathRows = db.prepare(` - SELECT path_json FROM packets WHERE ${timeWhere} AND path_json IS NOT NULL + SELECT path_json FROM packets_v WHERE ${timeWhere} AND path_json IS NOT NULL `).all(params); const hopCounts = {}; @@ -591,7 +597,7 @@ function getNodeAnalytics(pubkey, days) { // Peer interactions from decoded_json const decodedRows = db.prepare(` - SELECT decoded_json, timestamp FROM packets WHERE ${timeWhere} AND decoded_json IS NOT NULL + SELECT decoded_json, timestamp FROM packets_v WHERE ${timeWhere} AND decoded_json IS NOT NULL `).all(params); const peerMap = {}; @@ -617,11 +623,11 @@ function getNodeAnalytics(pubkey, days) { const uptimeHeatmap = db.prepare(` SELECT CAST(strftime('%w', timestamp) AS INTEGER) as dayOfWeek, CAST(strftime('%H', timestamp) AS INTEGER) as hour, COUNT(*) as count - FROM packets WHERE ${timeWhere} GROUP BY dayOfWeek, hour + FROM packets_v WHERE ${timeWhere} GROUP BY dayOfWeek, hour `).all(params); // Computed stats - const totalPackets = db.prepare(`SELECT COUNT(*) as count FROM packets WHERE ${timeWhere}`).get(params).count; + const totalPackets = db.prepare(`SELECT COUNT(*) as count FROM packets_v WHERE ${timeWhere}`).get(params).count; const uniqueObservers = observerCoverage.length; const uniquePeers = peerInteractions.length; const avgPacketsPerDay = days > 0 ? Math.round(totalPackets / days * 10) / 10 : totalPackets; @@ -633,7 +639,7 @@ function getNodeAnalytics(pubkey, days) { // Longest silence const timestamps = db.prepare(` - SELECT timestamp FROM packets WHERE ${timeWhere} ORDER BY timestamp + SELECT timestamp FROM packets_v WHERE ${timeWhere} ORDER BY timestamp `).all(params).map(r => new Date(r.timestamp).getTime()); let longestSilenceMs = 0, longestSilenceStart = null; @@ -673,4 +679,4 @@ function getNodeAnalytics(pubkey, days) { }; } -module.exports = { db, insertPacket, insertTransmission, insertPath, upsertNode, upsertObserver, updateObserverStatus, getPackets, getPacket, getTransmission, getNodes, getNode, getObservers, getStats, seed, searchNodes, getNodeHealth, getNodeAnalytics }; +module.exports = { db, insertTransmission, insertPath, upsertNode, upsertObserver, updateObserverStatus, getPackets, getPacket, getTransmission, getNodes, getNode, getObservers, getStats, seed, searchNodes, getNodeHealth, getNodeAnalytics }; diff --git a/packet-store.js b/packet-store.js index 73068a9d..d7553f19 100644 --- a/packet-store.js +++ b/packet-store.js @@ -167,7 +167,7 @@ class PacketStore { /** Fallback: load from legacy packets table */ _loadLegacy() { const rows = this.db.prepare( - 'SELECT * FROM packets ORDER BY timestamp DESC' + 'SELECT * FROM packets_v ORDER BY timestamp DESC' ).all(); for (const row of rows) { @@ -557,7 +557,7 @@ class PacketStore { /** Get timestamps for sparkline */ getTimestamps(since) { if (this.sqliteOnly) { - return this.db.prepare('SELECT timestamp FROM packets WHERE timestamp > ? ORDER BY timestamp ASC').all(since).map(r => r.timestamp); + return this.db.prepare('SELECT timestamp FROM packets_v WHERE timestamp > ? ORDER BY timestamp ASC').all(since).map(r => r.timestamp); } const results = []; for (const p of this.packets) { @@ -569,7 +569,7 @@ class PacketStore { /** Get a single packet by ID — checks observation IDs first (backward compat) */ getById(id) { - if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets WHERE id = ?').get(id) || null; + if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets_v WHERE id = ?').get(id) || null; return this.byId.get(id) || null; } @@ -582,20 +582,20 @@ class PacketStore { /** Get all siblings of a packet (same hash) — returns observations array */ getSiblings(hash) { const h = hash.toLowerCase(); - if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets WHERE hash = ? ORDER BY timestamp DESC').all(h); + if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets_v WHERE hash = ? ORDER BY timestamp DESC').all(h); const tx = this.byHash.get(h); return tx ? tx.observations : []; } /** Get all transmissions (backward compat — returns packets array) */ all() { - if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets ORDER BY timestamp DESC').all(); + if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets_v ORDER BY timestamp DESC').all(); return this.packets; } /** Get all transmissions matching a filter function */ filter(fn) { - if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets ORDER BY timestamp DESC').all().filter(fn); + if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets_v ORDER BY timestamp DESC').all().filter(fn); return this.packets.filter(fn); } @@ -629,8 +629,8 @@ class PacketStore { if (region) { where.push('observer_id IN (SELECT id FROM observers WHERE iata = ?)'); params.push(region); } if (node) { try { const nr = this.db.prepare('SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1').get(node, node); const pk = nr ? nr.public_key : node; where.push('(decoded_json LIKE ? OR id IN (SELECT packet_id FROM paths WHERE node_hash = ?))'); params.push('%' + pk + '%', pk.substring(0, 8)); } catch(e) { where.push('decoded_json LIKE ?'); params.push('%' + node + '%'); } } const w = where.length ? 'WHERE ' + where.join(' AND ') : ''; - const total = this.db.prepare(`SELECT COUNT(*) as c FROM packets ${w}`).get(...params).c; - const packets = this.db.prepare(`SELECT * FROM packets ${w} ORDER BY timestamp ${order === 'ASC' ? 'ASC' : 'DESC'} LIMIT ? OFFSET ?`).all(...params, limit, offset); + const total = this.db.prepare(`SELECT COUNT(*) as c FROM packets_v ${w}`).get(...params).c; + const packets = this.db.prepare(`SELECT * FROM packets_v ${w} ORDER BY timestamp ${order === 'ASC' ? 'ASC' : 'DESC'} LIMIT ? OFFSET ?`).all(...params, limit, offset); return { packets, total }; } @@ -651,10 +651,10 @@ class PacketStore { MAX(timestamp) as latest, MIN(observer_id) as observer_id, MIN(observer_name) as observer_name, MIN(path_json) as path_json, MIN(payload_type) as payload_type, MIN(raw_hex) as raw_hex, MIN(decoded_json) as decoded_json - FROM packets ${w} GROUP BY hash ORDER BY latest DESC LIMIT ? OFFSET ?`; + FROM packets_v ${w} GROUP BY hash ORDER BY latest DESC LIMIT ? OFFSET ?`; const packets = this.db.prepare(sql).all(...params, limit, offset); - const countSql = `SELECT COUNT(DISTINCT hash) as c FROM packets ${w}`; + const countSql = `SELECT COUNT(DISTINCT hash) as c FROM packets_v ${w}`; const total = this.db.prepare(countSql).get(...params).c; return { packets, total }; } diff --git a/server.js b/server.js index ac7240b9..28596102 100644 --- a/server.js +++ b/server.js @@ -2030,7 +2030,7 @@ app.get('/api/resolve-hops', (req, res) => { } else { // Fall back to averaging nearby nodes from adverts this observer received const obsNodes = db.db.prepare(` - SELECT n.lat, n.lon FROM packets p + SELECT n.lat, n.lon FROM packets_v p JOIN nodes n ON n.public_key = json_extract(p.decoded_json, '$.pubKey') WHERE (p.observer_id = ? OR p.observer_name = ?) AND p.payload_type = 4