Drop legacy packets + paths tables on startup, remove dead code

Migration runs automatically on next startup — drops paths first (FK to
packets), then packets. Removes insertPacket(), insertPath(), all
prepared statements and references to both tables. Server-side type/
observer filtering also removed (client does it in-memory).

Saves ~2M rows (paths) + full packets table worth of disk.

db.js

@@ -12,27 +12,19 @@ db.pragma('journal_mode = WAL');
 db.pragma('foreign_keys = ON');
 db.pragma('wal_autocheckpoint = 0'); // Disable auto-checkpoint — manual checkpoint on timer to avoid random event loop spikes
 
+// --- Migration: drop legacy tables (replaced by transmissions + observations in v2.3.0) ---
+// Drop paths first (has FK to packets)
+const legacyTables = ['paths', 'packets'];
+for (const t of legacyTables) {
+  const exists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(t);
+  if (exists) {
+    console.log(`[migration] Dropping legacy table: ${t}`);
+    db.exec(`DROP TABLE IF EXISTS ${t}`);
+  }
+}
+
 // --- Schema ---
 db.exec(`
-  CREATE TABLE IF NOT EXISTS packets (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    raw_hex TEXT NOT NULL,
-    timestamp TEXT NOT NULL,
-    observer_id TEXT,
-    observer_name TEXT,
-    direction TEXT,
-    snr REAL,
-    rssi REAL,
-    score INTEGER,
-    hash TEXT,
-    route_type INTEGER,
-    payload_type INTEGER,
-    payload_version INTEGER,
-    path_json TEXT,
-    decoded_json TEXT,
-    created_at TEXT DEFAULT (datetime('now'))
-  );
-
   CREATE TABLE IF NOT EXISTS nodes (
     public_key TEXT PRIMARY KEY,
     name TEXT,
@@ -60,16 +52,6 @@ db.exec(`
     noise_floor INTEGER
   );
 
-  CREATE TABLE IF NOT EXISTS paths (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    packet_id INTEGER REFERENCES packets(id),
-    hop_index INTEGER,
-    node_hash TEXT
-  );
-
-  CREATE INDEX IF NOT EXISTS idx_packets_timestamp ON packets(timestamp);
-  CREATE INDEX IF NOT EXISTS idx_packets_hash ON packets(hash);
-  CREATE INDEX IF NOT EXISTS idx_packets_payload_type ON packets(payload_type);
   CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen);
   CREATE INDEX IF NOT EXISTS idx_observers_last_seen ON observers(last_seen);
 
@@ -149,11 +131,6 @@ for (const col of ['model', 'firmware', 'client_version', 'radio', 'battery_mv',
 
 // --- Prepared statements ---
 const stmts = {
-  insertPacket: db.prepare(`
-    INSERT INTO packets (raw_hex, timestamp, observer_id, observer_name, direction, snr, rssi, score, hash, route_type, payload_type, payload_version, path_json, decoded_json)
-    VALUES (@raw_hex, @timestamp, @observer_id, @observer_name, @direction, @snr, @rssi, @score, @hash, @route_type, @payload_type, @payload_version, @path_json, @decoded_json)
-  `),
-  insertPath: db.prepare(`INSERT INTO paths (packet_id, hop_index, node_hash) VALUES (?, ?, ?)`),
   upsertNode: db.prepare(`
     INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count)
     VALUES (@public_key, @name, @role, @lat, @lon, @last_seen, @first_seen, 1)
@@ -197,7 +174,6 @@ const stmts = {
       noise_floor = COALESCE(@noise_floor, noise_floor)
   `),
   getPacket: db.prepare(`SELECT * FROM packets_v WHERE id = ?`),
-  getPathsForPacket: db.prepare(`SELECT * FROM paths WHERE packet_id = ? ORDER BY hop_index`),
   getNode: db.prepare(`SELECT * FROM nodes WHERE public_key = ?`),
   getRecentPacketsForNode: db.prepare(`
     SELECT * FROM packets_v WHERE decoded_json LIKE ? OR decoded_json LIKE ? OR decoded_json LIKE ? OR decoded_json LIKE ?
@@ -222,26 +198,6 @@ const stmts = {
 
 // --- Helper functions ---
 
-function insertPacket(data) {
-  const d = {
-    raw_hex: data.raw_hex,
-    timestamp: data.timestamp || new Date().toISOString(),
-    observer_id: data.observer_id || null,
-    observer_name: data.observer_name || null,
-    direction: data.direction || null,
-    snr: data.snr ?? null,
-    rssi: data.rssi ?? null,
-    score: data.score ?? null,
-    hash: data.hash || null,
-    route_type: data.route_type ?? null,
-    payload_type: data.payload_type ?? null,
-    payload_version: data.payload_version ?? null,
-    path_json: data.path_json || null,
-    decoded_json: data.decoded_json || null,
-  };
-  return stmts.insertPacket.run(d).lastInsertRowid;
-}
-
 function insertTransmission(data) {
   const hash = data.hash;
   if (!hash) return null; // Can't deduplicate without a hash
@@ -285,15 +241,6 @@ function insertTransmission(data) {
   return { transmissionId, observationId: obsResult.lastInsertRowid };
 }
 
-function insertPath(packetId, hops) {
-  const tx = db.transaction((hops) => {
-    for (let i = 0; i < hops.length; i++) {
-      stmts.insertPath.run(packetId, i, hops[i]);
-    }
-  });
-  tx(hops);
-}
-
 function upsertNode(data) {
   const now = new Date().toISOString();
   stmts.upsertNode.run({
@@ -365,7 +312,6 @@ function getTransmission(id) {
 function getPacket(id) {
   const packet = stmts.getPacket.get(id);
   if (!packet) return null;
-  packet.paths = stmts.getPathsForPacket.all(id);
   return packet;
 }
 
@@ -685,4 +631,4 @@ function getNodeAnalytics(pubkey, days) {
   };
 }
 
-module.exports = { db, insertPacket, insertTransmission, insertPath, upsertNode, upsertObserver, updateObserverStatus, getPackets, getPacket, getTransmission, getNodes, getNode, getObservers, getStats, seed, searchNodes, getNodeHealth, getNodeAnalytics };
+module.exports = { db, insertTransmission, upsertNode, upsertObserver, updateObserverStatus, getPackets, getPacket, getTransmission, getNodes, getNode, getObservers, getStats, seed, searchNodes, getNodeHealth, getNodeAnalytics };

packet-store.js

@@ -8,7 +8,7 @@
  */
 class PacketStore {
   constructor(dbModule, config = {}) {
-    this.dbModule = dbModule;  // The full db module (has .db, .insertPacket, .getPacket)
+    this.dbModule = dbModule;  // The full db module (has .db, .insertTransmission, .getPacket)
     this.db = dbModule.db;     // Raw better-sqlite3 instance for queries
     this.maxBytes = (config.maxMemoryMB || 1024) * 1024 * 1024;
     this.estPacketBytes = config.estimatedPacketBytes || 450;
@@ -327,11 +327,10 @@ class PacketStore {
 
   /** Insert a new packet (to both memory and SQLite) */
   insert(packetData) {
-    const id = this.dbModule.insertPacket(packetData);
-    // Also write to normalized tables and get the transmission ID
+    // Write to normalized tables and get the transmission ID
     const txResult = this.dbModule.insertTransmission ? this.dbModule.insertTransmission(packetData) : null;
     const transmissionId = txResult ? txResult.transmissionId : null;
-    const observationId = txResult ? txResult.observationId : id;
+    const observationId = txResult ? txResult.observationId : null;
 
     // Build row directly from packetData — avoids view ID mismatch issues
     const row = {
@@ -675,7 +674,7 @@ class PacketStore {
     if (since) { where.push('timestamp > ?'); params.push(since); }
     if (until) { where.push('timestamp < ?'); params.push(until); }
     if (region) { where.push('observer_id IN (SELECT id FROM observers WHERE iata = ?)'); params.push(region); }
-    if (node) { try { const nr = this.db.prepare('SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1').get(node, node); const pk = nr ? nr.public_key : node; where.push('(decoded_json LIKE ? OR id IN (SELECT packet_id FROM paths WHERE node_hash = ?))'); params.push('%' + pk + '%', pk.substring(0, 8)); } catch(e) { where.push('decoded_json LIKE ?'); params.push('%' + node + '%'); } }
+    if (node) { try { const nr = this.db.prepare('SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1').get(node, node); const pk = nr ? nr.public_key : node; where.push('decoded_json LIKE ?'); params.push('%' + pk + '%'); } catch(e) { where.push('decoded_json LIKE ?'); params.push('%' + node + '%'); } }
     const w = where.length ? 'WHERE ' + where.join(' AND ') : '';
     const total = this.db.prepare(`SELECT COUNT(*) as c FROM packets_v ${w}`).get(...params).c;
     const packets = this.db.prepare(`SELECT * FROM packets_v ${w} ORDER BY timestamp ${order === 'ASC' ? 'ASC' : 'DESC'} LIMIT ? OFFSET ?`).all(...params, limit, offset);
@@ -692,7 +691,7 @@ class PacketStore {
     if (since) { where.push('timestamp > ?'); params.push(since); }
     if (until) { where.push('timestamp < ?'); params.push(until); }
     if (region) { where.push('observer_id IN (SELECT id FROM observers WHERE iata = ?)'); params.push(region); }
-    if (node) { try { const nr = this.db.prepare('SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1').get(node, node); const pk = nr ? nr.public_key : node; where.push('(decoded_json LIKE ? OR id IN (SELECT packet_id FROM paths WHERE node_hash = ?))'); params.push('%' + pk + '%', pk.substring(0, 8)); } catch(e) { where.push('decoded_json LIKE ?'); params.push('%' + node + '%'); } }
+    if (node) { try { const nr = this.db.prepare('SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1').get(node, node); const pk = nr ? nr.public_key : node; where.push('decoded_json LIKE ?'); params.push('%' + pk + '%'); } catch(e) { where.push('decoded_json LIKE ?'); params.push('%' + node + '%'); } }
     const w = where.length ? 'WHERE ' + where.join(' AND ') : '';
 
     const sql = `SELECT hash, COUNT(*) as count, COUNT(DISTINCT observer_id) as observer_count,

server.js

@@ -697,7 +697,6 @@ for (const source of mqttSources) {
         try { db.insertTransmission(pktData); } catch (e) { console.error('[dual-write] transmission insert error:', e.message); }
 
         if (decoded.path.hops.length > 0) {
-          db.insertPath(packetId, decoded.path.hops);
           // Auto-create stub nodes from 2+ byte path hops
           autoLearnHopNodes(decoded.path.hops, now);
         }
@@ -920,9 +919,7 @@ app.get('/api/packets', (req, res) => {
       for (const p of found) allPackets.set(p.id, p);
     }
     let results = [...allPackets.values()].sort((a, b) => order === 'DESC' ? b.timestamp.localeCompare(a.timestamp) : a.timestamp.localeCompare(b.timestamp));
-    // Apply additional filters
-    if (type !== undefined) { const types = String(type).split(','); results = results.filter(p => types.includes(String(p.payload_type))); }
-    if (observer) { const obsIds = observer.split(','); results = results.filter(p => obsIds.includes(p.observer_id)); }
+    // Apply additional filters (type/observer filtering done client-side; server only filters for nodes query path)
     if (region) results = results.filter(p => (p.observer_id || '').includes(region) || (p.decoded_json || '').includes(region));
     if (since) results = results.filter(p => p.timestamp >= since);
     if (until) results = results.filter(p => p.timestamp <= until);
@@ -1093,7 +1090,6 @@ app.post('/api/packets', requireApiKey, (req, res) => {
     try { db.insertTransmission(apiPktData); } catch (e) { console.error('[dual-write] transmission insert error:', e.message); }
 
     if (decoded.path.hops.length > 0) {
-      db.insertPath(packetId, decoded.path.hops);
       autoLearnHopNodes(decoded.path.hops, new Date().toISOString());
     }