diff --git a/db.js b/db.js index fdde5348..66a1d13f 100644 --- a/db.js +++ b/db.js @@ -107,7 +107,13 @@ db.exec(` CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id); CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id); CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp); - CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(hash, observer_id, path_json); + DROP INDEX IF EXISTS idx_observations_dedup; + CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(hash, observer_id, COALESCE(path_json, '')); + + -- Clean up legacy duplicates (same hash+observer+path, keep lowest id) + DELETE FROM observations WHERE id NOT IN ( + SELECT MIN(id) FROM observations GROUP BY hash, observer_id, COALESCE(path_json, '') + ); CREATE VIEW IF NOT EXISTS packets_v AS SELECT o.id, t.raw_hex, o.timestamp, o.observer_id, o.observer_name, diff --git a/packet-store.js b/packet-store.js index cc080ccc..c4d085bd 100644 --- a/packet-store.js +++ b/packet-store.js @@ -126,6 +126,10 @@ class PacketStore { route_type: row.route_type, }; + // Dedup: skip if same observer + same path already loaded + const isDupeLoad = tx.observations.some(o => o.observer_id === obs.observer_id && (o.path_json || '') === (obs.path_json || '')); + if (isDupeLoad) continue; + tx.observations.push(obs); tx.observation_count++; @@ -227,7 +231,7 @@ class PacketStore { route_type: pkt.route_type, }; // Dedup: skip if same observer + same path already recorded for this transmission - const isDupe = tx.observations.some(o => o.observer_id === obs.observer_id && o.path_json === obs.path_json); + const isDupe = tx.observations.some(o => o.observer_id === obs.observer_id && (o.path_json || '') === (obs.path_json || '')); if (isDupe) return tx; tx.observations.push(obs);