From cb8e20ae7eea1e14dd45059df0fb6cd24f6a121a Mon Sep 17 00:00:00 2001
From: you <you@example.com>
Date: Sat, 21 Mar 2026 22:06:43 +0000
Subject: [PATCH] fix: deduplicate observations with NULL path_json

The UNIQUE index on (hash, observer_id, path_json) didn't prevent
duplicates when path_json was NULL because SQLite treats NULL != NULL
for uniqueness. Fixed by:

1. Using COALESCE(path_json, '') in the UNIQUE index expression
2. Adding migration to clean up existing duplicate rows
3. Adding NULL-safe dedup checks in PacketStore load and insert paths
---
 db.js           | 8 +++++++-
 packet-store.js | 6 +++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/db.js b/db.js
index fdde5348..66a1d13f 100644
--- a/db.js
+++ b/db.js
@@ -107,7 +107,13 @@ db.exec(`
   CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id);
   CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id);
   CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp);
-  CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(hash, observer_id, path_json);
+  DROP INDEX IF EXISTS idx_observations_dedup;
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(hash, observer_id, COALESCE(path_json, ''));
+
+  -- Clean up legacy duplicates (same hash+observer+path, keep lowest id)
+  DELETE FROM observations WHERE id NOT IN (
+    SELECT MIN(id) FROM observations GROUP BY hash, observer_id, COALESCE(path_json, '')
+  );
 
   CREATE VIEW IF NOT EXISTS packets_v AS
     SELECT o.id, t.raw_hex, o.timestamp, o.observer_id, o.observer_name,
diff --git a/packet-store.js b/packet-store.js
index cc080ccc..c4d085bd 100644
--- a/packet-store.js
+++ b/packet-store.js
@@ -126,6 +126,10 @@ class PacketStore {
           route_type: row.route_type,
         };
 
+        // Dedup: skip if same observer + same path already loaded
+        const isDupeLoad = tx.observations.some(o => o.observer_id === obs.observer_id && (o.path_json || '') === (obs.path_json || ''));
+        if (isDupeLoad) continue;
+
         tx.observations.push(obs);
         tx.observation_count++;
 
@@ -227,7 +231,7 @@ class PacketStore {
       route_type: pkt.route_type,
     };
     // Dedup: skip if same observer + same path already recorded for this transmission
-    const isDupe = tx.observations.some(o => o.observer_id === obs.observer_id && o.path_json === obs.path_json);
+    const isDupe = tx.observations.some(o => o.observer_id === obs.observer_id && (o.path_json || '') === (obs.path_json || ''));
     if (isDupe) return tx;
 
     tx.observations.push(obs);