fix: deduplicate observations with NULL path_json

The UNIQUE index on (hash, observer_id, path_json) didn't prevent
duplicates when path_json was NULL because SQLite treats NULL != NULL
for uniqueness. Fixed by:

1. Using COALESCE(path_json, '') in the UNIQUE index expression
2. Adding migration to clean up existing duplicate rows
3. Adding NULL-safe dedup checks in PacketStore load and insert paths
This commit is contained in:
you
2026-03-21 22:06:43 +00:00
parent 0dcf973e43
commit cb8e20ae7e
2 changed files with 12 additions and 2 deletions
+7 -1
View File
@@ -107,7 +107,13 @@ db.exec(`
CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id);
CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id);
CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp);
CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(hash, observer_id, path_json);
DROP INDEX IF EXISTS idx_observations_dedup;
CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(hash, observer_id, COALESCE(path_json, ''));
-- Clean up legacy duplicates (same hash+observer+path, keep lowest id)
DELETE FROM observations WHERE id NOT IN (
SELECT MIN(id) FROM observations GROUP BY hash, observer_id, COALESCE(path_json, '')
);
CREATE VIEW IF NOT EXISTS packets_v AS
SELECT o.id, t.raw_hex, o.timestamp, o.observer_id, o.observer_name,
+5 -1
View File
@@ -126,6 +126,10 @@ class PacketStore {
route_type: row.route_type,
};
// Dedup: skip if same observer + same path already loaded
const isDupeLoad = tx.observations.some(o => o.observer_id === obs.observer_id && (o.path_json || '') === (obs.path_json || ''));
if (isDupeLoad) continue;
tx.observations.push(obs);
tx.observation_count++;
@@ -227,7 +231,7 @@ class PacketStore {
route_type: pkt.route_type,
};
// Dedup: skip if same observer + same path already recorded for this transmission
const isDupe = tx.observations.some(o => o.observer_id === obs.observer_id && o.path_json === obs.path_json);
const isDupe = tx.observations.some(o => o.observer_id === obs.observer_id && (o.path_json || '') === (obs.path_json || ''));
if (isDupe) return tx;
tx.observations.push(obs);