mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-08-28 12:44:12 +00:00
fix: deduplicate observations with NULL path_json
The UNIQUE index on (hash, observer_id, path_json) didn't prevent duplicates when path_json was NULL because SQLite treats NULL != NULL for uniqueness. Fixed by: 1. Using COALESCE(path_json, '') in the UNIQUE index expression 2. Adding migration to clean up existing duplicate rows 3. Adding NULL-safe dedup checks in PacketStore load and insert paths
This commit is contained in:
@@ -107,7 +107,13 @@ db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(hash, observer_id, path_json);
|
||||
DROP INDEX IF EXISTS idx_observations_dedup;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(hash, observer_id, COALESCE(path_json, ''));
|
||||
|
||||
-- Clean up legacy duplicates (same hash+observer+path, keep lowest id)
|
||||
DELETE FROM observations WHERE id NOT IN (
|
||||
SELECT MIN(id) FROM observations GROUP BY hash, observer_id, COALESCE(path_json, '')
|
||||
);
|
||||
|
||||
CREATE VIEW IF NOT EXISTS packets_v AS
|
||||
SELECT o.id, t.raw_hex, o.timestamp, o.observer_id, o.observer_name,
|
||||
|
||||
+5
-1
@@ -126,6 +126,10 @@ class PacketStore {
|
||||
route_type: row.route_type,
|
||||
};
|
||||
|
||||
// Dedup: skip if same observer + same path already loaded
|
||||
const isDupeLoad = tx.observations.some(o => o.observer_id === obs.observer_id && (o.path_json || '') === (obs.path_json || ''));
|
||||
if (isDupeLoad) continue;
|
||||
|
||||
tx.observations.push(obs);
|
||||
tx.observation_count++;
|
||||
|
||||
@@ -227,7 +231,7 @@ class PacketStore {
|
||||
route_type: pkt.route_type,
|
||||
};
|
||||
// Dedup: skip if same observer + same path already recorded for this transmission
|
||||
const isDupe = tx.observations.some(o => o.observer_id === obs.observer_id && o.path_json === obs.path_json);
|
||||
const isDupe = tx.observations.some(o => o.observer_id === obs.observer_id && (o.path_json || '') === (obs.path_json || ''));
|
||||
if (isDupe) return tx;
|
||||
|
||||
tx.observations.push(obs);
|
||||
|
||||
Reference in New Issue
Block a user