fix: dedup observations - UNIQUE(hash,observer_id,path_json) + INSERT OR IGNORE

~26% of observations were duplicates from multi-broker MQTT ingestion.
Added UNIQUE index to prevent future dupes, INSERT OR IGNORE to skip
silently, and in-memory dedup check in packet-store.
This commit is contained in:
you
2026-03-21 02:31:51 +00:00
parent 8e373755c8
commit 6e0cf0fc3e
2 changed files with 6 additions and 1 deletions
+2 -1
View File
@@ -106,6 +106,7 @@ db.exec(`
CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id);
CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id);
CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp);
CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(hash, observer_id, path_json);
`);
// --- Migrations for existing DBs ---
@@ -186,7 +187,7 @@ const stmts = {
`),
updateTransmissionFirstSeen: db.prepare(`UPDATE transmissions SET first_seen = @first_seen WHERE id = @id`),
insertObservation: db.prepare(`
INSERT INTO observations (transmission_id, hash, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp)
INSERT OR IGNORE INTO observations (transmission_id, hash, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp)
VALUES (@transmission_id, @hash, @observer_id, @observer_name, @direction, @snr, @rssi, @score, @path_json, @timestamp)
`),
};
+4
View File
@@ -214,6 +214,10 @@ class PacketStore {
decoded_json: pkt.decoded_json,
route_type: pkt.route_type,
};
// Dedup: skip if same observer + same path already recorded for this transmission
const isDupe = tx.observations.some(o => o.observer_id === obs.observer_id && o.path_json === obs.path_json);
if (isDupe) return tx;
tx.observations.push(obs);
tx.observation_count++;