mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-09-01 20:28:24 +00:00
fix: dedup observations - UNIQUE(hash,observer_id,path_json) + INSERT OR IGNORE
~26% of observations were duplicates from multi-broker MQTT ingestion. Added UNIQUE index to prevent future dupes, INSERT OR IGNORE to skip silently, and in-memory dedup check in packet-store.
This commit is contained in:
@@ -106,6 +106,7 @@ db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(hash, observer_id, path_json);
|
||||
`);
|
||||
|
||||
// --- Migrations for existing DBs ---
|
||||
@@ -186,7 +187,7 @@ const stmts = {
|
||||
`),
|
||||
updateTransmissionFirstSeen: db.prepare(`UPDATE transmissions SET first_seen = @first_seen WHERE id = @id`),
|
||||
insertObservation: db.prepare(`
|
||||
INSERT INTO observations (transmission_id, hash, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp)
|
||||
INSERT OR IGNORE INTO observations (transmission_id, hash, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp)
|
||||
VALUES (@transmission_id, @hash, @observer_id, @observer_name, @direction, @snr, @rssi, @score, @path_json, @timestamp)
|
||||
`),
|
||||
};
|
||||
|
||||
@@ -214,6 +214,10 @@ class PacketStore {
|
||||
decoded_json: pkt.decoded_json,
|
||||
route_type: pkt.route_type,
|
||||
};
|
||||
// Dedup: skip if same observer + same path already recorded for this transmission
|
||||
const isDupe = tx.observations.some(o => o.observer_id === obs.observer_id && o.path_json === obs.path_json);
|
||||
if (isDupe) return tx;
|
||||
|
||||
tx.observations.push(obs);
|
||||
tx.observation_count++;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user