diff --git a/app/AGENTS.md b/app/AGENTS.md index 77ce4dbe..a5f00d0c 100644 --- a/app/AGENTS.md +++ b/app/AGENTS.md @@ -353,7 +353,7 @@ tests/ The MeshCore radio protocol encodes `sender_timestamp` as a 4-byte little-endian integer (Unix seconds). This is a firmware-level wire format — the radio, the Python library (`commands/messaging.py`), and the decoder (`decoder.py`) all read/write exactly 4 bytes. Millisecond Unix timestamps would overflow 4 bytes, so higher resolution is not possible without a firmware change. -**Consequence:** The dedup index `(type, conversation_key, text, COALESCE(sender_timestamp, 0))` operates at 1-second granularity. Sending identical text to the same conversation twice within one second will hit the UNIQUE constraint on the second insert, returning HTTP 500 *after* the radio has already transmitted. The message is sent over the air but not stored in the database. Do not attempt to fix this by switching to millisecond timestamps — it will break echo dedup (the echo's 4-byte timestamp won't match the stored value) and overflow `to_bytes(4, "little")`. +**Consequence:** Channel-message dedup still operates at 1-second granularity because the radio protocol only provides second-resolution `sender_timestamp`. Do not attempt to fix this by switching to millisecond timestamps — it will break echo dedup (the echo's 4-byte timestamp won't match the stored value) and overflow `to_bytes(4, "little")`. Direct messages no longer share that channel dedup index; they are deduplicated by raw-packet identity instead so legitimate same-text same-second DMs can coexist. ### Outgoing DM echoes remain undecrypted diff --git a/app/database.py b/app/database.py index 5e902156..838ba670 100644 --- a/app/database.py +++ b/app/database.py @@ -50,9 +50,10 @@ CREATE TABLE IF NOT EXISTS messages ( acked INTEGER DEFAULT 0, sender_name TEXT, sender_key TEXT - -- Deduplication: identical text + timestamp in the same conversation is treated as a - -- mesh echo/repeat. Outgoing sends allocate a collision-free sender_timestamp before - -- transmit so legitimate repeat sends do not collide with this index. + -- Deduplication: channel echoes/repeats use a channel-only unique index on + -- identical conversation/text/timestamp. Direct messages are deduplicated + -- separately via raw-packet linkage so legitimate same-text same-second DMs + -- can coexist. -- Enforced via idx_messages_dedup_null_safe (unique index) rather than a table constraint -- to avoid the storage overhead of SQLite's autoindex duplicating every message text. ); @@ -90,7 +91,8 @@ CREATE TABLE IF NOT EXISTS contact_name_history ( CREATE INDEX IF NOT EXISTS idx_messages_received ON messages(received_at); CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_dedup_null_safe - ON messages(type, conversation_key, text, COALESCE(sender_timestamp, 0)); + ON messages(type, conversation_key, text, COALESCE(sender_timestamp, 0)) + WHERE type = 'CHAN'; CREATE INDEX IF NOT EXISTS idx_raw_packets_message_id ON raw_packets(message_id); CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_packets_payload_hash ON raw_packets(payload_hash); CREATE INDEX IF NOT EXISTS idx_contacts_on_radio ON contacts(on_radio); diff --git a/app/migrations.py b/app/migrations.py index 8ef18593..31611a29 100644 --- a/app/migrations.py +++ b/app/migrations.py @@ -331,6 +331,13 @@ async def run_migrations(conn: aiosqlite.Connection) -> int: await set_version(conn, 42) applied += 1 + # Migration 43: Limit message dedup index to channel messages only + if version < 43: + logger.info("Applying migration 43: narrow message dedup index to channels") + await _migrate_043_split_message_dedup_by_type(conn) + await set_version(conn, 43) + applied += 1 + if applied > 0: logger.info( "Applied %d migration(s), schema now at version %d", applied, await get_version(conn) @@ -2443,3 +2450,29 @@ async def _migrate_042_add_channel_flood_scope_override(conn: aiosqlite.Connecti raise await conn.commit() + + +async def _migrate_043_split_message_dedup_by_type(conn: aiosqlite.Connection) -> None: + """Restrict the message dedup index to channel messages.""" + cursor = await conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='messages'" + ) + if await cursor.fetchone() is None: + await conn.commit() + return + + cursor = await conn.execute("PRAGMA table_info(messages)") + columns = {row[1] for row in await cursor.fetchall()} + required_columns = {"type", "conversation_key", "text", "sender_timestamp"} + if not required_columns.issubset(columns): + logger.debug("messages table missing dedup-index columns, skipping migration 43") + await conn.commit() + return + + await conn.execute("DROP INDEX IF EXISTS idx_messages_dedup_null_safe") + await conn.execute( + """CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_dedup_null_safe + ON messages(type, conversation_key, text, COALESCE(sender_timestamp, 0)) + WHERE type = 'CHAN'""" + ) + await conn.commit() diff --git a/app/repository/raw_packets.py b/app/repository/raw_packets.py index f661bc3e..3a31e233 100644 --- a/app/repository/raw_packets.py +++ b/app/repository/raw_packets.py @@ -109,6 +109,18 @@ class RawPacketRepository: ) await db.conn.commit() + @staticmethod + async def get_linked_message_id(packet_id: int) -> int | None: + """Return the linked message ID for a raw packet, if any.""" + cursor = await db.conn.execute( + "SELECT message_id FROM raw_packets WHERE id = ?", + (packet_id,), + ) + row = await cursor.fetchone() + if not row: + return None + return row["message_id"] + @staticmethod async def prune_old_undecrypted(max_age_days: int) -> int: """Delete undecrypted packets older than max_age_days. Returns count deleted.""" diff --git a/app/services/messages.py b/app/services/messages.py index d23169ec..980db94c 100644 --- a/app/services/messages.py +++ b/app/services/messages.py @@ -1,3 +1,4 @@ +import asyncio import logging import time from collections.abc import Callable @@ -13,6 +14,7 @@ logger = logging.getLogger(__name__) BroadcastFn = Callable[..., Any] LOG_MESSAGE_PREVIEW_LEN = 32 +_decrypted_dm_store_lock = asyncio.Lock() def _truncate_for_log(text: str, max_chars: int = LOG_MESSAGE_PREVIEW_LEN) -> str: @@ -125,6 +127,45 @@ async def increment_ack_and_broadcast( return ack_count +async def _reconcile_duplicate_message( + *, + existing_msg: Message, + packet_id: int | None, + path: str | None, + received_at: int, + path_len: int | None, + broadcast_fn: BroadcastFn, +) -> None: + logger.debug( + "Duplicate %s for %s (msg_id=%d, outgoing=%s) - adding path", + existing_msg.type, + existing_msg.conversation_key[:12], + existing_msg.id, + existing_msg.outgoing, + ) + + if path is not None: + paths = await MessageRepository.add_path(existing_msg.id, path, received_at, path_len) + else: + paths = existing_msg.paths or [] + + if existing_msg.outgoing and existing_msg.type == "CHAN": + ack_count = await MessageRepository.increment_ack_count(existing_msg.id) + else: + ack_count = existing_msg.acked + + if existing_msg.outgoing or path is not None: + broadcast_message_acked( + message_id=existing_msg.id, + ack_count=ack_count, + paths=paths, + broadcast_fn=broadcast_fn, + ) + + if packet_id is not None: + await RawPacketRepository.mark_decrypted(packet_id, existing_msg.id) + + async def handle_duplicate_message( *, packet_id: int | None, @@ -153,35 +194,15 @@ async def handle_duplicate_message( ) return - logger.debug( - "Duplicate %s for %s (msg_id=%d, outgoing=%s) - adding path", - msg_type, - conversation_key[:12], - existing_msg.id, - existing_msg.outgoing, + await _reconcile_duplicate_message( + existing_msg=existing_msg, + packet_id=packet_id, + path=path, + received_at=received_at, + path_len=path_len, + broadcast_fn=broadcast_fn, ) - if path is not None: - paths = await MessageRepository.add_path(existing_msg.id, path, received_at, path_len) - else: - paths = existing_msg.paths or [] - - if existing_msg.outgoing and existing_msg.type == "CHAN": - ack_count = await MessageRepository.increment_ack_count(existing_msg.id) - else: - ack_count = existing_msg.acked - - if existing_msg.outgoing or path is not None: - broadcast_message_acked( - message_id=existing_msg.id, - ack_count=ack_count, - paths=paths, - broadcast_fn=broadcast_fn, - ) - - if packet_id is not None: - await RawPacketRepository.mark_decrypted(packet_id, existing_msg.id) - async def create_message_from_decrypted( *, @@ -290,32 +311,64 @@ async def create_dm_message_from_decrypted( conversation_key = their_public_key.lower() sender_name = contact.name if contact and not outgoing else None - msg_id = await MessageRepository.create( - msg_type="PRIV", - text=decrypted.message, - conversation_key=conversation_key, - sender_timestamp=decrypted.timestamp, - received_at=received, - path=path, - path_len=path_len, - outgoing=outgoing, - sender_key=conversation_key if not outgoing else None, - sender_name=sender_name, - ) + async with _decrypted_dm_store_lock: + linked_message_id = await RawPacketRepository.get_linked_message_id(packet_id) + if linked_message_id is not None: + existing_msg = await MessageRepository.get_by_id(linked_message_id) + if existing_msg is not None: + await _reconcile_duplicate_message( + existing_msg=existing_msg, + packet_id=packet_id, + path=path, + received_at=received, + path_len=path_len, + broadcast_fn=broadcast_fn, + ) + return None - if msg_id is None: - await handle_duplicate_message( - packet_id=packet_id, + if outgoing: + existing_msg = await MessageRepository.get_by_content( + msg_type="PRIV", + conversation_key=conversation_key, + text=decrypted.message, + sender_timestamp=decrypted.timestamp, + ) + if existing_msg is not None: + await _reconcile_duplicate_message( + existing_msg=existing_msg, + packet_id=packet_id, + path=path, + received_at=received, + path_len=path_len, + broadcast_fn=broadcast_fn, + ) + return None + + msg_id = await MessageRepository.create( msg_type="PRIV", - conversation_key=conversation_key, text=decrypted.message, + conversation_key=conversation_key, sender_timestamp=decrypted.timestamp, - path=path, received_at=received, + path=path, path_len=path_len, - broadcast_fn=broadcast_fn, + outgoing=outgoing, + sender_key=conversation_key if not outgoing else None, + sender_name=sender_name, ) - return None + if msg_id is None: + await handle_duplicate_message( + packet_id=packet_id, + msg_type="PRIV", + conversation_key=conversation_key, + text=decrypted.message, + sender_timestamp=decrypted.timestamp, + path=path, + received_at=received, + path_len=path_len, + broadcast_fn=broadcast_fn, + ) + return None logger.info( 'Stored direct message "%s" for %r (msg ID %d in contact ID %s, outgoing=%s)', @@ -364,6 +417,15 @@ async def create_fallback_direct_message( message_repository=MessageRepository, ) -> Message | None: """Store and broadcast a CONTACT_MSG_RECV fallback direct message.""" + existing = await message_repository.get_by_content( + msg_type="PRIV", + conversation_key=conversation_key, + text=text, + sender_timestamp=sender_timestamp, + ) + if existing is not None: + return None + msg_id = await message_repository.create( msg_type="PRIV", text=text, diff --git a/frontend/prebuilt/assets/BotCodeEditor-B-rKDkVs.js b/frontend/prebuilt/assets/BotCodeEditor-CAcMF_eo.js similarity index 99% rename from frontend/prebuilt/assets/BotCodeEditor-B-rKDkVs.js rename to frontend/prebuilt/assets/BotCodeEditor-CAcMF_eo.js index 6fe75b68..5cf59fe3 100644 Binary files a/frontend/prebuilt/assets/BotCodeEditor-B-rKDkVs.js and b/frontend/prebuilt/assets/BotCodeEditor-CAcMF_eo.js differ diff --git a/frontend/prebuilt/assets/BotCodeEditor-B-rKDkVs.js.map b/frontend/prebuilt/assets/BotCodeEditor-CAcMF_eo.js.map similarity index 99% rename from frontend/prebuilt/assets/BotCodeEditor-B-rKDkVs.js.map rename to frontend/prebuilt/assets/BotCodeEditor-CAcMF_eo.js.map index 2b3bfa80..9b10c829 100644 Binary files a/frontend/prebuilt/assets/BotCodeEditor-B-rKDkVs.js.map and b/frontend/prebuilt/assets/BotCodeEditor-CAcMF_eo.js.map differ diff --git a/frontend/prebuilt/assets/CrackerPanel-DzEfUHYA.js b/frontend/prebuilt/assets/CrackerPanel-oiYTLODx.js similarity index 99% rename from frontend/prebuilt/assets/CrackerPanel-DzEfUHYA.js rename to frontend/prebuilt/assets/CrackerPanel-oiYTLODx.js index 51e0cacf..9a73fba4 100644 Binary files a/frontend/prebuilt/assets/CrackerPanel-DzEfUHYA.js and b/frontend/prebuilt/assets/CrackerPanel-oiYTLODx.js differ diff --git a/frontend/prebuilt/assets/CrackerPanel-DzEfUHYA.js.map b/frontend/prebuilt/assets/CrackerPanel-oiYTLODx.js.map similarity index 99% rename from frontend/prebuilt/assets/CrackerPanel-DzEfUHYA.js.map rename to frontend/prebuilt/assets/CrackerPanel-oiYTLODx.js.map index be61082e..ae9da129 100644 Binary files a/frontend/prebuilt/assets/CrackerPanel-DzEfUHYA.js.map and b/frontend/prebuilt/assets/CrackerPanel-oiYTLODx.js.map differ diff --git a/frontend/prebuilt/assets/MapView-CCM06mdk.js b/frontend/prebuilt/assets/MapView-D-l2CjgS.js similarity index 92% rename from frontend/prebuilt/assets/MapView-CCM06mdk.js rename to frontend/prebuilt/assets/MapView-D-l2CjgS.js index e589d418..6fa8ea87 100644 Binary files a/frontend/prebuilt/assets/MapView-CCM06mdk.js and b/frontend/prebuilt/assets/MapView-D-l2CjgS.js differ diff --git a/frontend/prebuilt/assets/MapView-CCM06mdk.js.map b/frontend/prebuilt/assets/MapView-D-l2CjgS.js.map similarity index 99% rename from frontend/prebuilt/assets/MapView-CCM06mdk.js.map rename to frontend/prebuilt/assets/MapView-D-l2CjgS.js.map index b01df0c9..37183972 100644 Binary files a/frontend/prebuilt/assets/MapView-CCM06mdk.js.map and b/frontend/prebuilt/assets/MapView-D-l2CjgS.js.map differ diff --git a/frontend/prebuilt/assets/NeighborsMiniMap-9qRN4lfs.js b/frontend/prebuilt/assets/NeighborsMiniMap-CstRT55H.js similarity index 85% rename from frontend/prebuilt/assets/NeighborsMiniMap-9qRN4lfs.js rename to frontend/prebuilt/assets/NeighborsMiniMap-CstRT55H.js index 1693c641..0fb0bcb4 100644 Binary files a/frontend/prebuilt/assets/NeighborsMiniMap-9qRN4lfs.js and b/frontend/prebuilt/assets/NeighborsMiniMap-CstRT55H.js differ diff --git a/frontend/prebuilt/assets/NeighborsMiniMap-9qRN4lfs.js.map b/frontend/prebuilt/assets/NeighborsMiniMap-CstRT55H.js.map similarity index 97% rename from frontend/prebuilt/assets/NeighborsMiniMap-9qRN4lfs.js.map rename to frontend/prebuilt/assets/NeighborsMiniMap-CstRT55H.js.map index b2cf6c5b..d02e183c 100644 Binary files a/frontend/prebuilt/assets/NeighborsMiniMap-9qRN4lfs.js.map and b/frontend/prebuilt/assets/NeighborsMiniMap-CstRT55H.js.map differ diff --git a/frontend/prebuilt/assets/PathRouteMap-BEJ9Rkwd.js b/frontend/prebuilt/assets/PathRouteMap-Bi6ttl0X.js similarity index 95% rename from frontend/prebuilt/assets/PathRouteMap-BEJ9Rkwd.js rename to frontend/prebuilt/assets/PathRouteMap-Bi6ttl0X.js index 05201c5b..0ad7dd03 100644 Binary files a/frontend/prebuilt/assets/PathRouteMap-BEJ9Rkwd.js and b/frontend/prebuilt/assets/PathRouteMap-Bi6ttl0X.js differ diff --git a/frontend/prebuilt/assets/PathRouteMap-BEJ9Rkwd.js.map b/frontend/prebuilt/assets/PathRouteMap-Bi6ttl0X.js.map similarity index 99% rename from frontend/prebuilt/assets/PathRouteMap-BEJ9Rkwd.js.map rename to frontend/prebuilt/assets/PathRouteMap-Bi6ttl0X.js.map index bce486ca..8cedc4c3 100644 Binary files a/frontend/prebuilt/assets/PathRouteMap-BEJ9Rkwd.js.map and b/frontend/prebuilt/assets/PathRouteMap-Bi6ttl0X.js.map differ diff --git a/frontend/prebuilt/assets/Popup-s69FReak.js b/frontend/prebuilt/assets/Popup-DqhwkenL.js similarity index 81% rename from frontend/prebuilt/assets/Popup-s69FReak.js rename to frontend/prebuilt/assets/Popup-DqhwkenL.js index 22661328..4e4c8d57 100644 Binary files a/frontend/prebuilt/assets/Popup-s69FReak.js and b/frontend/prebuilt/assets/Popup-DqhwkenL.js differ diff --git a/frontend/prebuilt/assets/Popup-s69FReak.js.map b/frontend/prebuilt/assets/Popup-DqhwkenL.js.map similarity index 97% rename from frontend/prebuilt/assets/Popup-s69FReak.js.map rename to frontend/prebuilt/assets/Popup-DqhwkenL.js.map index d9449f75..bae74710 100644 Binary files a/frontend/prebuilt/assets/Popup-s69FReak.js.map and b/frontend/prebuilt/assets/Popup-DqhwkenL.js.map differ diff --git a/frontend/prebuilt/assets/RepeaterDashboard-4YMEPkkr.js b/frontend/prebuilt/assets/RepeaterDashboard-DHzYsK_s.js similarity index 98% rename from frontend/prebuilt/assets/RepeaterDashboard-4YMEPkkr.js rename to frontend/prebuilt/assets/RepeaterDashboard-DHzYsK_s.js index 6639056c..33dd7df7 100644 Binary files a/frontend/prebuilt/assets/RepeaterDashboard-4YMEPkkr.js and b/frontend/prebuilt/assets/RepeaterDashboard-DHzYsK_s.js differ diff --git a/frontend/prebuilt/assets/RepeaterDashboard-4YMEPkkr.js.map b/frontend/prebuilt/assets/RepeaterDashboard-DHzYsK_s.js.map similarity index 99% rename from frontend/prebuilt/assets/RepeaterDashboard-4YMEPkkr.js.map rename to frontend/prebuilt/assets/RepeaterDashboard-DHzYsK_s.js.map index 456290e3..1cede9d7 100644 Binary files a/frontend/prebuilt/assets/RepeaterDashboard-4YMEPkkr.js.map and b/frontend/prebuilt/assets/RepeaterDashboard-DHzYsK_s.js.map differ diff --git a/frontend/prebuilt/assets/SearchView-4aOlPoPL.js b/frontend/prebuilt/assets/SearchView-CwhbvaNz.js similarity index 97% rename from frontend/prebuilt/assets/SearchView-4aOlPoPL.js rename to frontend/prebuilt/assets/SearchView-CwhbvaNz.js index a6fd9a69..3f5b89f5 100644 Binary files a/frontend/prebuilt/assets/SearchView-4aOlPoPL.js and b/frontend/prebuilt/assets/SearchView-CwhbvaNz.js differ diff --git a/frontend/prebuilt/assets/SearchView-4aOlPoPL.js.map b/frontend/prebuilt/assets/SearchView-CwhbvaNz.js.map similarity index 99% rename from frontend/prebuilt/assets/SearchView-4aOlPoPL.js.map rename to frontend/prebuilt/assets/SearchView-CwhbvaNz.js.map index a70278c2..20440050 100644 Binary files a/frontend/prebuilt/assets/SearchView-4aOlPoPL.js.map and b/frontend/prebuilt/assets/SearchView-CwhbvaNz.js.map differ diff --git a/frontend/prebuilt/assets/SettingsModal-Bje6NQB0.js b/frontend/prebuilt/assets/SettingsModal-BYAS3H3e.js similarity index 99% rename from frontend/prebuilt/assets/SettingsModal-Bje6NQB0.js rename to frontend/prebuilt/assets/SettingsModal-BYAS3H3e.js index 18b597e2..db36769f 100644 Binary files a/frontend/prebuilt/assets/SettingsModal-Bje6NQB0.js and b/frontend/prebuilt/assets/SettingsModal-BYAS3H3e.js differ diff --git a/frontend/prebuilt/assets/SettingsModal-Bje6NQB0.js.map b/frontend/prebuilt/assets/SettingsModal-BYAS3H3e.js.map similarity index 99% rename from frontend/prebuilt/assets/SettingsModal-Bje6NQB0.js.map rename to frontend/prebuilt/assets/SettingsModal-BYAS3H3e.js.map index acb8ebba..ce641911 100644 Binary files a/frontend/prebuilt/assets/SettingsModal-Bje6NQB0.js.map and b/frontend/prebuilt/assets/SettingsModal-BYAS3H3e.js.map differ diff --git a/frontend/prebuilt/assets/VisualizerView-DClWBHTO.js b/frontend/prebuilt/assets/VisualizerView-CgyTsQ_S.js similarity index 99% rename from frontend/prebuilt/assets/VisualizerView-DClWBHTO.js rename to frontend/prebuilt/assets/VisualizerView-CgyTsQ_S.js index b34b338a..ecbd5427 100644 Binary files a/frontend/prebuilt/assets/VisualizerView-DClWBHTO.js and b/frontend/prebuilt/assets/VisualizerView-CgyTsQ_S.js differ diff --git a/frontend/prebuilt/assets/VisualizerView-DClWBHTO.js.map b/frontend/prebuilt/assets/VisualizerView-CgyTsQ_S.js.map similarity index 99% rename from frontend/prebuilt/assets/VisualizerView-DClWBHTO.js.map rename to frontend/prebuilt/assets/VisualizerView-CgyTsQ_S.js.map index d0d7dfab..56713bce 100644 Binary files a/frontend/prebuilt/assets/VisualizerView-DClWBHTO.js.map and b/frontend/prebuilt/assets/VisualizerView-CgyTsQ_S.js.map differ diff --git a/frontend/prebuilt/assets/hooks-DIGY9mWZ.js b/frontend/prebuilt/assets/hooks-DIGY9mWZ.js new file mode 100644 index 00000000..69b24573 Binary files /dev/null and b/frontend/prebuilt/assets/hooks-DIGY9mWZ.js differ diff --git a/frontend/prebuilt/assets/hooks-DtoQuSJA.js.map b/frontend/prebuilt/assets/hooks-DIGY9mWZ.js.map similarity index 90% rename from frontend/prebuilt/assets/hooks-DtoQuSJA.js.map rename to frontend/prebuilt/assets/hooks-DIGY9mWZ.js.map index 573bb41c..b6fc5512 100644 Binary files a/frontend/prebuilt/assets/hooks-DtoQuSJA.js.map and b/frontend/prebuilt/assets/hooks-DIGY9mWZ.js.map differ diff --git a/frontend/prebuilt/assets/hooks-DtoQuSJA.js b/frontend/prebuilt/assets/hooks-DtoQuSJA.js deleted file mode 100644 index e3594f31..00000000 Binary files a/frontend/prebuilt/assets/hooks-DtoQuSJA.js and /dev/null differ diff --git a/frontend/prebuilt/assets/index-CM9d1p2a.js b/frontend/prebuilt/assets/index-BMtXbNEw.js similarity index 99% rename from frontend/prebuilt/assets/index-CM9d1p2a.js rename to frontend/prebuilt/assets/index-BMtXbNEw.js index 5f54af96..8f81542e 100644 Binary files a/frontend/prebuilt/assets/index-CM9d1p2a.js and b/frontend/prebuilt/assets/index-BMtXbNEw.js differ diff --git a/frontend/prebuilt/assets/index-CM9d1p2a.js.map b/frontend/prebuilt/assets/index-BMtXbNEw.js.map similarity index 99% rename from frontend/prebuilt/assets/index-CM9d1p2a.js.map rename to frontend/prebuilt/assets/index-BMtXbNEw.js.map index 25ed43d2..6708e379 100644 Binary files a/frontend/prebuilt/assets/index-CM9d1p2a.js.map and b/frontend/prebuilt/assets/index-BMtXbNEw.js.map differ diff --git a/frontend/prebuilt/assets/leaflet-CetcwTHt.js b/frontend/prebuilt/assets/leaflet-DgeNdCAZ.js similarity index 99% rename from frontend/prebuilt/assets/leaflet-CetcwTHt.js rename to frontend/prebuilt/assets/leaflet-DgeNdCAZ.js index 9517f74d..d639aa1b 100644 Binary files a/frontend/prebuilt/assets/leaflet-CetcwTHt.js and b/frontend/prebuilt/assets/leaflet-DgeNdCAZ.js differ diff --git a/frontend/prebuilt/assets/leaflet-CetcwTHt.js.map b/frontend/prebuilt/assets/leaflet-DgeNdCAZ.js.map similarity index 99% rename from frontend/prebuilt/assets/leaflet-CetcwTHt.js.map rename to frontend/prebuilt/assets/leaflet-DgeNdCAZ.js.map index 01f7bc05..6549b6f8 100644 Binary files a/frontend/prebuilt/assets/leaflet-CetcwTHt.js.map and b/frontend/prebuilt/assets/leaflet-DgeNdCAZ.js.map differ diff --git a/frontend/prebuilt/assets/separator-DiyBVm1x.js b/frontend/prebuilt/assets/separator-C-sbEem-.js similarity index 89% rename from frontend/prebuilt/assets/separator-DiyBVm1x.js rename to frontend/prebuilt/assets/separator-C-sbEem-.js index f28fff43..aa7ef80e 100644 Binary files a/frontend/prebuilt/assets/separator-DiyBVm1x.js and b/frontend/prebuilt/assets/separator-C-sbEem-.js differ diff --git a/frontend/prebuilt/assets/separator-DiyBVm1x.js.map b/frontend/prebuilt/assets/separator-C-sbEem-.js.map similarity index 98% rename from frontend/prebuilt/assets/separator-DiyBVm1x.js.map rename to frontend/prebuilt/assets/separator-C-sbEem-.js.map index f80d1763..dc04c9e5 100644 Binary files a/frontend/prebuilt/assets/separator-DiyBVm1x.js.map and b/frontend/prebuilt/assets/separator-C-sbEem-.js.map differ diff --git a/frontend/prebuilt/index.html b/frontend/prebuilt/index.html index f31ee8f4..4f2d6afb 100644 Binary files a/frontend/prebuilt/index.html and b/frontend/prebuilt/index.html differ diff --git a/tests/test_echo_dedup.py b/tests/test_echo_dedup.py index 772fc9ef..57317d57 100644 --- a/tests/test_echo_dedup.py +++ b/tests/test_echo_dedup.py @@ -331,12 +331,9 @@ class TestDMEchoDetection: assert msg_id is not None broadcasts.clear() - # Duplicate arrives via different path - pkt2, _ = await RawPacketRepository.create(b"dm_in_2", SENDER_TIMESTAMP + 1) - with patch("app.packet_processor.broadcast_event", mock_broadcast): result = await create_dm_message_from_decrypted( - packet_id=pkt2, + packet_id=pkt1, decrypted=decrypted, their_public_key=CONTACT_PUB, our_public_key=OUR_PUB, @@ -388,12 +385,9 @@ class TestDMEchoDetection: assert msg_id is not None broadcasts.clear() - # Duplicate arrives, also with no path - pkt2, _ = await RawPacketRepository.create(b"dm_np_2", SENDER_TIMESTAMP + 1) - with patch("app.packet_processor.broadcast_event", mock_broadcast): result = await create_dm_message_from_decrypted( - packet_id=pkt2, + packet_id=pkt1, decrypted=decrypted, their_public_key=CONTACT_PUB, our_public_key=OUR_PUB, @@ -832,21 +826,19 @@ class TestDirectMessageDirectionDetection: class TestConcurrentDMDedup: - """Test that concurrent DM processing deduplicates via atomic INSERT OR IGNORE. + """Test that concurrent DM processing deduplicates by raw-packet identity. - On a mesh network, the same DM packet can arrive via two RF paths nearly - simultaneously, causing two concurrent calls to create_dm_message_from_decrypted. - SQLite's INSERT OR IGNORE ensures only one message is stored. + On a mesh network, the same DM payload can be observed twice before the first + handler finishes. Both arrivals reuse the same raw_packets row and should end + up attached to a single message. """ @pytest.mark.asyncio - async def test_concurrent_identical_dms_only_store_once(self, test_db, captured_broadcasts): - """Two concurrent create_dm_message_from_decrypted calls with identical content - should result in exactly one stored message.""" + async def test_concurrent_same_packet_dms_only_store_once(self, test_db, captured_broadcasts): + """Two concurrent handlers for the same raw DM packet store one message.""" from app.packet_processor import create_dm_message_from_decrypted - pkt1, _ = await RawPacketRepository.create(b"concurrent_dm_1", SENDER_TIMESTAMP) - pkt2, _ = await RawPacketRepository.create(b"concurrent_dm_2", SENDER_TIMESTAMP + 1) + packet_id, _ = await RawPacketRepository.create(b"concurrent_dm_1", SENDER_TIMESTAMP) decrypted = DecryptedDirectMessage( timestamp=SENDER_TIMESTAMP, @@ -861,7 +853,7 @@ class TestConcurrentDMDedup: with patch("app.packet_processor.broadcast_event", mock_broadcast): results = await asyncio.gather( create_dm_message_from_decrypted( - packet_id=pkt1, + packet_id=packet_id, decrypted=decrypted, their_public_key=CONTACT_PUB, our_public_key=OUR_PUB, @@ -870,7 +862,7 @@ class TestConcurrentDMDedup: outgoing=False, ), create_dm_message_from_decrypted( - packet_id=pkt2, + packet_id=packet_id, decrypted=decrypted, their_public_key=CONTACT_PUB, our_public_key=OUR_PUB, diff --git a/tests/test_key_normalization.py b/tests/test_key_normalization.py index af309679..b558476c 100644 --- a/tests/test_key_normalization.py +++ b/tests/test_key_normalization.py @@ -78,8 +78,8 @@ async def test_null_sender_timestamp_defaults_to_received_at(test_db): @pytest.mark.asyncio -async def test_duplicate_with_same_text_and_null_timestamp_rejected(test_db): - """Two messages with same content and sender_timestamp should be deduped.""" +async def test_direct_messages_with_same_text_and_timestamp_are_allowed(test_db): + """Direct messages no longer share the channel echo dedup index.""" received_at = 600 msg_id1 = await MessageRepository.create( msg_type="PRIV", @@ -97,7 +97,8 @@ async def test_duplicate_with_same_text_and_null_timestamp_rejected(test_db): sender_timestamp=received_at, received_at=received_at, ) - assert msg_id2 is None # duplicate rejected + assert msg_id2 is not None + assert msg_id2 != msg_id1 @pytest.mark.asyncio diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 25c19c57..18a78d0b 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -574,7 +574,7 @@ class TestMigration019: @pytest.mark.asyncio async def test_migration_drops_messages_unique_constraint(self): - """Migration rebuilds messages without UNIQUE, preserving data and dedup index.""" + """Migration rebuilds messages without UNIQUE, preserving data and channel dedup index.""" conn = await aiosqlite.connect(":memory:") conn.row_factory = aiosqlite.Row try: @@ -657,7 +657,7 @@ class TestMigration019: assert rows[1]["type"] == "PRIV" assert rows[1]["outgoing"] == 1 - # Verify dedup index still works (INSERT OR IGNORE should ignore duplicates) + # Verify channel dedup index still works (INSERT OR IGNORE should ignore duplicates) cursor = await conn.execute( "INSERT OR IGNORE INTO messages (type, conversation_key, text, sender_timestamp, received_at) " "VALUES (?, ?, ?, ?, ?)", @@ -665,11 +665,25 @@ class TestMigration019: ) assert cursor.rowcount == 0 # Duplicate ignored + # Direct messages no longer use the shared dedup index. + cursor = await conn.execute( + "INSERT OR IGNORE INTO messages (type, conversation_key, text, sender_timestamp, received_at) " + "VALUES (?, ?, ?, ?, ?)", + ("PRIV", "abc123", "dm text", 2000, 9999), + ) + assert cursor.rowcount == 1 + # Verify dedup index exists cursor = await conn.execute( "SELECT name FROM sqlite_master WHERE name='idx_messages_dedup_null_safe'" ) assert await cursor.fetchone() is not None + + cursor = await conn.execute( + "SELECT sql FROM sqlite_master WHERE name='idx_messages_dedup_null_safe'" + ) + index_sql = (await cursor.fetchone())["sql"] + assert "WHERE type = 'CHAN'" in index_sql finally: await conn.close() @@ -1116,8 +1130,8 @@ class TestMigration039: applied = await run_migrations(conn) - assert applied == 4 - assert await get_version(conn) == 42 + assert applied == 5 + assert await get_version(conn) == 43 cursor = await conn.execute( """ @@ -1186,8 +1200,8 @@ class TestMigration039: applied = await run_migrations(conn) - assert applied == 4 - assert await get_version(conn) == 42 + assert applied == 5 + assert await get_version(conn) == 43 cursor = await conn.execute( """ @@ -1240,8 +1254,8 @@ class TestMigration040: applied = await run_migrations(conn) - assert applied == 3 - assert await get_version(conn) == 42 + assert applied == 4 + assert await get_version(conn) == 43 await conn.execute( """ @@ -1302,8 +1316,8 @@ class TestMigration041: applied = await run_migrations(conn) - assert applied == 2 - assert await get_version(conn) == 42 + assert applied == 3 + assert await get_version(conn) == 43 await conn.execute( """ @@ -1355,8 +1369,8 @@ class TestMigration042: applied = await run_migrations(conn) - assert applied == 1 - assert await get_version(conn) == 42 + assert applied == 2 + assert await get_version(conn) == 43 await conn.execute( """ diff --git a/tests/test_packet_pipeline.py b/tests/test_packet_pipeline.py index 24c56169..5da2cc99 100644 --- a/tests/test_packet_pipeline.py +++ b/tests/test_packet_pipeline.py @@ -896,8 +896,58 @@ class TestCreateDMMessageFromDecrypted: assert message_broadcasts[0]["data"]["outgoing"] is True @pytest.mark.asyncio - async def test_returns_none_for_duplicate_dm(self, test_db, captured_broadcasts): - """create_dm_message_from_decrypted returns None for duplicate DM.""" + async def test_returns_none_for_same_raw_packet_duplicate_dm( + self, test_db, captured_broadcasts + ): + """Reprocessing the same raw DM packet reuses the existing message.""" + from app.decoder import DecryptedDirectMessage + from app.packet_processor import create_dm_message_from_decrypted + + packet_id, _ = await RawPacketRepository.create(b"dm_packet_1", 1700000000) + + decrypted = DecryptedDirectMessage( + timestamp=1700000000, + flags=0, + message="Duplicate DM test", + dest_hash="fa", + src_hash="a1", + ) + + broadcasts, mock_broadcast = captured_broadcasts + + with patch("app.packet_processor.broadcast_event", mock_broadcast): + # First call creates the message + msg_id_1 = await create_dm_message_from_decrypted( + packet_id=packet_id, + decrypted=decrypted, + their_public_key=self.A1B2C3_PUB, + our_public_key=self.FACE12_PUB, + received_at=1700000001, + outgoing=False, + ) + + # Second call for the same packet returns None and does not create a new row + msg_id_2 = await create_dm_message_from_decrypted( + packet_id=packet_id, + decrypted=decrypted, + their_public_key=self.A1B2C3_PUB, + our_public_key=self.FACE12_PUB, + received_at=1700000002, + outgoing=False, + ) + + assert msg_id_1 is not None + assert msg_id_2 is None # Duplicate detected + + # Only one message broadcast + message_broadcasts = [b for b in broadcasts if b["type"] == "message"] + assert len(message_broadcasts) == 1 + + @pytest.mark.asyncio + async def test_allows_same_text_same_second_dms_from_distinct_packets( + self, test_db, captured_broadcasts + ): + """Distinct DM packets with the same text/timestamp both store.""" from app.decoder import DecryptedDirectMessage from app.packet_processor import create_dm_message_from_decrypted @@ -915,7 +965,6 @@ class TestCreateDMMessageFromDecrypted: broadcasts, mock_broadcast = captured_broadcasts with patch("app.packet_processor.broadcast_event", mock_broadcast): - # First call creates the message msg_id_1 = await create_dm_message_from_decrypted( packet_id=packet_id_1, decrypted=decrypted, @@ -924,8 +973,6 @@ class TestCreateDMMessageFromDecrypted: received_at=1700000001, outgoing=False, ) - - # Second call with same content returns None msg_id_2 = await create_dm_message_from_decrypted( packet_id=packet_id_2, decrypted=decrypted, @@ -936,11 +983,16 @@ class TestCreateDMMessageFromDecrypted: ) assert msg_id_1 is not None - assert msg_id_2 is None # Duplicate detected + assert msg_id_2 is not None + assert msg_id_1 != msg_id_2 + + messages = await MessageRepository.get_all( + msg_type="PRIV", conversation_key=self.A1B2C3_PUB.lower(), limit=10 + ) + assert len(messages) == 2 - # Only one message broadcast message_broadcasts = [b for b in broadcasts if b["type"] == "message"] - assert len(message_broadcasts) == 1 + assert len(message_broadcasts) == 2 @pytest.mark.asyncio async def test_links_raw_packet_to_dm_message(self, test_db, captured_broadcasts):