From 851ff062da84527daccb8773a03ae0bee443e696 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Wed, 30 Sep 2026 14:07:00 +0100 Subject: [PATCH] core: compress messages if needed to fit envelope, even if there is no PQ e2ee in connection (#7630) * core: compress messages if needed to fit envelope, even if there is no PQ e2ee in connection * simplify * remove noise * compress messages without PQ too * case * update query plans --------- Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com> --- docs/protocol/simplex-chat.md | 2 +- src/Simplex/Chat/Library/Commands.hs | 2 +- src/Simplex/Chat/Library/Internal.hs | 43 +++++++++++-------- src/Simplex/Chat/Library/Subscriber.hs | 2 +- src/Simplex/Chat/Protocol.hs | 8 ++-- .../SQLite/Migrations/chat_query_plans.txt | 12 ++++++ 6 files changed, 44 insertions(+), 25 deletions(-) diff --git a/docs/protocol/simplex-chat.md b/docs/protocol/simplex-chat.md index 45344d49e0..13e1464881 100644 --- a/docs/protocol/simplex-chat.md +++ b/docs/protocol/simplex-chat.md @@ -90,7 +90,7 @@ The syntax of compressed message is defined by the following ABNF notation: compressedMessage = %s"X" 1*15780 OCTET; compressed message data ``` -Compressed message is required to fit into 13388 bytes, accounting for agent overhead (see Protocol's maxCompressedMsgLength). +Compressed message is required to fit into 15602 bytes, or into 13380 bytes for a connection with PQ encryption, accounting for agent overhead (see Protocol's maxEncodedMsgLength and maxEncodedMsgLengthPQ). The actual JSON message is required to fit into 15610 bytes, accounting for group message forwarding (x.grp.msg.forward) overhead (see Protocol's maxEncodedMsgLength). diff --git a/src/Simplex/Chat/Library/Commands.hs b/src/Simplex/Chat/Library/Commands.hs index 15e3716112..4fe288ca8e 100644 --- a/src/Simplex/Chat/Library/Commands.hs +++ b/src/Simplex/Chat/Library/Commands.hs @@ -3981,7 +3981,7 @@ processChatCommand cxt nm = \case dm <- case gInfo_ of Just (Just gInfo@(GIK g gks)) | useRelays' g -> case relayMemberId_ of - Just relayMemberId -> encodeXMemberConnInfo gInfo relayMemberId profileToSend + Just relayMemberId -> encodeXMemberConnInfo pqSup gInfo relayMemberId profileToSend Nothing -> throwChatError $ CEInternalError "relay group join without target relay memberId" | otherwise -> encodeConnInfoPQ pqSup $ XContact profileToSend (Just $ groupMemberKey gks) (Just xContactId) welcomeSharedMsgId msg_ _ -> diff --git a/src/Simplex/Chat/Library/Internal.hs b/src/Simplex/Chat/Library/Internal.hs index be03d1277b..3d05ab4069 100644 --- a/src/Simplex/Chat/Library/Internal.hs +++ b/src/Simplex/Chat/Library/Internal.hs @@ -2438,6 +2438,19 @@ batchSendConnMessagesB mode _user conn msgFlags msgs_ = do batchSndMessagesJSON :: BatchMode -> NonEmpty (Either ChatError SndMessage) -> [Either ChatError MsgBatch] batchSndMessagesJSON mode = batchMessages mode maxEncodedMsgLength . L.toList +compressToLimit :: MonadError ChatError m => Int -> MsgBody -> m MsgBody +compressToLimit maxLen s + | B.length s <= maxLen = pure s + | B.length s' <= maxLen = pure s' + | otherwise = throwError $ ChatError $ CEException "large compressed body" + where + s' = compressedBatchMsgBody_ s + +compressConnInfo :: PQSupport -> MsgBody -> CM MsgBody +compressConnInfo pqSup = compressToLimit $ case pqSup of + PQSupportOn -> maxEncodedInfoLengthPQ + PQSupportOff -> maxEncodedInfoLength + encodeConnInfo :: MsgEncodingI e => ChatMsgEvent e -> CM ByteString encodeConnInfo = encodeConnInfoPQ PQSupportOff @@ -2446,32 +2459,27 @@ encodeConnInfoPQ pqSup chatMsgEvent = do cxt <- chatStoreCxt let info = ChatMessage {chatVRange = vr cxt, msgId = Nothing, chatMsgEvent} case encodeChatMessage maxEncodedInfoLength info of - ECMEncoded connInfo -> case pqSup of - PQSupportOn | B.length connInfo > maxCompressedInfoLength -> do - let connInfo' = compressedBatchMsgBody_ connInfo - when (B.length connInfo' > maxCompressedInfoLength) $ throwChatError $ CEException "large compressed info" - pure connInfo' - _ -> pure connInfo + ECMEncoded connInfo -> compressConnInfo pqSup connInfo ECMLarge -> throwChatError $ CEException "large info" -- conn-info wrapped as a signed element, so the receiver can verify the signature over the body -encodeSignedConnInfo :: MsgEncodingI e => MsgSigning -> ChatMsgEvent e -> CM ByteString -encodeSignedConnInfo signing chatMsgEvent = do +encodeSignedConnInfo :: MsgEncodingI e => PQSupport -> MsgSigning -> ChatMsgEvent e -> CM ByteString +encodeSignedConnInfo pqSup signing chatMsgEvent = do vr <- chatVersionRange let info = ChatMessage {chatVRange = vr, msgId = Nothing, chatMsgEvent} case encodeChatMessage maxEncodedInfoLength info of - ECMEncoded body -> pure $ encodeBatchElement (Just $ signChatMsgBody signing body) body + ECMEncoded body -> compressConnInfo pqSup $ encodeBatchElement (Just $ signChatMsgBody signing body) body ECMLarge -> throwChatError $ CEException "large signed info" -- signed XMember for a relay-group join: proves the joiner holds the member key it asserts, and carries -- viaRelay = the target relay's memberId inside the signed body so a sibling relay can't accept a replay -encodeXMemberConnInfo :: GroupInfoKeys -> MemberId -> Profile -> CM ByteString -encodeXMemberConnInfo (GIK gInfo@GroupInfo {membership = GroupMember {memberId}} gks) relayMemberId profileToSend = +encodeXMemberConnInfo :: PQSupport -> GroupInfoKeys -> MemberId -> Profile -> CM ByteString +encodeXMemberConnInfo pqSup (GIK gInfo@GroupInfo {membership = GroupMember {memberId}} gks) relayMemberId profileToSend = let memberPrivKey' = memberPrivKey gks xMemberEvt = XMember profileToSend memberId (MemberKey $ C.publicKey memberPrivKey') (Just relayMemberId) bindingData = groupBindingData gInfo memberId (C.publicKey memberPrivKey') signing = MsgSigning CBGroup bindingData KRMember memberPrivKey' - in encodeSignedConnInfo signing xMemberEvt + in encodeSignedConnInfo pqSup signing xMemberEvt deliverMessage :: Connection -> CMEventTag e -> MsgBody -> MessageId -> CM (Int64, PQEncryption) deliverMessage conn cmEventTag msgBody msgId = do @@ -2495,21 +2503,20 @@ deliverMessages msgs = deliverMessagesB $ L.map Right msgs deliverMessagesB :: NonEmpty (Either ChatError ChatMsgReq) -> CM (NonEmpty (Either ChatError ([Int64], PQEncryption))) deliverMessagesB msgReqs = do - msgReqs' <- if any connSupportsPQ msgReqs then liftIO compressBodies else pure msgReqs + msgReqs' <- liftIO compressBodies sent <- L.zipWith prepareBatch msgReqs' <$> withAgent (`sendMessagesB` snd (mapAccumL toAgent Nothing msgReqs')) lift . void $ withStoreBatch' $ \db -> map (updatePQSndEnabled db) (rights . L.toList $ sent) lift . withStoreBatch $ \db -> L.map (bindRight $ createDelivery db) sent where + -- group sends share bodies between connections via VRRef, so the smallest limit applies to the batch + maxLen = if any connSupportsPQ msgReqs then maxEncodedMsgLengthPQ else maxEncodedMsgLength connSupportsPQ = \case Right (Connection {pqSupport = PQSupportOn}, _, _) -> True _ -> False compressBodies = forME msgReqs $ \(conn, msgFlags, (mbr, msgIds)) -> runExceptT $ do mbr' <- case mbr of - VRValue i msgBody | B.length msgBody > maxCompressedMsgLength -> do - let msgBody' = compressedBatchMsgBody_ msgBody - when (B.length msgBody' > maxCompressedMsgLength) $ throwError $ ChatError $ CEException "large compressed message" - pure $ VRValue i msgBody' + VRValue i msgBody -> VRValue i <$> compressToLimit maxLen msgBody v -> pure v pure (conn, msgFlags, (mbr', msgIds)) toAgent prev = \case @@ -3045,7 +3052,7 @@ allowAgentConnectionAsync user conn@Connection {pqSupport} confId gInfo_ msg = d Just gInfo@(GIK g _) | useRelays' g || maxVersion (peerChatVRange conn) >= relayWebCapVersion -> groupMsgSigning False gInfo msg _ -> Nothing dm <- case signing_ of - Just signing -> encodeSignedConnInfo signing msg + Just signing -> encodeSignedConnInfo pqSupport signing msg Nothing -> encodeConnInfoPQ pqSupport msg allowAgentConnectionInfo user conn confId dm diff --git a/src/Simplex/Chat/Library/Subscriber.hs b/src/Simplex/Chat/Library/Subscriber.hs index 1dbf1adcb9..f956c292bc 100644 --- a/src/Simplex/Chat/Library/Subscriber.hs +++ b/src/Simplex/Chat/Library/Subscriber.hs @@ -1247,7 +1247,7 @@ processAgentMessageConn cxt user@User {userId} entity gks_ corrId agentConnId ag withStore' $ \db -> updateConnLinkData db user conn cReq cReqHash groupLinkId chatV pqSup let incognitoProfile = fromLocalProfile <$> incognitoMembershipProfile gInfo profileToSend <- presentUserBadge user incognitoProfile $ userProfileInGroup user gInfo incognitoProfile - dm <- encodeXMemberConnInfo g relayMemberId profileToSend + dm <- encodeXMemberConnInfo pqSup g relayMemberId profileToSend subMode <- chatReadVar subscriptionMode (cmdId, connId') <- prepareAgentJoin user (Just conn) True cReq joinAgentConnectionAsync cmdId True connId' True cReq dm subMode diff --git a/src/Simplex/Chat/Protocol.hs b/src/Simplex/Chat/Protocol.hs index 5f920f8906..723f78d6db 100644 --- a/src/Simplex/Chat/Protocol.hs +++ b/src/Simplex/Chat/Protocol.hs @@ -925,8 +925,8 @@ maxEncodedMsgLength :: Int maxEncodedMsgLength = 15602 -- maxEncodedMsgLength - 2222, see e2eEncUserMsgLength in agent -maxCompressedMsgLength :: Int -maxCompressedMsgLength = 13380 +maxEncodedMsgLengthPQ :: Int +maxEncodedMsgLengthPQ = 13380 maxDecompressedMsgLength :: Int maxDecompressedMsgLength = 65536 @@ -965,8 +965,8 @@ rosterBlobP = do maxEncodedInfoLength :: Int maxEncodedInfoLength = 14694 -maxCompressedInfoLength :: Int -maxCompressedInfoLength = 10968 -- maxEncodedInfoLength - 3726, see e2eEncConnInfoLength in agent +maxEncodedInfoLengthPQ :: Int +maxEncodedInfoLengthPQ = 10968 -- maxEncodedInfoLength - 3726, see e2eEncConnInfoLength in agent data EncodedChatMessage = ECMEncoded ByteString | ECMLarge diff --git a/src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt b/src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt index 152d6c0c8d..046d73936e 100644 --- a/src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt +++ b/src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt @@ -7378,6 +7378,14 @@ Query: SELECT chat_item_id FROM chat_items WHERE group_scope_tag = 'member_suppo Plan: SCAN chat_items +Query: SELECT chat_item_id FROM chat_items WHERE item_sent = 0 AND item_text LIKE '!2 SB-8H8V3-PF8CV-PQMA2-A54M3!%' +Plan: +SCAN chat_items + +Query: SELECT chat_item_id FROM chat_items WHERE item_sent = 0 AND item_text LIKE '!2 SB-Y14GX-Z83KW-0E3PK-DEN7X!%' +Plan: +SCAN chat_items + Query: SELECT chat_item_id FROM chat_items WHERE item_text LIKE '%' || ? || '%' ORDER BY chat_item_id DESC LIMIT 1 Plan: SCAN chat_items @@ -7676,6 +7684,10 @@ Query: SELECT note_folder_id FROM note_folders WHERE user_id = ? LIMIT 1 Plan: SEARCH note_folders USING COVERING INDEX note_folders_user_id (user_id=?) +Query: SELECT purchase_key FROM badge_code_redemptions +Plan: +SCAN badge_code_redemptions + Query: SELECT quota_err_counter FROM connections WHERE user_id = ? AND connection_id = ? Plan: SEARCH connections USING INTEGER PRIMARY KEY (rowid=?)