From 0c1ffa65bc46340af0a75c8aa0c76cac0d26d36c Mon Sep 17 00:00:00 2001 From: Evgeny Date: Thu, 8 Oct 2026 15:06:18 +0100 Subject: [PATCH 1/9] core: group fixes/improvements (#7660) * core: group fixes/improvements * Large messages dropped by relays and hosts (the same 15602 limit was applied, without accounting for forwarding header) * Members below v18 did not receive forwarded messages or history in p2p groups. * Join confirmations near the size limit could fail. * SMP servers are limited to 2 host names each. * simplify * remove legacy forwarding and history support * only restrict p2p groups --------- Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com> --- apps/ios/Shared/Model/AppAPITypes.swift | 5 ++ .../chat/simplex/common/model/SimpleXAPI.kt | 4 ++ .../moko-resources/base/strings.xml | 1 + src/Simplex/Chat/Library/Internal.hs | 17 +++--- src/Simplex/Chat/Library/Subscriber.hs | 10 ++-- src/Simplex/Chat/Operators.hs | 6 +- src/Simplex/Chat/Protocol.hs | 12 +++- src/Simplex/Chat/Store/Delivery.hs | 2 +- tests/ChatTests/Direct.hs | 7 +++ tests/ChatTests/Groups.hs | 59 ++++++++++++++++++- tests/MessageBatching.hs | 7 +++ 11 files changed, 112 insertions(+), 18 deletions(-) diff --git a/apps/ios/Shared/Model/AppAPITypes.swift b/apps/ios/Shared/Model/AppAPITypes.swift index eadca9f71e..26438bc60e 100644 --- a/apps/ios/Shared/Model/AppAPITypes.swift +++ b/apps/ios/Shared/Model/AppAPITypes.swift @@ -1931,10 +1931,13 @@ enum UserServersError: Decodable { case storageMissing(protocol: ServerProtocol, user: UserRef?) case proxyMissing(protocol: ServerProtocol, user: UserRef?) case duplicateServer(protocol: ServerProtocol, duplicateServer: String, duplicateHost: String) + case tooManyHosts(protocol: ServerProtocol, tooManyHostsServer: String) case duplicateChatRelayAddress(duplicateChatRelay: String, duplicateAddress: String) var globalError: String? { switch self { + case .tooManyHosts: + return globalSMPError case let .noServers(`protocol`, _): switch `protocol` { case .smp: return globalSMPError @@ -1977,6 +1980,8 @@ enum UserServersError: Decodable { } else { return text } + case let .tooManyHosts(.smp, server): + return String.localizedStringWithFormat(NSLocalizedString("Server %@ has more than two host names.", comment: "servers error"), serverHostname(server)) default: return nil } diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt index 706ee713fc..d879df1d27 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt @@ -4886,6 +4886,7 @@ sealed class UserServersError { @Serializable @SerialName("storageMissing") data class StorageMissing(val protocol: ServerProtocol, val user: UserRef?): UserServersError() @Serializable @SerialName("proxyMissing") data class ProxyMissing(val protocol: ServerProtocol, val user: UserRef?): UserServersError() @Serializable @SerialName("duplicateServer") data class DuplicateServer(val protocol: ServerProtocol, val duplicateServer: String, val duplicateHost: String): UserServersError() + @Serializable @SerialName("tooManyHosts") data class TooManyHosts(val protocol: ServerProtocol, val tooManyHostsServer: String): UserServersError() @Serializable @SerialName("duplicateChatRelayAddress") data class DuplicateChatRelayAddress(val duplicateChatRelay: String, val duplicateAddress: String): UserServersError() val globalError: String? @@ -4901,6 +4902,7 @@ sealed class UserServersError { is StorageMissing -> this.protocol is ProxyMissing -> this.protocol is DuplicateServer -> this.protocol + is TooManyHosts -> this.protocol is DuplicateChatRelayAddress -> null } @@ -4916,6 +4918,8 @@ sealed class UserServersError { is ProxyMissing -> this.user?.let { "${userStr(it)} ${generalGetString(MR.strings.no_message_servers_configured_for_private_routing)}" } ?: generalGetString(MR.strings.no_message_servers_configured_for_private_routing) + is TooManyHosts -> String.format(generalGetString(MR.strings.server_has_too_many_hosts), ServerAddress.parseServerAddress(this.tooManyHostsServer)?.hostnames?.firstOrNull() ?: this.tooManyHostsServer) + else -> null } } else { diff --git a/apps/multiplatform/common/src/commonMain/moko-resources/base/strings.xml b/apps/multiplatform/common/src/commonMain/moko-resources/base/strings.xml index 8a8aad0cfa..2d0db3e038 100644 --- a/apps/multiplatform/common/src/commonMain/moko-resources/base/strings.xml +++ b/apps/multiplatform/common/src/commonMain/moko-resources/base/strings.xml @@ -140,6 +140,7 @@ No message servers. No servers to receive messages. No servers for private message routing. + Server %1$s has more than two host names. No media & file servers. No servers to send files. No servers to receive files. diff --git a/src/Simplex/Chat/Library/Internal.hs b/src/Simplex/Chat/Library/Internal.hs index 1d4cf969a8..effa943cc7 100644 --- a/src/Simplex/Chat/Library/Internal.hs +++ b/src/Simplex/Chat/Library/Internal.hs @@ -1414,7 +1414,7 @@ sendHistory user gInfo@GroupInfo {membership} m@GroupMember {activeConn = Just c -- (regular groups only; never channels) is an authored element -- all batch together in order. let fwdEls = map (uncurry encodeFwdElement) (concat fwdMsgsByItem) welcomeEl <- welcomeElement - let (batches, dropped) = batchElements maxEncodedMsgLength (fwdEls <> maybe [] (: []) welcomeEl) + let (batches, dropped) = batchElements maxForwardBatchLength (fwdEls <> maybe [] (: []) welcomeEl) when (dropped > 0) $ toView $ CEvtChatErrors [ChatError $ CEInternalError ("sendHistory: dropped " <> show dropped <> " oversized history messages")] forM_ batches $ \body -> void $ withAgent $ \a -> sendMessages a [(aConnId conn, PQEncOff, MsgFlags False, VRValue Nothing body)] @@ -1552,9 +1552,7 @@ sendHistory user gInfo@GroupInfo {membership} m@GroupMember {activeConn = Just c pure $ map ((,) fwd) (contentVM : fileDescrVMs) memberShortenedName :: GroupMember -> ContactName -memberShortenedName GroupMember {memberProfile = LocalProfile {displayName}} - | T.length displayName <= 16 = displayName - | otherwise = T.take 16 displayName `T.snoc` '…' +memberShortenedName GroupMember {memberProfile = LocalProfile {displayName}} = fwdMemberName displayName -- the description proof travels on the last part, so that part leaves room for it badgeDescrPartSize :: Int @@ -2457,9 +2455,10 @@ compressToLimit maxLen s s' = compressedBatchMsgBody_ s compressConnInfo :: PQSupport -> MsgBody -> CM MsgBody -compressConnInfo pqSup = compressToLimit $ case pqSup of - PQSupportOn -> maxEncodedInfoLengthPQ - PQSupportOff -> maxEncodedInfoLength +compressConnInfo pqSup = compressToLimit $ e2eEncConnInfoLength pqSup - 2 - maxReplyQueueFraming + +maxReplyQueueFraming :: Int +maxReplyQueueFraming = 256 encodeConnInfo :: MsgEncodingI e => ChatMsgEvent e -> CM ByteString encodeConnInfo = encodeConnInfoPQ PQSupportOff @@ -2468,7 +2467,7 @@ encodeConnInfoPQ :: MsgEncodingI e => PQSupport -> ChatMsgEvent e -> CM ByteStri encodeConnInfoPQ pqSup chatMsgEvent = do cxt <- chatStoreCxt let info = ChatMessage {chatVRange = vr cxt, msgId = Nothing, chatMsgEvent} - case encodeChatMessage maxEncodedInfoLength info of + case encodeChatMessage maxEncodedProfileMsgLength info of ECMEncoded connInfo -> compressConnInfo pqSup connInfo ECMLarge -> throwChatError $ CEException "large info" @@ -2477,7 +2476,7 @@ encodeSignedConnInfo :: MsgEncodingI e => PQSupport -> MsgSigning -> ChatMsgEven encodeSignedConnInfo pqSup signing chatMsgEvent = do vr <- chatVersionRange let info = ChatMessage {chatVRange = vr, msgId = Nothing, chatMsgEvent} - case encodeChatMessage maxEncodedInfoLength info of + case encodeChatMessage maxEncodedProfileMsgLength info of ECMEncoded body -> compressConnInfo pqSup $ encodeBatchElement (Just $ signChatMsgBody signing body) body ECMLarge -> throwChatError $ CEException "large signed info" diff --git a/src/Simplex/Chat/Library/Subscriber.hs b/src/Simplex/Chat/Library/Subscriber.hs index 3f0e53fd8a..304aa15500 100644 --- a/src/Simplex/Chat/Library/Subscriber.hs +++ b/src/Simplex/Chat/Library/Subscriber.hs @@ -3937,7 +3937,7 @@ processAgentMessageConn cxt user@User {userId} entity gks_ corrId agentConnId ag createInternalChatItem user (CDDirectRcv ct) (CIRcvConnEvent RCEVerificationCodeReset) Nothing xGrpMsgForward :: GroupInfoKeys -> Maybe GroupChatScopeInfo -> GroupMember -> GrpMsgForward -> ParsedMsg 'Json -> UTCTime -> CM () - xGrpMsgForward g@(GIK gInfo _) scopeInfo m@GroupMember {localDisplayName} GrpMsgForward {fwdSender, fwdBrokerTs = msgTs} parsedMsg@(ParsedMsg _ _ chatMsg@ChatMessage {chatMsgEvent}) brokerTs = do + xGrpMsgForward g@(GIK gInfo@GroupInfo {membership} _) scopeInfo m@GroupMember {localDisplayName} GrpMsgForward {fwdSender, fwdBrokerTs = msgTs} parsedMsg@(ParsedMsg _ _ chatMsg@ChatMessage {chatMsgEvent}) brokerTs = do unless (isMemberGrpFwdRelay gInfo m) $ throwChatError (CEGroupContactRole localDisplayName) case fwdSender of FwdMember memberId memberName -> do @@ -3945,6 +3945,8 @@ processAgentMessageConn cxt user@User {userId} entity gks_ corrId agentConnId ag let allowCreate = toCMEventTag chatMsgEvent /= XGrpLeave_ withStore (\db -> getCreateUnknownGMByMemberId db cxt user gInfo memberId memberName unknownRole allowCreate) >>= \case Just (author, unknown) + | not (useRelays' gInfo) && groupMemberId' author == groupMemberId' membership -> + messageError $ "x.grp.msg.forward: content attributed to own membership, forwarder " <> tshow (groupMemberId' m) <> ", event " <> tshow (toCMEventTag chatMsgEvent) | memberRemoved author -> logInfo $ "x.grp.msg.forward: ignoring content from removed member, group " <> tshow (groupId' gInfo) <> ", member " <> safeDecodeUtf8 (strEncode memberId) <> ", event " <> tshow (toCMEventTag chatMsgEvent) | not (useRelays' gInfo) && not (expectedForwarder author) -> @@ -4160,7 +4162,7 @@ runDeliveryTaskWorker a deliveryKey Worker {doWork} = do withStore' $ \db -> setDeliveryTaskErrStatus db (deliveryTaskId task) "relay inactive" | otherwise -> withWorkItems a doWork (withStore' $ \db -> getNextDeliveryTasks db gInfo task) $ \nextTasks -> do - let (body_, acceptedTasks, largeTasks) = batchDeliveryTasks1 (vr cxt) maxEncodedMsgLength nextTasks + let (body_, acceptedTasks, largeTasks) = batchDeliveryTasks1 (vr cxt) maxForwardBatchLength nextTasks senderGMIds = S.toList . S.fromList $ map (\MessageDeliveryTask {senderGMId} -> senderGMId) acceptedTasks withStore' $ \db -> do forM_ body_ $ \body -> createMsgDeliveryJob db gInfo jobScope senderGMIds body @@ -4290,8 +4292,8 @@ runDeliveryJobWorker a deliveryKey Worker {doWork} = do else do -- all members' profiles disseminate; privileged key/role come from the roster, not here let (encoderErrs, validLabeled) = partitionEithers [(\bs -> (s, bs)) <$> encodeMemberNew (vr cxt) gInfo s | (s, _) <- senders] - (extBody', inBody, overflowLabeled, large1) = batchProfilesWithBody maxEncodedMsgLength body validLabeled - (overflowBatches', large2) = batchProfiles maxEncodedMsgLength overflowLabeled + (extBody', inBody, overflowLabeled, large1) = batchProfilesWithBody maxForwardBatchLength body validLabeled + (overflowBatches', large2) = batchProfiles maxForwardBatchLength overflowLabeled packerErrs = [ChatError (CEInternalError $ "oversized profile element for member " <> show (groupMemberId' s)) | s <- large1 <> large2] allErrs = encoderErrs <> packerErrs unless (null allErrs) $ do diff --git a/src/Simplex/Chat/Operators.hs b/src/Simplex/Chat/Operators.hs index a216bbb1fc..9ec5d0f48d 100644 --- a/src/Simplex/Chat/Operators.hs +++ b/src/Simplex/Chat/Operators.hs @@ -525,6 +525,7 @@ data UserServersError | USEStorageMissing {protocol :: AProtocolType, user :: Maybe User} | USEProxyMissing {protocol :: AProtocolType, user :: Maybe User} | USEDuplicateServer {protocol :: AProtocolType, duplicateServer :: Text, duplicateHost :: TransportHost} + | USETooManyHosts {protocol :: AProtocolType, tooManyHostsServer :: Text} | USEDuplicateChatRelayAddress {duplicateChatRelay :: Text, duplicateAddress :: ShortLinkContact} deriving (Show) @@ -547,13 +548,16 @@ validateUserServers curr others = (currUserErrs <> concatMap otherUserErrs other noServers cond = not $ any srvEnabled $ userServers p $ filter cond uss srvEnabled (AUS _ UserServer {deleted, enabled}) = enabled && not deleted serverErrs :: (UserServersClass u, ProtocolTypeI p, UserProtocol p) => SProtocolType p -> [u] -> [UserServersError] - serverErrs p uss = mapMaybe duplicateErr_ srvs + serverErrs p uss = mapMaybe duplicateErr_ srvs <> hostsErrs where p' = AProtocolType p srvs = filter (\(AUS _ UserServer {deleted}) -> not deleted) $ userServers p uss duplicateErr_ (AUS _ srv@UserServer {server}) = USEDuplicateServer p' (safeDecodeUtf8 $ strEncode server) <$> find (`S.member` duplicateHosts) (srvHost srv) + hostsErrs = case p of + SPSMP -> [USETooManyHosts p' (safeDecodeUtf8 $ strEncode server) | AUS _ srv@UserServer {server} <- srvs, L.length (srvHost srv) > 2] + _ -> [] duplicateHosts = snd $ foldl' addDuplicate (S.empty, S.empty) allHosts allHosts = concatMap (\(AUS _ srv) -> L.toList $ srvHost srv) srvs userServers :: (UserServersClass u, UserProtocol p) => SProtocolType p -> [u] -> [AUserServer p] diff --git a/src/Simplex/Chat/Protocol.hs b/src/Simplex/Chat/Protocol.hs index 723f78d6db..01cb868e83 100644 --- a/src/Simplex/Chat/Protocol.hs +++ b/src/Simplex/Chat/Protocol.hs @@ -924,6 +924,14 @@ $(JQ.deriveJSON defaultJSON ''MsgContainer) maxEncodedMsgLength :: Int maxEncodedMsgLength = 15602 +maxForwardBatchLength :: Int +maxForwardBatchLength = maxEncodedMsgLength + 161 + +fwdMemberName :: ContactName -> ContactName +fwdMemberName displayName + | T.length displayName <= 16 = displayName + | otherwise = T.take 16 displayName `T.snoc` '…' + -- maxEncodedMsgLength - 2222, see e2eEncUserMsgLength in agent maxEncodedMsgLengthPQ :: Int maxEncodedMsgLengthPQ = 13380 @@ -965,8 +973,8 @@ rosterBlobP = do maxEncodedInfoLength :: Int maxEncodedInfoLength = 14694 -maxEncodedInfoLengthPQ :: Int -maxEncodedInfoLengthPQ = 10968 -- maxEncodedInfoLength - 3726, see e2eEncConnInfoLength in agent +maxEncodedProfileMsgLength :: Int +maxEncodedProfileMsgLength = 16384 data EncodedChatMessage = ECMEncoded ByteString | ECMLarge diff --git a/src/Simplex/Chat/Store/Delivery.hs b/src/Simplex/Chat/Store/Delivery.hs index add8c64e7f..c80264da7a 100644 --- a/src/Simplex/Chat/Store/Delivery.hs +++ b/src/Simplex/Chat/Store/Delivery.hs @@ -156,7 +156,7 @@ getMsgDeliveryTask_ db taskId = toTask ((Only taskId') :. jobScopeRow :. (senderGMId, senderMemberId, senderMemberName, brokerTs, Binary msgBody, chatBinding_, sigs_, BI showGroupAsSender)) = case (toJobScope_ jobScopeRow, J.eitherDecodeStrict' msgBody) of (Just jobScope, Right chatMsg) -> - let fwdSender = if showGroupAsSender && isNothing chatBinding_ then FwdChannel else FwdMember senderMemberId senderMemberName + let fwdSender = if showGroupAsSender && isNothing chatBinding_ then FwdChannel else FwdMember senderMemberId (fwdMemberName senderMemberName) -- Re-parsed from msg_body: validates stored content against current code. -- Signed: original bytes preserved (re-encoding would invalidate signature). -- Unsigned: re-encoded from parsed ChatMessage on forward (sanitizes content). diff --git a/tests/ChatTests/Direct.hs b/tests/ChatTests/Direct.hs index b95d738589..930a0fa22d 100644 --- a/tests/ChatTests/Direct.hs +++ b/tests/ChatTests/Direct.hs @@ -1158,6 +1158,13 @@ testGetSetSMPServers = alice <## " SMP servers" alice <## " smp://2345-w==@smp2.example.im" alice <## " smp://3456-w==@smp3.example.im:5224" + alice #$> ("/smp smp://2345-w==@smp2.example.im,smp4.example.im", id, "ok") + alice ##> "/smp smp://2345-w==@smp2.example.im,smp4.example.im,smp5.example.im" + alice <##. "bad chat command: user servers validation error(s): [USETooManyHosts" + alice ##> "/smp" + alice <## "Your servers" + alice <## " SMP servers" + alice <## " smp://2345-w==@smp2.example.im,smp4.example.im" testTestSMPServerConnection :: HasCallStack => TestParams -> IO () testTestSMPServerConnection = diff --git a/tests/ChatTests/Groups.hs b/tests/ChatTests/Groups.hs index 39e50dc16b..3608e5f7e9 100644 --- a/tests/ChatTests/Groups.hs +++ b/tests/ChatTests/Groups.hs @@ -38,7 +38,7 @@ import Simplex.Chat.Messages (CIMention (..), CIMentionMember (..), ChatItemId) import Simplex.Chat.Messages.Batch (encodeBinaryBatch, encodeFwdElement) import Simplex.Chat.Messages.CIContent (publicGroupNoE2EText) import Simplex.Chat.Options -import Simplex.Chat.Protocol (ChatMessage (ChatMessage), ChatMsgEvent (XGrpMemNew, XMsgUpdate, XMsgNew, XMsgDel), FwdSender (FwdMember, FwdChannel), GrpMsgForward (GrpMsgForward), MsgContainer (..), MsgMention (..), MsgContent (..), VerifiedMsg (VMUnsigned), mcSimple, msgContentText) +import Simplex.Chat.Protocol (ChatMessage (ChatMessage), ChatMsgEvent (XGrpMemNew, XInfo, XMsgUpdate, XMsgNew, XMsgDel), FwdSender (FwdMember, FwdChannel), GrpMsgForward (GrpMsgForward), MsgContainer (..), MsgMention (..), MsgContent (..), VerifiedMsg (VMUnsigned), mcSimple, msgContentText) import Simplex.Chat.Types import Simplex.Chat.Types.MemberRelations (MemberRelation (..), getRelation, setRelation) import Simplex.Chat.Types.Shared (GroupMemberRole (..), GroupAcceptance (..)) @@ -119,6 +119,7 @@ chatGroupTests = do xit "create and join group when clients go offline" testGroupAsync describe "group links" $ do it "create group link, join via group link" testGroupLink + it "join via group link with group profile near the size limit" testGroupLinkLargeGroupProfile it "invitees were previously connected as contacts" testGroupLinkInviteesWereConnected it "all members were previously connected as contacts" testGroupLinkAllMembersWereConnected it "delete group, re-join via same link" testGroupLinkDeleteGroupRejoin @@ -176,6 +177,7 @@ chatGroupTests = do it "manually accept contact with group member incognito" testMemberContactAcceptIncognito describe "group message forwarding" $ do it "forward messages between invitee and introduced (x.msg.new)" testGroupMsgForwardMessage + it "reject forwarded content attributed to own membership" testGroupMsgForwardOwnMembershipRejected it "forward batched messages" testGroupMsgForwardBatched it "forward reports to moderators, don't forward to members (x.msg.new, MCReport)" testGroupMsgForwardReport it "deduplicate forwarded messages" testGroupMsgForwardDeduplicate @@ -3042,6 +3044,39 @@ testPlanGroupLinkLeaveRejoin = bob <## "group link: known group #team_1" bob <## "use #team_1 to send messages" +testGroupLinkLargeGroupProfile :: HasCallStack => TestParams -> IO () +testGroupLinkLargeGroupProfile = + testChatOpts2 testOptsNoFullLinks aliceProfile bobProfile $ + \alice bob -> do + threadDelay 100000 + alice ##> "/g team" + alice <## "group #team is created" + alice <## "to add members use /a team or /create link #team" + alice ##> "/set history #team off" + alice <## "updated group preferences:" + alice <## "Recent history: off" + withCCTransaction alice $ \db -> + DB.execute db "UPDATE group_profiles SET description = ? WHERE display_name = ?" (T.pack welcome, "team" :: T.Text) + alice ##> "/create link #team" + gLink <- getGroupLink_ alice "team" GRMember True + alice ("/c " <> gLink) + bob <## "connection request sent!" + alice <## "bob (Bob): accepting request to join group #team..." + concurrentlyN_ + [ alice <## "#team: bob joined the group", + do + bob <## "#team: joining the group..." + bob <## "#team: you joined the group" + ] + alice #> "#team hello" + let waitHello = do + l <- getTermLine bob + if "alice> hello" `isSuffixOf` l then pure () else waitHello + waitHello + where + welcome = unwords $ replicate 1775 "welcome" + testGroupLink :: HasCallStack => TestParams -> IO () testGroupLink = testChatOpts3 testOptsNoFullLinks aliceProfile bobProfile cathProfile $ @@ -5399,6 +5434,28 @@ testGroupMsgForwardMessage = cath <# "#team bob> hi there [>>]" cath <# "#team hey team" +testGroupMsgForwardOwnMembershipRejected :: HasCallStack => TestParams -> IO () +testGroupMsgForwardOwnMembershipRejected = + testChat2 aliceProfile bobProfile $ + \alice bob -> do + createGroup2 "team" alice bob + threadDelay 1000000 + [Only bobMemId] <- withCCTransaction bob $ \db -> + DB.query db "SELECT member_id FROM group_members WHERE member_category = ?" (Only ("user" :: T.Text)) :: IO [Only ByteString] + connId <- relayConnIdToMember alice "bob" + ts <- getCurrentTime + let ChatController {smpAgent = aliceAgent} = chatController alice + forgedProfile = (bobProfile :: Profile) {displayName = "mallory", fullName = "Mallory"} + chatMsg = ChatMessage chatInitialVRange (Just $ SharedMsgId "forged_info1") (XInfo forgedProfile Nothing) + fwd = GrpMsgForward (FwdMember (MemberId bobMemId) "bob") ts + body = encodeBinaryBatch [encodeFwdElement fwd (VMUnsigned chatMsg)] + sent <- runExceptT $ sendMessages aliceAgent [(connId, PQEncOff, MsgFlags False, vrValue body)] + either (fail . show) (const $ pure ()) sent + bob <##. "error: x.grp.msg.forward: content attributed to own membership" + bob ##> "/p" + bob <## "user profile: bob (Bob)" + bob <## "use /p [] to change it" + testGroupMsgForwardBatched :: HasCallStack => TestParams -> IO () testGroupMsgForwardBatched = testChat3 aliceProfile bobProfile cathProfile $ diff --git a/tests/MessageBatching.hs b/tests/MessageBatching.hs index 01515fb6a3..71cf2d5f49 100644 --- a/tests/MessageBatching.hs +++ b/tests/MessageBatching.hs @@ -33,6 +33,7 @@ import Simplex.Chat.Protocol GrpMsgForward (GrpMsgForward), MsgContent (MCText), VerifiedMsg (VMUnsigned), + fwdMemberName, maxBatchElementCount, maxEncodedMsgLength, mcSimple, @@ -49,6 +50,7 @@ batchingTests = describe "message batching tests" $ do it "splits a batch that exceeds the element count limit" testBatchElementCountLimit it "does not create a relay delivery body when every task is oversized" testRelayBatchAllLarge it "classifies a task that fits raw but not as a framed singleton as large" testRelayBatchSingletonOverflow + it "shortens forwarded member names" testFwdMemberName instance IsString SndMessage where fromString s = SndMessage {msgId, sharedMsgId = SharedMsgId "", msgBody = s', signedMsg_ = Nothing} @@ -190,6 +192,11 @@ testRelayBatchSingletonOverflow = do map deliveryTaskId accepted `shouldBe` [] map deliveryTaskId large `shouldBe` [1] +testFwdMemberName :: IO () +testFwdMemberName = do + fwdMemberName "sixteen_chars_ab" `shouldBe` "sixteen_chars_ab" + fwdMemberName "seventeen_chars_a" `shouldBe` "seventeen_chars_…" + runBatcherTest :: BatchMode -> Int -> [SndMessage] -> [ChatError] -> [ByteString] -> Spec runBatcherTest mode maxLen msgs expectedErrors expectedBatches = it From 96970373b03766dd7406afe9c1292c975526f4d4 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Thu, 8 Oct 2026 16:31:29 +0100 Subject: [PATCH 2/9] core: add status when switching queue (#7664) * core: add status when switching queue * fix --------- Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com> --- apps/ios/SimpleXChat/APITypes.swift | 1 + .../commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt | 3 ++- cabal.project | 2 +- scripts/nix/sha256map.nix | 2 +- src/Simplex/Chat/Messages.hs | 4 ++-- src/Simplex/Chat/View.hs | 1 + 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/ios/SimpleXChat/APITypes.swift b/apps/ios/SimpleXChat/APITypes.swift index 75d66ebb6f..ecf389563e 100644 --- a/apps/ios/SimpleXChat/APITypes.swift +++ b/apps/ios/SimpleXChat/APITypes.swift @@ -598,6 +598,7 @@ public enum RcvSwitchStatus: String, Decodable, Hashable { case sendingQADD = "sending_qadd" case sendingQUSE = "sending_quse" case receivedMessage = "received_message" + case receivedQEND = "received_qend" } public struct SndQueueInfo: Decodable, Hashable { diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt index d879df1d27..2672d0e996 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt @@ -7573,7 +7573,8 @@ enum class RcvSwitchStatus { @SerialName("switch_started") SwitchStarted, @SerialName("sending_qadd") SendingQADD, @SerialName("sending_quse") SendingQUSE, - @SerialName("received_message") ReceivedMessage + @SerialName("received_message") ReceivedMessage, + @SerialName("received_qend") ReceivedQEND } @Serializable diff --git a/cabal.project b/cabal.project index ed446d492c..e5f2c65eda 100644 --- a/cabal.project +++ b/cabal.project @@ -21,7 +21,7 @@ constraints: zip +disable-bzip2 +disable-zstd source-repository-package type: git location: https://github.com/simplex-chat/simplexmq.git - tag: c0a1b7bd2d79c4af5163d9a863cb0780f556d7a9 + tag: e24d537eaaa32230eec1fefcd957d474d7b8d4d5 source-repository-package type: git diff --git a/scripts/nix/sha256map.nix b/scripts/nix/sha256map.nix index 1795f6e865..d6c932df39 100644 --- a/scripts/nix/sha256map.nix +++ b/scripts/nix/sha256map.nix @@ -1,5 +1,5 @@ { - "https://github.com/simplex-chat/simplexmq.git"."c0a1b7bd2d79c4af5163d9a863cb0780f556d7a9" = "0l10gc46l1i1wwalapg4wa0y1x3j4l4465242701k9lvvgywjh9z"; + "https://github.com/simplex-chat/simplexmq.git"."e24d537eaaa32230eec1fefcd957d474d7b8d4d5" = "1ximwqi1jaiq1ml05y0mcsrgvabhp5is1ry2h91w5jjarjb1a5fj"; "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38"; "https://github.com/simplex-chat/direct-sqlcipher.git"."2330df3dc8c4b660674d3c02d76bb86f66c2abee" = "11pfj1vqsjdl26nplkr2l6z1xj7cn153qijj5gknmlzq3krn7x1i"; "https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl"; diff --git a/src/Simplex/Chat/Messages.hs b/src/Simplex/Chat/Messages.hs index 3c6dca2ac8..50035b55be 100644 --- a/src/Simplex/Chat/Messages.hs +++ b/src/Simplex/Chat/Messages.hs @@ -55,7 +55,7 @@ import Simplex.Messaging.Crypto.File (CryptoFile (..)) import qualified Simplex.Messaging.Crypto.File as CF import Simplex.Messaging.Encoding.String import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON, parseAll, sumTypeJSON) -import Simplex.Messaging.Protocol (BlockingInfo, MsgBody, XFTPServer) +import Simplex.Messaging.Protocol (BlockingInfo, MsgBody, XFTPServer, unMsgId) import Simplex.Messaging.Util (eitherToMaybe, safeDecodeUtf8, (<$?>)) data ChatType = CTDirect | CTGroup | CTLocal | CTContactRequest | CTContactConnection @@ -1228,7 +1228,7 @@ msgMetaToJson MsgMeta {integrity, recipient = (rcvId, rcvTs), broker = (serverId { integrity = (decodeLatin1 . strEncode) integrity, rcvId, rcvTs, - serverId = (decodeLatin1 . B64.encode) serverId, + serverId = (decodeLatin1 . B64.encode . unMsgId) serverId, serverTs, sndId } diff --git a/src/Simplex/Chat/View.hs b/src/Simplex/Chat/View.hs index 1c62bc267b..aa509d49bf 100644 --- a/src/Simplex/Chat/View.hs +++ b/src/Simplex/Chat/View.hs @@ -1961,6 +1961,7 @@ viewRcvQueuesInfo = plain . T.intercalate ", " . map showQueueInfo RSSendingQADD -> "switch started" RSSendingQUSE -> "switch confirmed" RSReceivedMessage -> "switch secured" + RSReceivedQEND -> "switch completed" viewSndQueuesInfo :: [SndQueueInfo] -> StyledString viewSndQueuesInfo = plain . T.intercalate ", " . map showQueueInfo From 64d68ff7109d3f633ed8d1ebdd1833fcd3df736d Mon Sep 17 00:00:00 2001 From: Evgeny Date: Thu, 8 Oct 2026 17:11:46 +0100 Subject: [PATCH 3/9] core: fix test (#7666) Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com> --- tests/ChatClient.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ChatClient.hs b/tests/ChatClient.hs index fa6d0f37d3..4a052d7da5 100644 --- a/tests/ChatClient.hs +++ b/tests/ChatClient.hs @@ -641,7 +641,7 @@ smpServerCfg p = maxJournalMsgCount = 24, maxJournalStateLines = 4, queueIdBytes = 24, - msgIdBytes = 6, + msgIdBytes = 24, serverStoreCfg = SSCMemory Nothing, -- $ Just StorePaths {storeLogFile = "tmp/smp-server-store.log", storeMsgsFile = Just "tmp/smp-server-messages.log"}, storeNtfsFile = Nothing, allowNewQueues = True, From a24a3b19116a91e51d7ba3f376b67d3dcb93f789 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Thu, 8 Oct 2026 18:10:37 +0100 Subject: [PATCH 4/9] website: redirect page (#7667) * website: redirect page * script --------- Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com> --- website/src/wefunder.html | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 website/src/wefunder.html diff --git a/website/src/wefunder.html b/website/src/wefunder.html new file mode 100644 index 0000000000..7fc7623d69 --- /dev/null +++ b/website/src/wefunder.html @@ -0,0 +1,23 @@ +--- +templateEngineOverride: njk +--- + + + + + + + + Invest in SimpleX Chat + + + + + + + + + + + + From 9c9c86f9c07620dab852fcac4b095846e30d3003 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Thu, 8 Oct 2026 22:53:59 +0100 Subject: [PATCH 5/9] core, ui: fix profiles, add error alerts (#7663) * core, ui: fix profiles, add error alerts * simplify * query plans --------- Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com> --- apps/ios/Shared/Model/ChatModel.swift | 2 + apps/ios/Shared/Model/SimpleXAPI.swift | 8 +- .../Views/UserSettings/UserAddressView.swift | 22 +++- .../Views/UserSettings/UserProfile.swift | 9 +- apps/ios/SimpleXChat/ChatTypes.swift | 3 +- .../chat/simplex/common/model/ChatModel.kt | 2 +- .../chat/simplex/common/model/SimpleXAPI.kt | 11 +- .../views/usersettings/UserAddressView.kt | 1 + .../moko-resources/base/strings.xml | 1 + src/Simplex/Chat/Core.hs | 6 +- src/Simplex/Chat/Library/Commands.hs | 59 +++++---- src/Simplex/Chat/Library/Internal.hs | 4 + src/Simplex/Chat/Store/Profiles.hs | 34 ++--- .../SQLite/Migrations/agent_query_plans.txt | 4 + .../SQLite/Migrations/chat_query_plans.txt | 34 +++-- src/Simplex/Chat/Store/Shared.hs | 9 +- tests/ChatTests/Groups.hs | 4 +- tests/ChatTests/Names.hs | 120 ++++++++++++++++++ tests/ChatTests/Profiles.hs | 115 +++++++++++++++++ tests/ValidNames.hs | 7 + 20 files changed, 382 insertions(+), 73 deletions(-) diff --git a/apps/ios/Shared/Model/ChatModel.swift b/apps/ios/Shared/Model/ChatModel.swift index 6a5b1de353..7c1e8e65ab 100644 --- a/apps/ios/Shared/Model/ChatModel.swift +++ b/apps/ios/Shared/Model/ChatModel.swift @@ -935,6 +935,8 @@ final class ChatModel: ObservableObject { func updateCurrentUser(_ newProfile: Profile, _ preferences: FullPreferences? = nil) { if let current = currentUser { currentUser?.profile = toLocalProfile(current.profile.profileId, newProfile, "") + currentUser?.profile.localBadge = current.profile.localBadge + currentUser?.profile.contactDomainVerified = current.profile.contactDomainVerified if let preferences = preferences { currentUser?.fullPreferences = preferences } diff --git a/apps/ios/Shared/Model/SimpleXAPI.swift b/apps/ios/Shared/Model/SimpleXAPI.swift index be2cde7cd6..21335cee64 100644 --- a/apps/ios/Shared/Model/SimpleXAPI.swift +++ b/apps/ios/Shared/Model/SimpleXAPI.swift @@ -1359,8 +1359,12 @@ func apiUpdateProfile(profile: Profile) async throws -> (Profile, [Contact])? { switch r { case .result(.userProfileNoChange): return (profile, []) case let .result(.userProfileUpdated(_, _, toProfile, updateSummary)): return (toProfile, updateSummary.changedContacts) - case .error(.errorStore(.duplicateName)): return nil; - default: throw r.unexpected + case .error(.errorStore(.duplicateName)), .error(.error(.userExists)): return nil; + default: + await MainActor.run { + showErrorAlert(r.unexpected, NSLocalizedString("Error saving profile", comment: "alert title")) + } + throw r.unexpected } } diff --git a/apps/ios/Shared/Views/UserSettings/UserAddressView.swift b/apps/ios/Shared/Views/UserSettings/UserAddressView.swift index 9711f3d6f8..61c808dda1 100644 --- a/apps/ios/Shared/Views/UserSettings/UserAddressView.swift +++ b/apps/ios/Shared/Views/UserSettings/UserAddressView.swift @@ -139,7 +139,7 @@ struct UserAddressView: View { title: Text("Share address with SimpleX contacts?"), message: Text("Add address to your profile, so that your SimpleX contacts can share it with other people. Profile update will be sent to your SimpleX contacts."), primaryButton: .default(Text("Share")) { - setProfileAddress($progressIndicator, true) + setProfileAddress($progressIndicator, true) { shareViaProfile = false } shareViaProfile = true }, secondaryButton: .cancel() ) @@ -318,7 +318,7 @@ struct UserAddressView: View { alert = .shareOnCreate progressIndicator = false } else { - setProfileAddress($progressIndicator, true) + setProfileAddress($progressIndicator, true) { shareViaProfile = false } shareViaProfile = true } } else { @@ -601,7 +601,7 @@ struct AddressSettingsState: Equatable { } } -private func setProfileAddress(_ progressIndicator: Binding, _ on: Bool) { +private func setProfileAddress(_ progressIndicator: Binding, _ on: Bool, revert: @escaping () -> Void) { progressIndicator.wrappedValue = true Task { do { @@ -613,7 +613,11 @@ private func setProfileAddress(_ progressIndicator: Binding, _ on: Bool) { await MainActor.run { progressIndicator.wrappedValue = false } } catch let error { logger.error("apiSetProfileAddress: \(responseError(error))") - await MainActor.run { progressIndicator.wrappedValue = false } + await MainActor.run { + progressIndicator.wrappedValue = false + revert() + showErrorAlert(error, NSLocalizedString("Error saving profile", comment: "alert title")) + } } } } @@ -709,7 +713,10 @@ struct UserAddressSettingsView: View { title: NSLocalizedString("Share", comment: "alert action"), style: .default, handler: { _ in - setProfileAddress($progressIndicator, on) + setProfileAddress($progressIndicator, on) { + ignoreShareViaProfileChange = true + shareViaProfile = !on + } } ) ]} @@ -731,7 +738,10 @@ struct UserAddressSettingsView: View { title: NSLocalizedString("Stop sharing", comment: "alert action"), style: .default, handler: { _ in - setProfileAddress($progressIndicator, on) + setProfileAddress($progressIndicator, on) { + ignoreShareViaProfileChange = true + shareViaProfile = !on + } } ) ]} diff --git a/apps/ios/Shared/Views/UserSettings/UserProfile.swift b/apps/ios/Shared/Views/UserSettings/UserProfile.swift index a2a62b557c..8fe8a1c45a 100644 --- a/apps/ios/Shared/Views/UserSettings/UserProfile.swift +++ b/apps/ios/Shared/Views/UserSettings/UserProfile.swift @@ -36,7 +36,7 @@ struct UserProfile: View { HStack { TextField("Enter your name…", text: $profile.displayName) .focused($focusDisplayName) - if !validDisplayName(profile.displayName) { + if !validNewProfileName { Button { alert = .invalidNameError(validName: mkValidName(profile.displayName)) } label: { @@ -147,10 +147,15 @@ struct UserProfile: View { (chatModel.currentUser?.profile.description ?? "") != description.trimmingCharacters(in: .whitespacesAndNewlines) ) && profile.displayName.trimmingCharacters(in: .whitespaces) != "" && - validDisplayName(profile.displayName) && + validNewProfileName && bioFitsLimit() } + private var validNewProfileName: Bool { + profile.displayName == chatModel.currentUser?.profile.displayName + || validDisplayName(profile.displayName) + } + private func saveProfile() { focusDisplayName = false Task { diff --git a/apps/ios/SimpleXChat/ChatTypes.swift b/apps/ios/SimpleXChat/ChatTypes.swift index 9dbbffd4ce..4a95ed23eb 100644 --- a/apps/ios/SimpleXChat/ChatTypes.swift +++ b/apps/ios/SimpleXChat/ChatTypes.swift @@ -583,7 +583,8 @@ public func toLocalProfile (_ profileId: Int64, _ profile: Profile, _ localAlias contactLink: profile.contactLink, preferences: profile.preferences, peerType: profile.peerType, - localAlias: localAlias + localAlias: localAlias, + contactDomain: profile.contactDomain ) } diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/ChatModel.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/ChatModel.kt index c10471317e..c33a077ae9 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/ChatModel.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/ChatModel.kt @@ -1115,7 +1115,7 @@ object ChatModel { fun updateCurrentUser(rhId: Long?, newProfile: Profile, preferences: FullChatPreferences? = null) { val current = currentUser.value ?: return val updated = current.copy( - profile = newProfile.toLocalProfile(current.profile.profileId), + profile = newProfile.toLocalProfile(current.profile.profileId).copy(localBadge = current.profile.localBadge, contactDomainVerified = current.profile.contactDomainVerified), fullPreferences = preferences ?: current.fullPreferences ) val i = users.indexOfFirst { it.user.userId == current.userId && it.user.remoteHostId == rhId } diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt index 2672d0e996..5f1603423a 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt @@ -1927,8 +1927,10 @@ object ChatController { val r = sendCmd(rh, CC.ApiUpdateProfile(userId, profile)) if (r is API.Result && r.res is CR.UserProfileNoChange) return profile to emptyList() if (r is API.Result && r.res is CR.UserProfileUpdated) return r.res.toProfile to r.res.updateSummary.changedContacts - if (r is API.Error && r.err is ChatError.ChatErrorStore && r.err.storeError is StoreError.DuplicateName) { + if (r is API.Error && (r.err is ChatError.ChatErrorStore && r.err.storeError is StoreError.DuplicateName || r.err is ChatError.ChatErrorChat && r.err.errorType is ChatErrorType.UserExists)) { AlertManager.shared.showAlertMsg(generalGetString(MR.strings.failed_to_create_user_duplicate_title), generalGetString(MR.strings.failed_to_create_user_duplicate_desc)) + } else if (!(networkErrorAlert(r))) { + AlertManager.shared.showAlertMsg(generalGetString(MR.strings.error_saving_profile), "${r.responseType}: ${r.details}") } Log.e(TAG, "apiUpdateProfile bad response: ${r.responseType} ${r.details}") return null @@ -1940,7 +1942,12 @@ object ChatController { return when { r is API.Result && r.res is CR.UserProfileNoChange -> null r is API.Result && r.res is CR.UserProfileUpdated -> r.res.user.updateRemoteHostId(rh) - else -> throw Exception("failed to set profile address: ${r.responseType} ${r.details}") + else -> { + if (!(networkErrorAlert(r))) { + apiErrorAlert("apiSetProfileAddress", generalGetString(MR.strings.error_saving_profile), r) + } + throw Exception("failed to set profile address: ${r.responseType} ${r.details}") + } } } diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/usersettings/UserAddressView.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/usersettings/UserAddressView.kt index 5b22d5f363..826648ddb8 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/usersettings/UserAddressView.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/usersettings/UserAddressView.kt @@ -63,6 +63,7 @@ fun UserAddressView( chatModel.updateUser(u) } } catch (e: Exception) { + shareViaProfile.value = !on Log.e(TAG, "UserAddressView apiSetProfileAddress: ${e.stackTraceToString()}") } finally { progressIndicator.value = false diff --git a/apps/multiplatform/common/src/commonMain/moko-resources/base/strings.xml b/apps/multiplatform/common/src/commonMain/moko-resources/base/strings.xml index 2d0db3e038..71831e1c89 100644 --- a/apps/multiplatform/common/src/commonMain/moko-resources/base/strings.xml +++ b/apps/multiplatform/common/src/commonMain/moko-resources/base/strings.xml @@ -255,6 +255,7 @@ Error opening chat Error opening group Error changing profile + Error saving profile Instant notifications diff --git a/src/Simplex/Chat/Core.hs b/src/Simplex/Chat/Core.hs index 942b07dea0..3b8ffb34bb 100644 --- a/src/Simplex/Chat/Core.hs +++ b/src/Simplex/Chat/Core.hs @@ -75,10 +75,10 @@ simplexChatCore cfg@ChatConfig {confirmMigrations, testView, chatHooks} opts@Cha noMaintenance img_ <- mapM loadImageFile userImageFile createActiveUser cc coreOptions createBot userDisplayName img_ - Just u@User {localDisplayName} -> do + Just u@User {profile = LocalProfile {displayName}} -> do forM_ userDisplayName $ \name -> - when (localDisplayName /= name) $ do - putStrLn $ "Active user display name " <> show localDisplayName <> " does not match --user-display-name " <> show name + when (displayName /= name) $ do + putStrLn $ "Active user display name " <> show displayName <> " does not match --user-display-name " <> show name exitFailure -- --user-image-file only applies when the profile is created; ignore it for an existing user forM_ userImageFile $ \_ -> diff --git a/src/Simplex/Chat/Library/Commands.hs b/src/Simplex/Chat/Library/Commands.hs index bf9326871b..4ffd27f428 100644 --- a/src/Simplex/Chat/Library/Commands.hs +++ b/src/Simplex/Chat/Library/Commands.hs @@ -174,7 +174,9 @@ checkProfileImageSize = mapM_ $ \(ImageData t) -> in when (size > maxProfileImageSize) $ throwCmdError $ "Profile image is too large " <> show size checkProfileSize :: Profile -> CM () -checkProfileSize p = checkInfoSize "Profile" (XInfo p Nothing) +checkProfileSize p@Profile {contactDomain} = checkInfoSize "Profile" (XInfo p' Nothing) + where + p' = (p :: Profile) {badge = Nothing, contactDomain = (\d -> d {proof = Nothing} :: SimplexDomainClaim) <$> contactDomain} checkGroupProfileSize :: GroupProfile -> CM () checkGroupProfileSize p = checkInfoSize "Group profile" (XGrpInfo p) @@ -454,7 +456,7 @@ processChatCommand cxt nm = \case p@Profile {displayName} <- liftIO $ maybe generateRandomProfile pure profile u <- asks currentUser users <- withFastStore' getUsers - forM_ users $ \User {localDisplayName = n, activeUser, viewPwdHash, userChatRelay = userChatRelay'} -> do + forM_ users $ \User {profile = LocalProfile {displayName = n}, activeUser, viewPwdHash, userChatRelay = userChatRelay'} -> do when (n == displayName) . throwChatError $ if activeUser || isNothing viewPwdHash then CEUserExists displayName else CEInvalidDisplayName {displayName, validName = ""} when (isTrue userChatRelay && isTrue userChatRelay') $ throwChatError CEChatRelayExists @@ -1596,8 +1598,8 @@ processChatCommand cxt nm = \case APICallStatus contactId receivedStatus -> withCurrentCall contactId $ \user ct call -> updateCallItemStatus user ct call receivedStatus Nothing $> Just call - APIUpdateProfile userId profile -> withUserId userId (`updateProfile` profile) - APISetUserDomain userId strDomain_ -> withUserId userId $ \user@User {profile = p@LocalProfile {contactLink, contactDomain}} -> do + APIUpdateProfile userId profile -> withUserProfileLock "updateProfile" $ withUserId userId (`updateProfile` profile) + APISetUserDomain userId strDomain_ -> withUserProfileLock "setUserDomain" $ withUserId userId $ \user@User {profile = p@LocalProfile {contactLink, contactDomain}} -> do let domain_ = unStrJSON <$> strDomain_ if (claimDomain <$> contactDomain) == domain_ then pure $ CRUserProfileNoChange user @@ -1637,7 +1639,7 @@ processChatCommand cxt nm = \case APISetUserUIThemes uId uiThemes -> withUser $ \user@User {userId} -> do user'@User {userId = uId'} <- withFastStore $ \db -> do user' <- getUser db uId - liftIO $ setUserUIThemes db user uiThemes + liftIO $ setUserUIThemes db user' uiThemes pure user' when (userId == uId') $ chatWriteVar currentUser $ Just (user :: User) {uiThemes} ok user' @@ -2480,12 +2482,12 @@ processChatCommand cxt nm = \case pure $ CRUserContactLinkCreated user ccLink'' CreateMyAddress ratchetKeys_ -> withUser $ \User {userId} -> processChatCommand cxt nm $ APICreateMyAddress userId Nothing ratchetKeys_ - APIDeleteMyAddress userId -> withUserId userId $ \user@User {profile = p} -> do + APIDeleteMyAddress userId -> withUserProfileLock "deleteMyAddress" $ withUserId userId $ \user@User {profile = p} -> do conn <- withFastStore $ \db -> getUserAddressConnection db cxt user withChatLock "deleteMyAddress" $ do deleteAgentConnectionAsync $ aConnId conn withFastStore' (`deleteUserAddress` user) - let p' = (fromLocalProfile p :: Profile) {contactLink = Nothing} + let p' = (fromLocalProfile p :: Profile) {contactLink = Nothing, contactDomain = Nothing} r <- updateProfile_ user p' False $ withFastStore' $ \db -> setUserProfileContactLink db user Nothing let user' = case r of CRUserProfileUpdated u' _ _ _ -> u' @@ -2503,10 +2505,10 @@ processChatCommand cxt nm = \case CRUserContactLink user <$> (withFastStore (`getUserAddress` user) >>= setMyAddressData False pqInitKeys user) APIRotateAddressRatchetKeys userId -> withUserId' userId $ \user -> CRUserContactLink user <$> (withFastStore (`getUserAddress` user) >>= setMyAddressData True (Just IKUsePQ) user) - APISetProfileAddress userId False -> withUserId userId $ \user@User {profile = p} -> do - let p' = (fromLocalProfile p :: Profile) {contactLink = Nothing} + APISetProfileAddress userId False -> withUserProfileLock "setProfileAddress" $ withUserId userId $ \user@User {profile = p} -> do + let p' = (fromLocalProfile p :: Profile) {contactLink = Nothing, contactDomain = Nothing} updateProfile_ user p' True $ withFastStore' $ \db -> setUserProfileContactLink db user Nothing - APISetProfileAddress userId True -> withUserId userId $ \user@User {profile = p} -> do + APISetProfileAddress userId True -> withUserProfileLock "setProfileAddress" $ withUserId userId $ \user@User {profile = p} -> do ucl <- withFastStore (`getUserAddress` user) -- TODO [short links] replace with short links let p' = (fromLocalProfile p :: Profile) {contactLink = Just $ profileContactLink ucl} @@ -3582,23 +3584,23 @@ processChatCommand cxt nm = \case -- after the write, so the pass it signals arms a wake for the snooze rather than raising again lift $ startBadgeWork user CRBadgeState user <$> getUserBadgeState user - SetBotCommands commands -> withUser $ \user@User {profile} -> do + SetBotCommands commands -> withUserProfileLock "updateProfile" $ withUser $ \user@User {profile} -> do let LocalProfile {preferences} = profile prefs = Just (fromMaybe emptyChatPrefs preferences :: Preferences) {commands = Just commands} p = (fromLocalProfile profile :: Profile) {preferences = prefs, peerType = Just CPTBot} updateProfile user p - UpdateProfile displayName shortDescr -> withUser $ \user@User {profile} -> do + UpdateProfile displayName shortDescr -> withUserProfileLock "updateProfile" $ withUser $ \user@User {profile} -> do let p = (fromLocalProfile profile :: Profile) {displayName, shortDescr, fullName = ""} updateProfile user p - UpdateProfileImage image -> withUser $ \user@User {profile} -> do + UpdateProfileImage image -> withUserProfileLock "updateProfile" $ withUser $ \user@User {profile} -> do let p = (fromLocalProfile profile :: Profile) {image} updateProfile user p - UpdateProfileImageFromFile path -> withUser $ \user@User {profile} -> do + UpdateProfileImageFromFile path -> withUserProfileLock "updateProfile" $ withUser $ \user@User {profile} -> do img <- readProfileImageFile path let p = (fromLocalProfile profile :: Profile) {image = Just img} updateProfile user p ShowProfileImage -> withUser $ \user@User {profile} -> pure $ CRUserProfileImage user $ fromLocalProfile profile - SetUserFeature (ACF f) allowed -> withUser $ \user@User {profile} -> do + SetUserFeature (ACF f) allowed -> withUserProfileLock "updateProfile" $ withUser $ \user@User {profile} -> do let p = (fromLocalProfile profile :: Profile) {preferences = Just . setPreference f (Just allowed) $ preferences' user} updateProfile user p SetContactFeature (ACF f) cName allowed_ -> withUser $ \user -> do @@ -3616,7 +3618,7 @@ processChatCommand cxt nm = \case case memberAdmission of Nothing -> p {memberAdmission = Just (emptyGroupMemberAdmission :: GroupMemberAdmission) {review = reviewAdmissionApplication}} Just ma -> p {memberAdmission = Just (ma :: GroupMemberAdmission) {review = reviewAdmissionApplication}} - SetUserTimedMessages onOff -> withUser $ \user@User {profile} -> do + SetUserTimedMessages onOff -> withUserProfileLock "updateProfile" $ withUser $ \user@User {profile} -> do let allowed = if onOff then FAYes else FANo pref = TimedMessagesPreference allowed Nothing p = (fromLocalProfile profile :: Profile) {preferences = Just . setPreference' SCFTimedMessages (Just pref) $ preferences' user} @@ -3685,6 +3687,7 @@ processChatCommand cxt nm = \case CLContactRequest crId -> "ContactRequest " <> tshow crId CLFile fId -> "File " <> tshow fId CLBadgeUser uId -> "BadgeUser " <> tshow uId + CLUserProfile -> "UserProfile" DebugEvent event -> toView event >> ok_ GetAgentSubsTotal userId -> withUserId userId $ \user -> do users <- withStore' $ \db -> getUsers db @@ -4009,20 +4012,29 @@ processChatCommand cxt nm = \case when (fileSize > maxSndXFTPFileSize lims now sndBadge) $ throwChatError $ CEFileSize f pure fileSize updateProfile :: User -> Profile -> CM ChatResponse - updateProfile user p' = updateProfile_ user p' True $ withFastStore $ \db -> updateUserProfile db user p' + updateProfile user@User {profile = LocalProfile {contactDomain}} p = updateProfile_ user p' True $ withFastStore $ \db -> updateUserProfile db user p' + where + p' = (p :: Profile) {contactDomain} updateProfile_ :: User -> Profile -> Bool -> CM User -> CM ChatResponse updateProfile_ user@User {profile = p@LocalProfile {displayName = n}} p'@Profile {displayName = n', image = img'} shouldUpdateAddressData updateUser | p' == fromLocalProfile p = pure $ CRUserProfileNoChange user | otherwise = do - when (n /= n') $ checkValidName n' + when (n /= n') $ do + checkValidName n' + users <- withFastStore' getUsers + forM_ users $ \User {profile = LocalProfile {displayName}, activeUser, viewPwdHash} -> + when (displayName == n') . throwChatError $ + if activeUser || isNothing viewPwdHash then CEUserExists n' else CEInvalidDisplayName {displayName = n', validName = ""} checkProfileImageSize img' checkProfileSize p' + when shouldUpdateAddressData $ do + ts <- liftIO getCurrentTime + setMyAddressData' (user :: User) {profile = updatedUserProfile user p' ts} -- read contacts before user update to correctly merge preferences contacts <- withFastStore' $ \db -> getUserContacts db cxt user user' <- updateUser asks currentUser >>= atomically . (`writeTVar` Just user') withChatLock "updateProfile" $ do - when shouldUpdateAddressData $ setMyAddressData' user' summary <- sendUpdateToContacts user' contacts pure $ CRUserProfileUpdated user' (fromLocalProfile p) p' summary where @@ -5630,7 +5642,7 @@ presentIssuedBadge user p@UserBadgePurchase {badgePurchaseId, shown} now | otherwise = do cred_ <- withStore' (`getLatestIssuedCredential` badgePurchaseId) forM_ cred_ $ \cred@(BadgeCredential _ _ _ info) -> - when (presentDue cred) $ do + when (presentDue cred) $ withUserProfileLock "presentIssuedBadge" $ do user' <- withStore $ \db -> setUserBadge db user (Just $ OwnBadge cred (mkBadgeStatus now (Just True) info)) presentUserBadgeToContacts user' where @@ -5643,7 +5655,7 @@ presentIssuedBadge user p@UserBadgePurchase {badgePurchaseId, shown} now retireExpiredBadge :: User -> UserBadgePurchase -> UTCTime -> StatementEntry -> CM Bool retireExpiredBadge user UserBadgePurchase {badgePurchaseId, shown} now balance | not (shown && L.paidThrough balance <= now) = pure False - | otherwise = do + | otherwise = withUserProfileLock "retireExpiredBadge" $ do user' <- withStore $ \db -> do liftIO $ clearShownBadge db user badgePurchaseId setUserBadge db user Nothing @@ -6678,8 +6690,11 @@ displayNameP_ = quoted '\'' <|> takeNameTill (\c -> isSpace c || c == ',') refChar c = c > ' ' && c /= '#' && c /= '@' && c /= '\'' mkValidName :: String -> String -mkValidName = dropWhileEnd isSpace . take 50 . reverse . fst3 . foldl' addChar ("", '\NUL', 0 :: Int) +mkValidName = noLdnSuffix . dropWhileEnd isSpace . take 50 . reverse . fst3 . foldl' addChar ("", '\NUL', 0 :: Int) where + noLdnSuffix s = case break (== '_') (reverse s) of + (ds@(_ : _), '_' : rest@(_ : _)) | all isDigit ds -> reverse rest <> ('-' : reverse ds) + _ -> s fst3 (x, _, _) = x addChar (r, prev, punct) c' = if validChar then (c : r, c, punct') else (r, prev, punct) where diff --git a/src/Simplex/Chat/Library/Internal.hs b/src/Simplex/Chat/Library/Internal.hs index effa943cc7..4008bc2727 100644 --- a/src/Simplex/Chat/Library/Internal.hs +++ b/src/Simplex/Chat/Library/Internal.hs @@ -160,6 +160,10 @@ withFileLock :: Text -> Int64 -> CM a -> CM a withFileLock name = withEntityLock name . CLFile {-# INLINE withFileLock #-} +withUserProfileLock :: Text -> CM a -> CM a +withUserProfileLock name = withEntityLock name CLUserProfile +{-# INLINE withUserProfileLock #-} + useServerCfgs :: forall p. UserProtocol p => SProtocolType p -> RandomAgentServers -> [(Text, ServerOperator)] -> [UserServer p] -> NonEmpty (ServerCfg p) useServerCfgs p RandomAgentServers {smpServers, xftpServers} opDomains = fromMaybe (rndAgentServers p) . L.nonEmpty . agentServerCfgs p opDomains diff --git a/src/Simplex/Chat/Store/Profiles.hs b/src/Simplex/Chat/Store/Profiles.hs index c3d9f2c0bd..3b52c46fef 100644 --- a/src/Simplex/Chat/Store/Profiles.hs +++ b/src/Simplex/Chat/Store/Profiles.hs @@ -44,6 +44,7 @@ module Simplex.Chat.Store.Profiles updateUserAutoAcceptMemberContacts, updateUserAutoAcceptGroupInvitations, updateUserProfile, + updatedUserProfile, setUserBadge, setUserProfileContactLink, getUserContactProfiles, @@ -169,7 +170,7 @@ createUserRecordAt db (AgentUserId auId) userChatRelay clientService Profile {di (profileId, displayName, userId, BI True, currentTs, currentTs, currentTs) contactId <- insertedRowId db DB.execute db "UPDATE users SET contact_id = ? WHERE user_id = ?" (contactId, userId) - pure $ toUser currentTs $ (userId, auId, contactId, profileId, BI activeUser, order) :. (displayName, fullName, shortDescr, description, image, Nothing, peerType, userPreferences) :. (BI showNtfs, BI sendRcptsContacts, BI sendRcptsSmallGroups, BI autoAcceptMemberContacts, BI autoAcceptGroupInvitations, Nothing, Nothing, Nothing, BI userChatRelay, BI clientService, Nothing) :. localBadgeToRow Nothing :. (Nothing, Nothing, Nothing) + pure $ toUser currentTs $ (userId, auId, contactId, profileId, BI activeUser, order, displayName) :. (displayName, fullName, shortDescr, description, image, Nothing, peerType, userPreferences) :. (BI showNtfs, BI sendRcptsContacts, BI sendRcptsSmallGroups, BI autoAcceptMemberContacts, BI autoAcceptGroupInvitations, Nothing, Nothing, Nothing, BI userChatRelay, BI clientService, Nothing) :. localBadgeToRow Nothing :. (Nothing, Nothing, Nothing) -- TODO [mentions] getUsersInfo :: DB.Connection -> IO [UserInfo] @@ -344,21 +345,15 @@ updateUserProfile db user p' currentTs <- getCurrentTime updateUserProfileFields_' db userId profileId p' currentTs userMemberProfileUpdatedAt' <- updateUserMemberProfileUpdatedAt_ currentTs - pure user {profile = (toLocalProfile profileId p' localAlias currentTs (Just False) Nothing) {localBadge}, fullPreferences, userMemberProfileUpdatedAt = userMemberProfileUpdatedAt'} + pure user {profile = updatedUserProfile user p' currentTs, fullPreferences, userMemberProfileUpdatedAt = userMemberProfileUpdatedAt'} | otherwise = - checkConstraint SEDuplicateName . liftIO $ do + checkConstraint SEDuplicateName . ExceptT . withLocalDisplayName db userId newName $ \newLDN -> do currentTs <- getCurrentTime - -- Insert first: checkConstraint returns the violation as a value, so the - -- transaction commits, keeping whatever ran before the failing insert. - DB.execute - db - "INSERT INTO display_names (local_display_name, ldn_base, user_id, created_at, updated_at) VALUES (?,?,?,?,?)" - (newName, newName, userId, currentTs, currentTs) - DB.execute db "UPDATE users SET local_display_name = ?, updated_at = ? WHERE user_id = ?" (newName, currentTs, userId) + DB.execute db "UPDATE users SET local_display_name = ?, updated_at = ? WHERE user_id = ?" (newLDN, currentTs, userId) userMemberProfileUpdatedAt' <- updateUserMemberProfileUpdatedAt_ currentTs updateUserProfileFields_' db userId profileId p' currentTs - updateContactLDN_ db user userContactId localDisplayName newName currentTs - pure user {localDisplayName = newName, profile = (toLocalProfile profileId p' localAlias currentTs (Just False) Nothing) {localBadge}, fullPreferences, userMemberProfileUpdatedAt = userMemberProfileUpdatedAt'} + updateContactLDN_ db user userContactId localDisplayName newLDN currentTs + pure $ Right user {localDisplayName = newLDN, profile = updatedUserProfile user p' currentTs, fullPreferences, userMemberProfileUpdatedAt = userMemberProfileUpdatedAt'} where updateUserMemberProfileUpdatedAt_ currentTs | userMemberProfileChanged = do @@ -366,10 +361,14 @@ updateUserProfile db user p' pure $ Just currentTs | otherwise = pure userMemberProfileUpdatedAt userMemberProfileChanged = newName /= displayName || fn' /= fullName || d' /= shortDescr || desc' /= description || img' /= image - User {userId, userContactId, localDisplayName, profile = LocalProfile {profileId, displayName, fullName, shortDescr, description, image, localBadge, localAlias}, userMemberProfileUpdatedAt} = user + User {userId, userContactId, localDisplayName, profile = LocalProfile {profileId, displayName, fullName, shortDescr, description, image}, userMemberProfileUpdatedAt} = user Profile {displayName = newName, fullName = fn', shortDescr = d', description = desc', image = img', preferences} = p' fullPreferences = fullPreferences' preferences +updatedUserProfile :: User -> Profile -> UTCTime -> LocalProfile +updatedUserProfile User {profile = LocalProfile {profileId, localBadge, localAlias}} p' ts = + (toLocalProfile profileId p' localAlias ts (Just False) Nothing) {localBadge} + -- own profile field update; leaves the badge columns alone (the credential is owned by setUserBadge/addUserBadge) updateUserProfileFields_' :: DB.Connection -> UserId -> ProfileId -> Profile -> UTCTime -> IO () updateUserProfileFields_' db userId profileId Profile {displayName, fullName, shortDescr, description, image, contactLink, preferences, peerType} updatedAt = @@ -411,19 +410,20 @@ setUserSimplexDomain db user@User {userId, profile = p@LocalProfile {profileId}} pure (user :: User) {profile = p {contactDomain = mkDomainClaim <$> domain_}} setUserProfileContactLink :: DB.Connection -> User -> Maybe UserContactLink -> IO User -setUserProfileContactLink db user@User {userId, profile = p@LocalProfile {profileId}} ucl_ = do +setUserProfileContactLink db user@User {userId, profile = p@LocalProfile {profileId, contactDomain}} ucl_ = do ts <- getCurrentTime DB.execute db [sql| UPDATE contact_profiles - SET contact_link = ?, updated_at = ? + SET contact_link = ?, contact_domain = ?, updated_at = ? WHERE user_id = ? AND contact_profile_id = ? |] - (contactLink, ts, userId, profileId) - pure (user :: User) {profile = p {contactLink}} + (contactLink, claimDomain <$> contactDomain', ts, userId, profileId) + pure (user :: User) {profile = p {contactLink, contactDomain = contactDomain'}} where contactLink = profileContactLink <$> ucl_ + contactDomain' = if isJust contactLink then contactDomain else Nothing -- only used in tests getUserContactProfiles :: DB.Connection -> User -> IO [Profile] diff --git a/src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt b/src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt index 7c2e56c2b7..00886165b5 100644 --- a/src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt +++ b/src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt @@ -1391,6 +1391,10 @@ Query: SELECT conn_id FROM connections WHERE user_id = ? Plan: SEARCH connections USING COVERING INDEX idx_connections_user (user_id=?) +Query: SELECT conn_id FROM rcv_queues WHERE switch_status = ? AND deleted = 0 +Plan: +SCAN rcv_queues + Query: SELECT conn_id, rc_verify_code_ad, rc_verify_code_pq, CASE WHEN rc_verify_code_ad IS NULL THEN ratchet_state END FROM ratchets WHERE conn_id = ? Plan: SEARCH ratchets USING PRIMARY KEY (conn_id=?) diff --git a/src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt b/src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt index c169b95d9a..7b76c06527 100644 --- a/src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt +++ b/src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt @@ -5344,7 +5344,7 @@ SEARCH connections_sync USING INTEGER PRIMARY KEY (rowid=?) Query: UPDATE contact_profiles - SET contact_link = ?, updated_at = ? + SET contact_link = ?, contact_domain = ?, updated_at = ? WHERE user_id = ? AND contact_profile_id = ? Plan: @@ -6365,7 +6365,7 @@ Plan: SEARCH server_operators USING INTEGER PRIMARY KEY (rowid=?) Query: - SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, + SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, u.show_ntfs, u.send_rcpts_contacts, u.send_rcpts_small_groups, u.auto_accept_member_contacts, u.auto_accept_group_invitations, u.view_pwd_hash, u.view_pwd_salt, u.user_member_profile_updated_at, u.is_user_chat_relay, u.client_service, u.ui_themes, ucp.badge_proof, ucp.badge_pres_header, ucp.badge_expiry, ucp.badge_type, ucp.badge_verified, ucp.badge_extra, ucp.badge_master_key, ucp.badge_signature, ucp.badge_key_idx, ucp.contact_domain, ucp.contact_domain_proof, ucp.contact_domain_verified FROM users u @@ -6378,7 +6378,7 @@ SEARCH uct USING INTEGER PRIMARY KEY (rowid=?) SEARCH ucp USING INTEGER PRIMARY KEY (rowid=?) Query: - SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, + SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, u.show_ntfs, u.send_rcpts_contacts, u.send_rcpts_small_groups, u.auto_accept_member_contacts, u.auto_accept_group_invitations, u.view_pwd_hash, u.view_pwd_salt, u.user_member_profile_updated_at, u.is_user_chat_relay, u.client_service, u.ui_themes, ucp.badge_proof, ucp.badge_pres_header, ucp.badge_expiry, ucp.badge_type, ucp.badge_verified, ucp.badge_extra, ucp.badge_master_key, ucp.badge_signature, ucp.badge_key_idx, ucp.contact_domain, ucp.contact_domain_proof, ucp.contact_domain_verified FROM users u @@ -6392,7 +6392,7 @@ SEARCH uct USING INTEGER PRIMARY KEY (rowid=?) SEARCH ucp USING INTEGER PRIMARY KEY (rowid=?) Query: - SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, + SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, u.show_ntfs, u.send_rcpts_contacts, u.send_rcpts_small_groups, u.auto_accept_member_contacts, u.auto_accept_group_invitations, u.view_pwd_hash, u.view_pwd_salt, u.user_member_profile_updated_at, u.is_user_chat_relay, u.client_service, u.ui_themes, ucp.badge_proof, ucp.badge_pres_header, ucp.badge_expiry, ucp.badge_type, ucp.badge_verified, ucp.badge_extra, ucp.badge_master_key, ucp.badge_signature, ucp.badge_key_idx, ucp.contact_domain, ucp.contact_domain_proof, ucp.contact_domain_verified FROM users u @@ -6406,7 +6406,7 @@ SEARCH uct USING INTEGER PRIMARY KEY (rowid=?) SEARCH ucp USING INTEGER PRIMARY KEY (rowid=?) Query: - SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, + SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, u.show_ntfs, u.send_rcpts_contacts, u.send_rcpts_small_groups, u.auto_accept_member_contacts, u.auto_accept_group_invitations, u.view_pwd_hash, u.view_pwd_salt, u.user_member_profile_updated_at, u.is_user_chat_relay, u.client_service, u.ui_themes, ucp.badge_proof, ucp.badge_pres_header, ucp.badge_expiry, ucp.badge_type, ucp.badge_verified, ucp.badge_extra, ucp.badge_master_key, ucp.badge_signature, ucp.badge_key_idx, ucp.contact_domain, ucp.contact_domain_proof, ucp.contact_domain_verified FROM users u @@ -6421,7 +6421,7 @@ SEARCH uct USING INTEGER PRIMARY KEY (rowid=?) SEARCH ucp USING INTEGER PRIMARY KEY (rowid=?) Query: - SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, + SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, u.show_ntfs, u.send_rcpts_contacts, u.send_rcpts_small_groups, u.auto_accept_member_contacts, u.auto_accept_group_invitations, u.view_pwd_hash, u.view_pwd_salt, u.user_member_profile_updated_at, u.is_user_chat_relay, u.client_service, u.ui_themes, ucp.badge_proof, ucp.badge_pres_header, ucp.badge_expiry, ucp.badge_type, ucp.badge_verified, ucp.badge_extra, ucp.badge_master_key, ucp.badge_signature, ucp.badge_key_idx, ucp.contact_domain, ucp.contact_domain_proof, ucp.contact_domain_verified FROM users u @@ -6435,7 +6435,7 @@ SEARCH uct USING INTEGER PRIMARY KEY (rowid=?) SEARCH ucp USING INTEGER PRIMARY KEY (rowid=?) Query: - SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, + SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, u.show_ntfs, u.send_rcpts_contacts, u.send_rcpts_small_groups, u.auto_accept_member_contacts, u.auto_accept_group_invitations, u.view_pwd_hash, u.view_pwd_salt, u.user_member_profile_updated_at, u.is_user_chat_relay, u.client_service, u.ui_themes, ucp.badge_proof, ucp.badge_pres_header, ucp.badge_expiry, ucp.badge_type, ucp.badge_verified, ucp.badge_extra, ucp.badge_master_key, ucp.badge_signature, ucp.badge_key_idx, ucp.contact_domain, ucp.contact_domain_proof, ucp.contact_domain_verified FROM users u @@ -6449,7 +6449,7 @@ SEARCH uct USING INTEGER PRIMARY KEY (rowid=?) SEARCH ucp USING INTEGER PRIMARY KEY (rowid=?) Query: - SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, + SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, u.show_ntfs, u.send_rcpts_contacts, u.send_rcpts_small_groups, u.auto_accept_member_contacts, u.auto_accept_group_invitations, u.view_pwd_hash, u.view_pwd_salt, u.user_member_profile_updated_at, u.is_user_chat_relay, u.client_service, u.ui_themes, ucp.badge_proof, ucp.badge_pres_header, ucp.badge_expiry, ucp.badge_type, ucp.badge_verified, ucp.badge_extra, ucp.badge_master_key, ucp.badge_signature, ucp.badge_key_idx, ucp.contact_domain, ucp.contact_domain_proof, ucp.contact_domain_verified FROM users u @@ -6463,7 +6463,7 @@ SEARCH uct USING INTEGER PRIMARY KEY (rowid=?) SEARCH ucp USING INTEGER PRIMARY KEY (rowid=?) Query: - SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, + SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, u.show_ntfs, u.send_rcpts_contacts, u.send_rcpts_small_groups, u.auto_accept_member_contacts, u.auto_accept_group_invitations, u.view_pwd_hash, u.view_pwd_salt, u.user_member_profile_updated_at, u.is_user_chat_relay, u.client_service, u.ui_themes, ucp.badge_proof, ucp.badge_pres_header, ucp.badge_expiry, ucp.badge_type, ucp.badge_verified, ucp.badge_extra, ucp.badge_master_key, ucp.badge_signature, ucp.badge_key_idx, ucp.contact_domain, ucp.contact_domain_proof, ucp.contact_domain_verified FROM users u @@ -6477,7 +6477,7 @@ SEARCH uct USING INTEGER PRIMARY KEY (rowid=?) SEARCH ucp USING INTEGER PRIMARY KEY (rowid=?) Query: - SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, + SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, u.show_ntfs, u.send_rcpts_contacts, u.send_rcpts_small_groups, u.auto_accept_member_contacts, u.auto_accept_group_invitations, u.view_pwd_hash, u.view_pwd_salt, u.user_member_profile_updated_at, u.is_user_chat_relay, u.client_service, u.ui_themes, ucp.badge_proof, ucp.badge_pres_header, ucp.badge_expiry, ucp.badge_type, ucp.badge_verified, ucp.badge_extra, ucp.badge_master_key, ucp.badge_signature, ucp.badge_key_idx, ucp.contact_domain, ucp.contact_domain_proof, ucp.contact_domain_verified FROM users u @@ -6490,7 +6490,7 @@ SEARCH uct USING INTEGER PRIMARY KEY (rowid=?) SEARCH ucp USING INTEGER PRIMARY KEY (rowid=?) Query: - SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, + SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, u.show_ntfs, u.send_rcpts_contacts, u.send_rcpts_small_groups, u.auto_accept_member_contacts, u.auto_accept_group_invitations, u.view_pwd_hash, u.view_pwd_salt, u.user_member_profile_updated_at, u.is_user_chat_relay, u.client_service, u.ui_themes, ucp.badge_proof, ucp.badge_pres_header, ucp.badge_expiry, ucp.badge_type, ucp.badge_verified, ucp.badge_extra, ucp.badge_master_key, ucp.badge_signature, ucp.badge_key_idx, ucp.contact_domain, ucp.contact_domain_proof, ucp.contact_domain_verified FROM users u @@ -7632,6 +7632,10 @@ Query: SELECT member_id FROM group_members WHERE local_display_name = ? Plan: SCAN group_members +Query: SELECT member_id FROM group_members WHERE member_category = ? +Plan: +SCAN group_members + Query: SELECT member_id FROM group_members WHERE member_role = ? LIMIT 1 Plan: SCAN group_members @@ -7908,6 +7912,10 @@ Query: UPDATE contact_profiles SET contact_domain = ?, updated_at = ? WHERE user Plan: SEARCH contact_profiles USING INTEGER PRIMARY KEY (rowid=?) +Query: UPDATE contact_profiles SET contact_domain_verified = NULL WHERE user_id = ? AND contact_profile_id = ? +Plan: +SEARCH contact_profiles USING INTEGER PRIMARY KEY (rowid=?) + Query: UPDATE contact_profiles SET image = ? WHERE display_name = ? Plan: SEARCH contact_profiles USING INDEX contact_profiles_index (display_name=?) @@ -8100,6 +8108,10 @@ Query: UPDATE group_members SET xgrplinkmem_received = ?, updated_at = ? WHERE g Plan: SEARCH group_members USING INTEGER PRIMARY KEY (rowid=?) +Query: UPDATE group_profiles SET description = ? WHERE display_name = ? +Plan: +SCAN group_profiles + Query: UPDATE group_relays SET relay_status = ?, updated_at = ? WHERE group_relay_id = ? Plan: SEARCH group_relays USING INTEGER PRIMARY KEY (rowid=?) diff --git a/src/Simplex/Chat/Store/Shared.hs b/src/Simplex/Chat/Store/Shared.hs index c3d42de3d1..908d5a823b 100644 --- a/src/Simplex/Chat/Store/Shared.hs +++ b/src/Simplex/Chat/Store/Shared.hs @@ -74,6 +74,7 @@ data ChatLockEntity | CLContactRequest Int64 | CLFile Int64 | CLBadgeUser Int64 -- one signed badge request per profile in flight + | CLUserProfile deriving (Eq, Ord) -- These error type constructors must be added to mobile apps @@ -558,7 +559,7 @@ toContactRequest now ((contactRequestId, localDisplayName, agentInvitationId, co userQuery :: Query userQuery = [sql| - SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, + SELECT u.user_id, u.agent_user_id, u.contact_id, ucp.contact_profile_id, u.active_user, u.active_order, u.local_display_name, ucp.display_name, ucp.full_name, ucp.short_descr, ucp.description, ucp.image, ucp.contact_link, ucp.chat_peer_type, ucp.preferences, u.show_ntfs, u.send_rcpts_contacts, u.send_rcpts_small_groups, u.auto_accept_member_contacts, u.auto_accept_group_invitations, u.view_pwd_hash, u.view_pwd_salt, u.user_member_profile_updated_at, u.is_user_chat_relay, u.client_service, u.ui_themes, ucp.badge_proof, ucp.badge_pres_header, ucp.badge_expiry, ucp.badge_type, ucp.badge_verified, ucp.badge_extra, ucp.badge_master_key, ucp.badge_signature, ucp.badge_key_idx, ucp.contact_domain, ucp.contact_domain_proof, ucp.contact_domain_verified FROM users u @@ -566,9 +567,9 @@ userQuery = JOIN contact_profiles ucp ON ucp.contact_profile_id = uct.contact_profile_id |] -toUser :: UTCTime -> (UserId, UserId, ContactId, ProfileId, BoolInt, Int64) :. (ContactName, Text, Maybe Text, Maybe Text, Maybe ImageData, Maybe ConnLinkContact, Maybe ChatPeerType, Maybe Preferences) :. (BoolInt, BoolInt, BoolInt, BoolInt, BoolInt, Maybe B64UrlByteString, Maybe B64UrlByteString, Maybe UTCTime, BoolInt, BoolInt, Maybe UIThemeEntityOverrides) :. BadgeRow :. ContactDomainRow -> User -toUser now ((userId, auId, userContactId, profileId, BI activeUser, activeOrder) :. (displayName, fullName, shortDescr, description, image, contactLink, peerType, userPreferences) :. (BI showNtfs, BI sendRcptsContacts, BI sendRcptsSmallGroups, BI autoAcceptMemberContacts, BI autoAcceptGroupInv, viewPwdHash_, viewPwdSalt_, userMemberProfileUpdatedAt, BI userChatRelay, BI clientService, uiThemes) :. badgeRow :. domainRow) = - User {userId, agentUserId = AgentUserId auId, userContactId, localDisplayName = displayName, profile, activeUser, activeOrder, fullPreferences, showNtfs, sendRcptsContacts, sendRcptsSmallGroups, autoAcceptMemberContacts, autoAcceptGroupInvitations, viewPwdHash, userMemberProfileUpdatedAt, userChatRelay = BoolDef userChatRelay, clientService = BoolDef clientService, uiThemes} +toUser :: UTCTime -> (UserId, UserId, ContactId, ProfileId, BoolInt, Int64, ContactName) :. (ContactName, Text, Maybe Text, Maybe Text, Maybe ImageData, Maybe ConnLinkContact, Maybe ChatPeerType, Maybe Preferences) :. (BoolInt, BoolInt, BoolInt, BoolInt, BoolInt, Maybe B64UrlByteString, Maybe B64UrlByteString, Maybe UTCTime, BoolInt, BoolInt, Maybe UIThemeEntityOverrides) :. BadgeRow :. ContactDomainRow -> User +toUser now ((userId, auId, userContactId, profileId, BI activeUser, activeOrder, localDisplayName) :. (displayName, fullName, shortDescr, description, image, contactLink, peerType, userPreferences) :. (BI showNtfs, BI sendRcptsContacts, BI sendRcptsSmallGroups, BI autoAcceptMemberContacts, BI autoAcceptGroupInv, viewPwdHash_, viewPwdSalt_, userMemberProfileUpdatedAt, BI userChatRelay, BI clientService, uiThemes) :. badgeRow :. domainRow) = + User {userId, agentUserId = AgentUserId auId, userContactId, localDisplayName, profile, activeUser, activeOrder, fullPreferences, showNtfs, sendRcptsContacts, sendRcptsSmallGroups, autoAcceptMemberContacts, autoAcceptGroupInvitations, viewPwdHash, userMemberProfileUpdatedAt, userChatRelay = BoolDef userChatRelay, clientService = BoolDef clientService, uiThemes} where profile = LocalProfile {profileId, displayName, fullName, shortDescr, description, image, contactLink, contactDomain = rowToContactDomain domainRow, contactDomainVerified = rowToDomainVerified domainRow, peerType, localBadge = rowToBadge now badgeRow, preferences = userPreferences, localAlias = ""} fullPreferences = fullPreferences' userPreferences diff --git a/tests/ChatTests/Groups.hs b/tests/ChatTests/Groups.hs index 3608e5f7e9..e349a71f0a 100644 --- a/tests/ChatTests/Groups.hs +++ b/tests/ChatTests/Groups.hs @@ -8036,8 +8036,8 @@ testForwardQuoteMention = bob <# "alice> -> forwarded" bob <## " hello @alice @cath" -- member renamed to duplicate name - cath ##> "/p alice_1" - cath <## "user profile is changed to alice_1 (your 1 contacts are notified)" + cath ##> "/p alice" + cath <## "user profile is changed to alice (your 1 contacts are notified)" alice <## "contact cath changed to alice_1" alice <## "use @alice_1 to send messages" -- mention changed in quoted mentions diff --git a/tests/ChatTests/Names.hs b/tests/ChatTests/Names.hs index 6421459124..29a8c00c4f 100644 --- a/tests/ChatTests/Names.hs +++ b/tests/ChatTests/Names.hs @@ -16,6 +16,9 @@ import Test.Hspec hiding (it) chatNamesTests :: SpecWith TestParams chatNamesTests = do it "connect by resolved name" testConnectByName + it "profile update without the name keeps the name" testUpdateProfileKeepsName + it "stopping address sharing removes the name" testAddressSharingOffRemovesName + it "deleting the address removes the name" testAddressDeleteRemovesName it "connect by name not claimed in link profile is rejected" testConnectByNameNotClaimed it "connect by name to a known contact not claimed in profile is rejected" testConnectByNameKnownContactNotClaimed it "connect by unregistered name fails to resolve" testConnectByNameNotFound @@ -62,6 +65,123 @@ testConnectByName ps = withSmpServerAndNames ps $ \reg -> _ <- getTermLine bob pure () +testUpdateProfileKeepsName :: HasCallStack => TestParams -> IO () +testUpdateProfileKeepsName ps = withSmpServerAndNames ps $ \reg -> + testChat2 aliceProfile bobProfile (test reg) ps + where + aliceName = SimplexNameInfo NTContact (SimplexDomain TLDSimplex "alice" []) + test reg alice bob = do + mapM_ enableNamesRole [alice, bob] + alice ##> "/ad" + (shortLink, _) <- getContactLinks alice True + registerName reg aliceName (contactNameRecord "alice.simplex" (T.pack shortLink)) + alice ##> "/_set domain 1 alice.simplex" + alice <## "new contact address set" + bob ##> "/c @alice.simplex" + bob <## "alice: connection started" + alice <## "bob (Bob) wants to connect to you!" + alice <## "to accept: /ac bob" + alice <## "to reject: /rc bob (the sender will NOT be notified)" + alice ##> "/ac bob" + alice <## "bob (Bob): accepting contact request, you can send messages to contact" + concurrently_ + (bob <## "alice (Alice): contact is connected") + (alice <## "bob (Bob): contact is connected") + alice ##> ("/_profile 1 {\"displayName\": \"alice\", \"fullName\": \"\", \"shortDescr\": \"new bio\", \"contactLink\": \"" <> shortLink <> "\"}") + alice <## "user bio changed to new bio (your 1 contacts are notified)" + bob <## "contact alice updated bio: new bio" + bob ##> "/i alice" + bob <## "contact ID: 2" + bob <## "receiving messages via: localhost" + bob <## "sending messages via: localhost" + _ <- getTermLine bob + bob <## "SimpleX name: @alice.simplex (verified)" + bob <## "you've shared main profile with this contact" + bob <## "connection not verified, use /code command to see security code" + bob <## "quantum resistant end-to-end encryption" + _ <- getTermLine bob + pure () + +testAddressSharingOffRemovesName :: HasCallStack => TestParams -> IO () +testAddressSharingOffRemovesName ps = withSmpServerAndNames ps $ \reg -> + testChat2 aliceProfile bobProfile (test reg) ps + where + aliceName = SimplexNameInfo NTContact (SimplexDomain TLDSimplex "alice" []) + test reg alice bob = do + mapM_ enableNamesRole [alice, bob] + alice ##> "/ad" + (shortLink, _) <- getContactLinks alice True + registerName reg aliceName (contactNameRecord "alice.simplex" (T.pack shortLink)) + alice ##> "/_set domain 1 alice.simplex" + alice <## "new contact address set" + bob ##> "/c @alice.simplex" + bob <## "alice: connection started" + alice <## "bob (Bob) wants to connect to you!" + alice <## "to accept: /ac bob" + alice <## "to reject: /rc bob (the sender will NOT be notified)" + alice ##> "/ac bob" + alice <## "bob (Bob): accepting contact request, you can send messages to contact" + concurrently_ + (bob <## "alice (Alice): contact is connected") + (alice <## "bob (Bob): contact is connected") + alice ##> "/pa off" + alice <## "contact address removed" + bob <## "alice removed contact address" + checkNoName bob + alice ##> "/pa on" + alice <## "new contact address set" + bob <## "alice set new contact address, use /info alice to view" + bob ##> "/i alice" + bob <## "contact ID: 2" + bob <## "receiving messages via: localhost" + bob <## "sending messages via: localhost" + _ <- getTermLine bob + bob <## "you've shared main profile with this contact" + bob <## "connection not verified, use /code command to see security code" + bob <## "quantum resistant end-to-end encryption" + _ <- getTermLine bob + pure () + +testAddressDeleteRemovesName :: HasCallStack => TestParams -> IO () +testAddressDeleteRemovesName ps = withSmpServerAndNames ps $ \reg -> + testChat2 aliceProfile bobProfile (test reg) ps + where + aliceName = SimplexNameInfo NTContact (SimplexDomain TLDSimplex "alice" []) + test reg alice bob = do + mapM_ enableNamesRole [alice, bob] + alice ##> "/ad" + (shortLink, _) <- getContactLinks alice True + registerName reg aliceName (contactNameRecord "alice.simplex" (T.pack shortLink)) + alice ##> "/_set domain 1 alice.simplex" + alice <## "new contact address set" + bob ##> "/c @alice.simplex" + bob <## "alice: connection started" + alice <## "bob (Bob) wants to connect to you!" + alice <## "to accept: /ac bob" + alice <## "to reject: /rc bob (the sender will NOT be notified)" + alice ##> "/ac bob" + alice <## "bob (Bob): accepting contact request, you can send messages to contact" + concurrently_ + (bob <## "alice (Alice): contact is connected") + (alice <## "bob (Bob): contact is connected") + alice ##> "/da" + alice <## "Your chat address is deleted - accepted contacts will remain connected." + alice <## "To create a new chat address use /ad" + bob <## "alice removed contact address" + checkNoName bob + +checkNoName :: HasCallStack => TestCC -> IO () +checkNoName bob = do + bob ##> "/i alice" + bob <## "contact ID: 2" + bob <## "receiving messages via: localhost" + bob <## "sending messages via: localhost" + bob <## "you've shared main profile with this contact" + bob <## "connection not verified, use /code command to see security code" + bob <## "quantum resistant end-to-end encryption" + _ <- getTermLine bob + pure () + testConnectByNameNotClaimed :: HasCallStack => TestParams -> IO () testConnectByNameNotClaimed ps = withSmpServerAndNames ps $ \reg -> testChat2 aliceProfile bobProfile (test reg) ps diff --git a/tests/ChatTests/Profiles.hs b/tests/ChatTests/Profiles.hs index 31ca1bae73..06b85fdb72 100644 --- a/tests/ChatTests/Profiles.hs +++ b/tests/ChatTests/Profiles.hs @@ -46,6 +46,10 @@ chatProfileTests :: SpecWith TestParams chatProfileTests = do describe "user profiles" $ do it "update user profile and notify contacts" testUpdateProfile + it "profile is not saved when its address data upload fails" testUpdateProfileAddressDataError + it "profile name used by a contact gets a local name with suffix" testUpdateProfileNameUsedByContact + it "profile name used by another user is rejected, names with _N suffix are invalid" testUpdateProfileNameUsedByUser + it "profile name used by a hidden user is rejected" testUpdateProfileNameUsedByHiddenUser it "profile description round-trips and shows in contact info" testProfileDescriptionShown it "member profile description is redacted for members without a direct contact" testMemberDescriptionRedacted it "update user profile with image" testUpdateProfileImage @@ -210,6 +214,117 @@ testUpdateProfile = bob <## "use @cat to send messages" ] +testUpdateProfileAddressDataError :: HasCallStack => TestParams -> IO () +testUpdateProfileAddressDataError ps = do + let cfg' = testCfg {agentConfig = testAgentCfg {persistErrorInterval = 0}} + withNewTestChatCfgOpts ps cfg' opts' "alice" aliceProfile $ \alice -> + withNewTestChatCfgOpts ps cfg' opts' "bob" bobProfile $ \bob -> do + withSmpServer' serverCfg' $ do + connectUsers alice bob + alice ##> "/ad" + _ <- getContactLinks alice True + pure () + alice <## "disconnected 2 connections on server localhost" + bob <## "disconnected 1 connections on server localhost" + alice ##> "/p alisa" + alice <##. "smp agent error: BROKER" + alice ##> "/p" + alice <## "user profile: alice (Alice)" + alice <## "use /p [] to change it" + withSmpServer' serverCfg' $ do + alice <## "subscribed 2 connections on server localhost" + bob <## "subscribed 1 connections on server localhost" + alice ##> "/p alisa" + alice <## "user profile is changed to alisa (your 1 contacts are notified)" + bob <## "contact alice changed to alisa" + bob <## "use @alisa to send messages" + alice <## "disconnected 2 connections on server localhost" + bob <## "disconnected 1 connections on server localhost" + where + serverCfg' = + (smpServerCfg ps) + { transports = [(smpTestPort2 ps, transport @TLS, False)], + serverStoreCfg = persistentServerStoreCfg (tmpPath ps) + } + opts' = + testOpts + { coreOptions = + testCoreOpts + { smpServers = ["smp://LcJUMfVhwD8yxjAiSaDzzGF3-kLG4Uh0Fl_ZIjrRwjI=:server_password@localhost:7003"] + } + } + +testUpdateProfileNameUsedByContact :: HasCallStack => TestParams -> IO () +testUpdateProfileNameUsedByContact = + testChat3 aliceProfile bobProfile cathProfile $ + \alice bob cath -> do + connectUsers alice bob + alice ##> "/ad" + (shortLink, _) <- getContactLinks alice True + alice ##> "/p bob" + alice <## "user profile is changed to bob (your 1 contacts are notified)" + bob <## "contact alice changed to bob_1" + bob <## "use @bob_1 to send messages" + alice ##> "/p" + alice <## "user profile: bob" + alice <## "use /p [] to change it" + alice ##> "/users" + alice <## "bob_1 (active)" + alice #> "@bob hi" + bob <# "bob_1> hi" + cath ##> ("/_connect plan 1 " <> shortLink) + cath <## "contact address: ok to connect" + sLinkData <- getTermLine cath + sLinkData `shouldContain` "\"displayName\":\"bob\"" + alice ##> "/p alice" + alice <## "user profile is changed to alice (your 1 contacts are notified)" + bob <## "contact bob_1 changed to alice" + bob <## "use @alice to send messages" + alice ##> "/users" + alice <## "alice (active)" + +testUpdateProfileNameUsedByHiddenUser :: HasCallStack => TestParams -> IO () +testUpdateProfileNameUsedByHiddenUser = + testChat aliceProfile $ + \alice -> do + alice ##> "/create user alisa" + showActiveUser alice "alisa" + alice ##> "/hide user my_password" + alice <## "current user alisa:" + alice <## "messages are hidden (use /tail to view)" + alice <## "profile is hidden" + alice ##> "/user alice" + showActiveUser alice "alice (Alice)" + alice ##> "/p alisa" + alice <## "invalid display name: alisa" + +testUpdateProfileNameUsedByUser :: HasCallStack => TestParams -> IO () +testUpdateProfileNameUsedByUser = + testChat2 aliceProfile bobProfile $ + \alice bob -> do + connectUsers alice bob + alice ##> "/create user alisa" + showActiveUser alice "alisa" + alice ##> "/user alice" + showActiveUser alice "alice (Alice)" + alice ##> "/p alisa" + alice <## "user with the name alisa already exists" + alice ##> "/p alisa_1" + alice <## "invalid display name: alisa_1" + alice <## "you could use this one: alisa-1" + alice ##> "/p alisa-1" + alice <## "user profile is changed to alisa-1 (your 1 contacts are notified)" + bob <## "contact alice changed to alisa-1" + bob <## "use @alisa-1 to send messages" + alice ##> "/create user alisa-1" + alice <## "user with the name alisa-1 already exists" + alice ##> "/create user bob_2" + alice <## "invalid display name: bob_2" + alice <## "you could use this one: bob-2" + alice ##> "/users" + alice <## "alisa" + alice <## "alisa-1 (active)" + -- Profile.description survives the connect-time round-trip and is shown in the contact /i view. testProfileDescriptionShown :: HasCallStack => TestParams -> IO () testProfileDescriptionShown = diff --git a/tests/ValidNames.hs b/tests/ValidNames.hs index dd8433d231..a69f897516 100644 --- a/tests/ValidNames.hs +++ b/tests/ValidNames.hs @@ -57,3 +57,10 @@ testMkValidName = do mkValidName "123 alice" `shouldBe` "123 alice" mkValidName "01234567890123456789012345678901234567890123456789extra" `shouldBe` "01234567890123456789012345678901234567890123456789" mkValidName "0123456789012345678901234567890123456789012345678 extra" `shouldBe` "0123456789012345678901234567890123456789012345678" + mkValidName "alice_1" `shouldBe` "alice-1" + mkValidName "alice_bob_12" `shouldBe` "alice_bob-12" + mkValidName "alice _1" `shouldBe` "alice -1" + mkValidName "alice_" `shouldBe` "alice_" + mkValidName "alice_bob" `shouldBe` "alice_bob" + mkValidName "alice_1b" `shouldBe` "alice_1b" + mkValidName (mkValidName "alice_1") `shouldBe` "alice-1" From f2b3fe385cecddaa1a9ede2e6ed8dfdb63cee51f Mon Sep 17 00:00:00 2001 From: Evgeny Date: Fri, 9 Oct 2026 07:00:27 +0100 Subject: [PATCH 6/9] core: remove legacy field to support contact connections in groups (#7671) Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com> --- apps/ios/SimpleXChat/ChatTypes.swift | 1 + docs/protocol/simplex-chat.schema.json | 3 --- src/Simplex/Chat/Library/Subscriber.hs | 20 ++++++--------- src/Simplex/Chat/Store/Groups.hs | 34 ++------------------------ src/Simplex/Chat/Types.hs | 3 +-- src/Simplex/Chat/protocol.md | 6 ++--- tests/ProtocolTests.hs | 12 +++------ 7 files changed, 17 insertions(+), 62 deletions(-) diff --git a/apps/ios/SimpleXChat/ChatTypes.swift b/apps/ios/SimpleXChat/ChatTypes.swift index 4a95ed23eb..a8c5de3672 100644 --- a/apps/ios/SimpleXChat/ChatTypes.swift +++ b/apps/ios/SimpleXChat/ChatTypes.swift @@ -133,6 +133,7 @@ public struct Profile: Codable, NamedChat, Hashable { self.image = image self.contactLink = contactLink self.preferences = preferences + self.peerType = peerType self.contactDomain = contactDomain } diff --git a/docs/protocol/simplex-chat.schema.json b/docs/protocol/simplex-chat.schema.json index 8a71e715db..a2400e9947 100644 --- a/docs/protocol/simplex-chat.schema.json +++ b/docs/protocol/simplex-chat.schema.json @@ -325,9 +325,6 @@ "introInvitation": { "properties": { "groupConnReq": {"ref": "connReqUri"} - }, - "optionalProperties": { - "directConnReq": {"ref": "connReqUri"} } }, "groupMemberRole": { diff --git a/src/Simplex/Chat/Library/Subscriber.hs b/src/Simplex/Chat/Library/Subscriber.hs index 304aa15500..4ff32429b9 100644 --- a/src/Simplex/Chat/Library/Subscriber.hs +++ b/src/Simplex/Chat/Library/Subscriber.hs @@ -774,7 +774,7 @@ processAgentMessageConn cxt user@User {userId} entity gks_ corrId agentConnId ag hostConnId <- withStore $ \db -> do liftIO $ setConnConnReqInv db user connId cReq getHostConnId db user groupId - sendXGrpMemInv hostConnId Nothing XGrpMemIntroCont {groupId, groupMemberId, memberId, groupConnReq} + sendXGrpMemInv hostConnId XGrpMemIntroCont {groupId, groupMemberId, memberId, groupConnReq} _ -> throwChatError $ CECommandError "unexpected cmdFunction" CRContactUri _ _ -> throwChatError $ CECommandError "unexpected ConnectionRequestUri type" CONF confId _pqSupport _ connInfo -> do @@ -3304,10 +3304,10 @@ processAgentMessageConn cxt user@User {userId} entity gks_ corrId agentConnId ag withAgent $ \a -> createConnectionAsync a (aCorrId cmdId) connId (chatHasNtfs chatSettings) SCMInvitation CR.IKPQOff True subMode _ -> messageError "x.grp.mem.intro can be only sent by host member" - sendXGrpMemInv :: Int64 -> Maybe ConnReqInvitation -> XGrpMemIntroCont -> CM () - sendXGrpMemInv hostConnId directConnReq XGrpMemIntroCont {groupId, groupMemberId, memberId, groupConnReq} = do + sendXGrpMemInv :: Int64 -> XGrpMemIntroCont -> CM () + sendXGrpMemInv hostConnId XGrpMemIntroCont {groupId, groupMemberId, memberId, groupConnReq} = do hostConn <- withStore $ \db -> getConnectionById db cxt user hostConnId - let msg = XGrpMemInv memberId IntroInvitation {groupConnReq, directConnReq} + let msg = XGrpMemInv memberId IntroInvitation {groupConnReq} void $ sendDirectMemberMessage hostConn msg groupId withStore' $ \db -> updateGroupMemberStatusById db userId groupMemberId GSMemIntroInvited @@ -3321,7 +3321,7 @@ processAgentMessageConn cxt user@User {userId} entity gks_ corrId agentConnId ag _ -> messageError "x.grp.mem.inv can be only sent by invitee member" xGrpMemFwd :: GroupInfo -> GroupMember -> MemberInfo -> IntroInvitation -> CM () - xGrpMemFwd gInfo@GroupInfo {membership, chatSettings} m memInfo@(MemberInfo memId memRole memChatVRange _ _) IntroInvitation {groupConnReq, directConnReq} = do + xGrpMemFwd gInfo@GroupInfo {membership, chatSettings} m memInfo@(MemberInfo memId memRole memChatVRange _ _) IntroInvitation {groupConnReq} = do let GroupMember {memberId = membershipMemId} = membership checkHostRole m memRole toMember <- withStore $ \db -> do @@ -3341,20 +3341,16 @@ processAgentMessageConn cxt user@User {userId} entity gks_ corrId agentConnId ag liftIO $ updateGroupMemberStatus db userId toMember newMemberStatus pure toMember subMode <- chatReadVar subscriptionMode - -- [incognito] send membership incognito profile, create direct connection as incognito + -- [incognito] send membership incognito profile membershipProfile <- presentUserBadge user (incognitoMembershipProfile gInfo) $ redactedMemberProfile gInfo membership $ fromLocalProfile $ memberProfile membership dm <- encodeConnInfo $ XGrpMemInfo membershipMemId membershipProfile -- [async agent commands] no continuation needed, but commands should be asynchronous for stability let enableNtfsGrp = chatHasNtfs chatSettings groupConnIds@(gCmdId, gAcId) <- prepareAgentJoin user Nothing enableNtfsGrp groupConnReq - directConnIds <- mapM (prepareAgentJoin user Nothing True) directConnReq - let customUserProfileId = localProfileId <$> incognitoMembershipProfile gInfo - mcvr = maybe chatInitialVRange fromChatVRange memChatVRange + let mcvr = maybe chatInitialVRange fromChatVRange memChatVRange chatV = vr cxt `peerConnChatVersion` mcvr - withStore' $ \db -> createIntroToMemberContact db user m toMember chatV mcvr groupConnIds directConnIds customUserProfileId subMode + withStore' $ \db -> createIntroToMemberContact db user m toMember chatV mcvr groupConnIds subMode joinAgentConnectionAsync gCmdId False gAcId enableNtfsGrp groupConnReq dm subMode - forM_ ((,) <$> directConnIds <*> directConnReq) $ \((dCmdId, dAcId), dcr) -> - joinAgentConnectionAsync dCmdId False dAcId True dcr dm subMode -- rollback defense (channels): apply an owner-signed role/removal only at a version >= the persisted -- roster_version (not the batch-constant gInfo, which a relay can stale by reordering events in one diff --git a/src/Simplex/Chat/Store/Groups.hs b/src/Simplex/Chat/Store/Groups.hs index 66a5dc7783..a376ee7ca5 100644 --- a/src/Simplex/Chat/Store/Groups.hs +++ b/src/Simplex/Chat/Store/Groups.hs @@ -2645,42 +2645,12 @@ createIntroReMemberConn liftIO $ setCommandConnId db user groupCmdId groupConnId pure (reMember :: GroupMember) {activeConn = Just conn} -createIntroToMemberContact :: DB.Connection -> User -> GroupMember -> GroupMember -> VersionChat -> VersionRangeChat -> (CommandId, ConnId) -> Maybe (CommandId, ConnId) -> Maybe ProfileId -> SubscriptionMode -> IO () -createIntroToMemberContact db user@User {userId} GroupMember {memberContactId = viaContactId, activeConn} _to@GroupMember {groupMemberId, localDisplayName} chatV mcvr (groupCmdId, groupAgentConnId) directConnIds customUserProfileId subMode = do +createIntroToMemberContact :: DB.Connection -> User -> GroupMember -> GroupMember -> VersionChat -> VersionRangeChat -> (CommandId, ConnId) -> SubscriptionMode -> IO () +createIntroToMemberContact db user@User {userId} GroupMember {memberContactId = viaContactId, activeConn} _to@GroupMember {groupMemberId} chatV mcvr (groupCmdId, groupAgentConnId) subMode = do let cLevel = 1 + maybe 0 (\Connection {connLevel} -> connLevel) activeConn currentTs <- getCurrentTime Connection {connId = groupConnId} <- createMemberConnection_ db userId groupMemberId groupAgentConnId chatV mcvr viaContactId cLevel currentTs subMode setCommandConnId db user groupCmdId groupConnId - forM_ directConnIds $ \(directCmdId, directAgentConnId) -> do - Connection {connId = directConnId} <- createConnection_ db userId ConnContact Nothing directAgentConnId ConnNew chatV mcvr viaContactId Nothing customUserProfileId cLevel currentTs subMode PQSupportOff - setCommandConnId db user directCmdId directConnId - contactId <- createMemberContact_ directConnId currentTs - updateMember_ contactId currentTs - where - createMemberContact_ :: Int64 -> UTCTime -> IO Int64 - createMemberContact_ connId ts = do - DB.execute - db - [sql| - INSERT INTO contacts (contact_profile_id local_display_name, user_id, created_at, updated_at, chat_ts) - SELECT contact_profile_id, ?, ?, ?, ?, ? - FROM group_members - WHERE group_member_id = ? - |] - (localDisplayName, userId, ts, ts, ts, groupMemberId) - contactId <- insertedRowId db - DB.execute db "UPDATE connections SET contact_id = ?, updated_at = ? WHERE connection_id = ?" (contactId, ts, connId) - pure contactId - updateMember_ :: Int64 -> UTCTime -> IO () - updateMember_ contactId ts = - DB.execute - db - [sql| - UPDATE group_members - SET contact_id = ?, updated_at = ? - WHERE group_member_id = ? - |] - (contactId, ts, groupMemberId) createMemberConnection_ :: DB.Connection -> UserId -> Int64 -> ConnId -> VersionChat -> VersionRangeChat -> Maybe Int64 -> Int -> UTCTime -> SubscriptionMode -> IO Connection createMemberConnection_ db userId groupMemberId agentConnId chatV peerChatVRange viaContact connLevel currentTs subMode = diff --git a/src/Simplex/Chat/Types.hs b/src/Simplex/Chat/Types.hs index ce5879538b..cea169a4a4 100644 --- a/src/Simplex/Chat/Types.hs +++ b/src/Simplex/Chat/Types.hs @@ -1085,8 +1085,7 @@ data MemberIdRole = MemberIdRole deriving (Eq, Show) data IntroInvitation = IntroInvitation - { groupConnReq :: ConnReqInvitation, - directConnReq :: Maybe ConnReqInvitation + { groupConnReq :: ConnReqInvitation } deriving (Eq, Show) diff --git a/src/Simplex/Chat/protocol.md b/src/Simplex/Chat/protocol.md index 33161a45a7..06b34d92eb 100644 --- a/src/Simplex/Chat/protocol.md +++ b/src/Simplex/Chat/protocol.md @@ -197,8 +197,7 @@ Events: { "memberId": "", "memberIntro": { - "groupConnReq": "", - "directConnReq": "" + "groupConnReq": "" } } @@ -214,8 +213,7 @@ Events: }, }, "memberIntro": { - "groupConnReq": "", - "directConnReq": "" + "groupConnReq": "" } } diff --git a/tests/ProtocolTests.hs b/tests/ProtocolTests.hs index e341a210a7..2d9842e9ee 100644 --- a/tests/ProtocolTests.hs +++ b/tests/ProtocolTests.hs @@ -369,17 +369,11 @@ decodeChatMessageTest = describe "Chat message encoding/decoding" $ do "{\"v\":\"9\",\"event\":\"x.grp.mem.intro\",\"params\":{\"memberRestrictions\":{\"restriction\":\"blocked\"},\"memberInfo\":{\"memberRole\":\"admin\",\"memberId\":\"AQIDBA==\",\"profile\":{\"fullName\":\"Alice\",\"displayName\":\"alice\",\"image\":\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII=\",\"preferences\":{\"reactions\":{\"allow\":\"yes\"},\"voice\":{\"allow\":\"yes\"}}}}}}" #==# XGrpMemIntro MemberInfo {memberId = MemberId "\1\2\3\4", memberRole = GRAdmin, v = Nothing, profile = testProfile, memberKey = Nothing} (Just MemberRestrictions {restriction = MRSBlocked}) it "x.grp.mem.inv" $ - "{\"v\":\"9\",\"event\":\"x.grp.mem.inv\",\"params\":{\"memberId\":\"AQIDBA==\",\"memberIntro\":{\"directConnReq\":\"simplex:/invitation#/?v=1&smp=smp%3A%2F%2F1234-w%3D%3D%40smp.simplex.im%3A5223%2F3456-w%3D%3D%23%2F%3Fv%3D1-4%26dh%3DMCowBQYDK2VuAyEAjiswwI3O_NlS8Fk3HJUW870EY2bAwmttMBsvRB9eV3o%253D&e2e=v%3D3%26x3dh%3DMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D%2CMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D\",\"groupConnReq\":\"simplex:/invitation#/?v=1&smp=smp%3A%2F%2F1234-w%3D%3D%40smp.simplex.im%3A5223%2F3456-w%3D%3D%23%2F%3Fv%3D1-4%26dh%3DMCowBQYDK2VuAyEAjiswwI3O_NlS8Fk3HJUW870EY2bAwmttMBsvRB9eV3o%253D&e2e=v%3D3%26x3dh%3DMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D%2CMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D\"}}}" - #==# XGrpMemInv (MemberId "\1\2\3\4") IntroInvitation {groupConnReq = testConnReq, directConnReq = Just testConnReq} - it "x.grp.mem.inv w/t directConnReq" $ "{\"v\":\"9\",\"event\":\"x.grp.mem.inv\",\"params\":{\"memberId\":\"AQIDBA==\",\"memberIntro\":{\"groupConnReq\":\"simplex:/invitation#/?v=1&smp=smp%3A%2F%2F1234-w%3D%3D%40smp.simplex.im%3A5223%2F3456-w%3D%3D%23%2F%3Fv%3D1-4%26dh%3DMCowBQYDK2VuAyEAjiswwI3O_NlS8Fk3HJUW870EY2bAwmttMBsvRB9eV3o%253D&e2e=v%3D3%26x3dh%3DMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D%2CMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D\"}}}" - #==# XGrpMemInv (MemberId "\1\2\3\4") IntroInvitation {groupConnReq = testConnReq, directConnReq = Nothing} - it "x.grp.mem.fwd" $ - "{\"v\":\"9\",\"event\":\"x.grp.mem.fwd\",\"params\":{\"memberIntro\":{\"directConnReq\":\"simplex:/invitation#/?v=1&smp=smp%3A%2F%2F1234-w%3D%3D%40smp.simplex.im%3A5223%2F3456-w%3D%3D%23%2F%3Fv%3D1-4%26dh%3DMCowBQYDK2VuAyEAjiswwI3O_NlS8Fk3HJUW870EY2bAwmttMBsvRB9eV3o%253D&e2e=v%3D3%26x3dh%3DMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D%2CMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D\",\"groupConnReq\":\"simplex:/invitation#/?v=1&smp=smp%3A%2F%2F1234-w%3D%3D%40smp.simplex.im%3A5223%2F3456-w%3D%3D%23%2F%3Fv%3D1-4%26dh%3DMCowBQYDK2VuAyEAjiswwI3O_NlS8Fk3HJUW870EY2bAwmttMBsvRB9eV3o%253D&e2e=v%3D3%26x3dh%3DMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D%2CMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D\"},\"memberInfo\":{\"memberRole\":\"admin\",\"memberId\":\"AQIDBA==\",\"profile\":{\"fullName\":\"Alice\",\"displayName\":\"alice\",\"image\":\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII=\",\"preferences\":{\"reactions\":{\"allow\":\"yes\"},\"voice\":{\"allow\":\"yes\"}}}}}}" - #==# XGrpMemFwd MemberInfo {memberId = MemberId "\1\2\3\4", memberRole = GRAdmin, v = Nothing, profile = testProfile, memberKey = Nothing} IntroInvitation {groupConnReq = testConnReq, directConnReq = Just testConnReq} - it "x.grp.mem.fwd with member chat version range and w/t directConnReq" $ + #==# XGrpMemInv (MemberId "\1\2\3\4") IntroInvitation {groupConnReq = testConnReq} + it "x.grp.mem.fwd with member chat version range" $ "{\"v\":\"9\",\"event\":\"x.grp.mem.fwd\",\"params\":{\"memberIntro\":{\"groupConnReq\":\"simplex:/invitation#/?v=1&smp=smp%3A%2F%2F1234-w%3D%3D%40smp.simplex.im%3A5223%2F3456-w%3D%3D%23%2F%3Fv%3D1-4%26dh%3DMCowBQYDK2VuAyEAjiswwI3O_NlS8Fk3HJUW870EY2bAwmttMBsvRB9eV3o%253D&e2e=v%3D3%26x3dh%3DMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D%2CMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D\"},\"memberInfo\":{\"memberRole\":\"admin\",\"memberId\":\"AQIDBA==\",\"v\":\"9-21\",\"profile\":{\"fullName\":\"Alice\",\"displayName\":\"alice\",\"image\":\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII=\",\"preferences\":{\"reactions\":{\"allow\":\"yes\"},\"voice\":{\"allow\":\"yes\"}}}}}}" - #==# XGrpMemFwd MemberInfo {memberId = MemberId "\1\2\3\4", memberRole = GRAdmin, v = Just $ ChatVersionRange supportedChatVRange, profile = testProfile, memberKey = Nothing} IntroInvitation {groupConnReq = testConnReq, directConnReq = Nothing} + #==# XGrpMemFwd MemberInfo {memberId = MemberId "\1\2\3\4", memberRole = GRAdmin, v = Just $ ChatVersionRange supportedChatVRange, profile = testProfile, memberKey = Nothing} IntroInvitation {groupConnReq = testConnReq} it "x.grp.mem.info" $ "{\"v\":\"9\",\"event\":\"x.grp.mem.info\",\"params\":{\"memberId\":\"AQIDBA==\",\"profile\":{\"fullName\":\"Alice\",\"displayName\":\"alice\",\"image\":\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII=\",\"preferences\":{\"reactions\":{\"allow\":\"yes\"},\"voice\":{\"allow\":\"yes\"}}}}}" #==# XGrpMemInfo (MemberId "\1\2\3\4") testProfile From f924d233e45623028e2aaabc8eb5855bd338b0ae Mon Sep 17 00:00:00 2001 From: Evgeny Date: Fri, 9 Oct 2026 07:48:04 +0100 Subject: [PATCH 7/9] core: fix encoding for name proof (#7672) Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com> --- src/Simplex/Chat/Badges.hs | 8 +++++++- tests/BadgeTests.hs | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/Simplex/Chat/Badges.hs b/src/Simplex/Chat/Badges.hs index 62683788d5..f34ba8ef93 100644 --- a/src/Simplex/Chat/Badges.hs +++ b/src/Simplex/Chat/Badges.hs @@ -295,7 +295,13 @@ data ProofPresHeader | PHFileDescr {chatBinding :: ByteString, fileSize :: Int64, descrHash :: ByteString, fileExpires :: Maybe UTCTime} | PHUnknown Char ByteString deriving (Eq, Show) - deriving (ToJSON, FromJSON) via (StrJSON "ProofPresHeader" ProofPresHeader) + +instance ToJSON ProofPresHeader where + toJSON = toJSON . BBSPresHeader . strEncode + toEncoding = toEncoding . BBSPresHeader . strEncode + +instance FromJSON ProofPresHeader where + parseJSON v = parseJSON v >>= \(BBSPresHeader ph) -> either fail pure $ strDecode ph instance StrEncoding ProofPresHeader where strEncode = \case diff --git a/tests/BadgeTests.hs b/tests/BadgeTests.hs index bd8f62d3fd..ced51a5bbf 100644 --- a/tests/BadgeTests.hs +++ b/tests/BadgeTests.hs @@ -51,6 +51,7 @@ badgeTests = do it "should accept unknown badge types" testUnknownBadgeType it "credential serializes to a paste-able token and back" testCredentialSerialization it "presentation headers encode and decode" testPresHeaderEncoding + it "presentation headers with binary bytes round-trip through JSON" testPresHeaderJSON it "should reject a proof presented under another chat binding" testOtherChatBinding describe "redemption codes" $ do it "a generated code reads back" testCodeRoundTrip @@ -209,6 +210,21 @@ testPresHeaderEncoding = PHUnknown 'Z' "payload" ] +testPresHeaderJSON :: IO () +testPresHeaderJSON = + mapM_ + ( \ph -> do + J.eitherDecode (J.encode ph) `shouldBe` Right ph + J.toJSON ph `shouldBe` J.toJSON (BBSPresHeader $ strEncode ph) + ) + [ PHChat binaryBytes, + PHFileInv {chatBinding = binaryBytes, fileSize = 139737}, + PHFileDescr {chatBinding = binaryBytes, fileSize = 139737, descrHash = binaryBytes, fileExpires = Just futureTime}, + PHUnknown 'Z' binaryBytes + ] + where + binaryBytes = "\0\1\31\34\92\127\128\195\255" + testOtherChatBinding :: IO () testOtherChatBinding = do let ph = PHFileInv {chatBinding = aliceBinding, fileSize = 139737} From 10166454fdc624b71acc5f76cf752905d340c212 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Fri, 9 Oct 2026 07:53:40 +0100 Subject: [PATCH 8/9] website: add sitemap, robots, llms.txt and markdown pages (#7670) * website: markdown pages for llms * website: add sitemap, robots, llms.txt and markdown pages * faster site generation * more faster * faster * update * optimize build --------- Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com> --- README.md | 13 + docs/WHY.md | 2 + website/.eleventy.js | 278 +++++++++++------- website/copy_sources.js | 81 +++++ website/customize_docs_frontmatter.js | 131 ++++----- website/langs/cs.json | 2 +- website/langs/en.json | 1 + website/markdown_pages.js | 272 +++++++++++++++++ website/package.json | 4 +- website/plans/2026-10-08-llms.md | 99 +++++++ website/run.sh | 2 + website/src/_data/llms_facts.json | 219 ++++++++++++++ website/src/_includes/contact_page.html | 12 +- website/src/_includes/layouts/article.html | 1 + website/src/_includes/layouts/channel.html | 13 + website/src/_includes/layouts/doc.html | 1 + website/src/_includes/layouts/jobs.html | 1 + website/src/_includes/layouts/main.html | 1 + website/src/_includes/layouts/page.html | 1 + website/src/_includes/layouts/privacy.html | 1 + website/src/_includes/layouts/token.html | 1 + website/src/_includes/llms/agents.md | 157 ++++++++++ website/src/_includes/llms/answers.md | 73 +++++ website/src/_includes/llms/facts.md | 12 + website/src/_includes/llms/timeline.md | 27 ++ website/src/_includes/llms/wefunder.md | 194 ++++++++++++ website/src/_includes/markdown-alternate.html | 3 + .../_includes/sections/simplex_unique.html | 2 +- website/src/crowdfunding-news.html | 18 +- website/src/crowdfunding.html | 4 +- website/src/directory.html | 1 + website/src/index.html | 11 +- website/src/links.html | 6 +- website/src/livestream.html | 1 + website/src/llms/agents.njk | 5 + website/src/llms/answers.njk | 5 + website/src/llms/docs.njk | 11 + website/src/llms/facts-json.njk | 5 + website/src/llms/facts.njk | 5 + website/src/llms/llms-full.njk | 63 ++++ website/src/llms/llms-txt.njk | 88 ++++++ website/src/llms/robots.njk | 8 + website/src/llms/sitemap.njk | 12 + website/src/llms/timeline.njk | 5 + website/src/llms/wefunder.njk | 5 + website/src/messaging.html | 2 +- website/src/news.html | 19 +- website/src/wefunder.html | 1 + website/src/why.html | 4 + website/web.sh | 17 +- 50 files changed, 1664 insertions(+), 236 deletions(-) create mode 100644 website/copy_sources.js create mode 100644 website/markdown_pages.js create mode 100644 website/plans/2026-10-08-llms.md create mode 100644 website/src/_data/llms_facts.json create mode 100644 website/src/_includes/layouts/channel.html create mode 100644 website/src/_includes/llms/agents.md create mode 100644 website/src/_includes/llms/answers.md create mode 100644 website/src/_includes/llms/facts.md create mode 100644 website/src/_includes/llms/timeline.md create mode 100644 website/src/_includes/llms/wefunder.md create mode 100644 website/src/_includes/markdown-alternate.html create mode 100644 website/src/llms/agents.njk create mode 100644 website/src/llms/answers.njk create mode 100644 website/src/llms/docs.njk create mode 100644 website/src/llms/facts-json.njk create mode 100644 website/src/llms/facts.njk create mode 100644 website/src/llms/llms-full.njk create mode 100644 website/src/llms/llms-txt.njk create mode 100644 website/src/llms/robots.njk create mode 100644 website/src/llms/sitemap.njk create mode 100644 website/src/llms/timeline.njk create mode 100644 website/src/llms/wefunder.njk diff --git a/README.md b/README.md index fa4247caf8..cb792e84f5 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,7 @@ SimpleX Chat founder - [Privacy and security: technical details and limitations](#privacy-and-security-technical-details-and-limitations) - [For developers](#for-developers) - [Develop a chat bot](#develop-a-chat-bot) +- [Agents](#agents) - [Roadmap](#roadmap) - [Disclaimers, Security contact, License](#disclaimers) @@ -331,6 +332,18 @@ See [our new bot API reference](./bots/README.md). Most of it is automatically g Also see [TypeScript SimpleX Chat client](./packages/simplex-chat-client/) and [JavaScript chat bot example](./packages/simplex-chat-client/typescript/examples/squaring-bot.js). +## Agents + +- For AI assistants: [simplex.chat/llms.txt](https://simplex.chat/llms.txt), an index of the documentation in Markdown, and [SimpleX for AI agents and developers](https://simplex.chat/llms/agents.md). +- Every page of simplex.chat is also available as Markdown: replace `.html` with `.md` ([simplex.chat/docs/simplex.md](https://simplex.chat/docs/simplex.md)), or add `.md` to the path ([simplex.chat/why.md](https://simplex.chat/why.md)). +- Libraries: [Node.js](./packages/simplex-chat-nodejs/) ([npm](https://www.npmjs.com/package/simplex-chat)) and [Python](./packages/simplex-chat-python/) ([PyPI](https://pypi.org/project/simplex-chat/)). +- [Bot API guide](./bots/README.md) and [API reference](./bots/api/README.md): commands, events and types. +- Example bots: [support bot](./apps/simplex-support-bot/) and [calculator](./apps/simplex-calculator-bot/) in TypeScript, [support bot light](./apps/simplex-support-bot-light/) in Python, [directory service](./apps/simplex-directory-service/) and [broadcast bot](./apps/simplex-broadcast-bot/) in Haskell. +- [Terminal client](./docs/CLI.md): installation, building from source and usage. +- Servers: [SMP](./docs/SERVER.md), [XFTP](./docs/XFTP-SERVER.md) and [chat relay](./docs/CHAT-RELAY.md). +- Protocols: [SimpleX network](https://github.com/simplex-chat/simplexmq/blob/stable/protocol/overview-tjr.md), [chat](./docs/protocol/simplex-chat.md), [channels](./docs/protocol/channels-overview.md) and [public names](./docs/protocol/names-overview.md). +- For coding agents working in this repository: [project structure](./docs/contributing/PROJECT.md) and [coding and building](./docs/contributing/CODE.md). + ## Roadmap - ✅ Easy to deploy SimpleX server with in-memory message storage, without any dependencies. diff --git a/docs/WHY.md b/docs/WHY.md index e1582984df..b36df4a154 100644 --- a/docs/WHY.md +++ b/docs/WHY.md @@ -17,3 +17,5 @@ The oldest human freedom — to speak to another person without being watched Because we destroyed the power to know who you are. So that your power can never be taken. Be free in your network. + +[Watch the film](https://www.youtube.com/watch?v=k-GhLNZm8G0) diff --git a/website/.eleventy.js b/website/.eleventy.js index a96614e4ba..dcd0f67916 100644 --- a/website/.eleventy.js +++ b/website/.eleventy.js @@ -4,22 +4,44 @@ const markdownItReplaceLink = require('markdown-it-replace-link') const markdownItFootnote = require('markdown-it-footnote') const slugify = require("slugify") const uri = require('fast-uri') -const i18n = require('eleventy-plugin-i18n') const fs = require("fs") const path = require("path") const matter = require('gray-matter') const pluginRss = require('@11ty/eleventy-plugin-rss') const { JSDOM } = require('jsdom') +const parse5 = require('parse5') +const markdownPages = require('./markdown_pages') // Links page data const parseLinks = require('./parse_links') const linksFilePath = path.resolve(__dirname, '../docs/LINKS.md') -const linksData = fs.existsSync(linksFilePath) ? parseLinks(linksFilePath) : [] const linkImagesDir = path.resolve(__dirname, 'src/link-images') -linksData.forEach(entry => { - entry.imageExists = entry.image && fs.existsSync(path.join(linkImagesDir, entry.image)) -}) + +function readLinksPage() { + const entries = fs.existsSync(linksFilePath) ? parseLinks(linksFilePath) : [] + entries.forEach(entry => { + entry.imageExists = entry.image && fs.existsSync(path.join(linkImagesDir, entry.image)) + }) + + const catCounts = {} + entries.forEach(e => { if (e.category) { const c = e.category.toLowerCase(); catCounts[c] = (catCounts[c] || 0) + 1 } }) + const mediaPills = ["Video", "Audio"].filter(p => entries.some(e => e.mediaType === p.toLowerCase())) + const catPills = Object.keys(catCounts).sort() + + return { + entries, + languages: [...new Set(entries.map(e => e.language).filter(Boolean))].sort(), + pills: mediaPills.concat(catPills), + } +} + +const permalinks = new Map() + +function filePermalink(file) { + if (!permalinks.has(file)) permalinks.set(file, matter(fs.readFileSync(file, 'utf8')).data?.permalink) + return permalinks.get(file) +} // Rewrites relative markdown links to website permalinks or GitHub URLs. // Shared by the main markdown renderer (markdownLib) and the glossary renderer @@ -39,8 +61,7 @@ function replaceLink(link, _env) { if (fs.existsSync(linkFile) && fs.statSync(linkFile).isFile()) { // this condition works if the link is a valid website file - const fileContent = fs.readFileSync(linkFile, 'utf8') - parsed.path = (matter(fileContent).data?.permalink || parsed.path).replace(/\.md$/, ".html").toLowerCase() + parsed.path = (filePermalink(linkFile) || parsed.path).replace(/\.md$/, ".html").toLowerCase() } else if (!fs.existsSync(linkFile)) { linkFile = linkFile.replace('/website/src', '') if (fs.existsSync(linkFile)) { @@ -94,8 +115,41 @@ glossary.forEach(item => { item.tooltip = firstParagraph item.hasMultipleParagraphs = paragraphCount > 1 } + + const definitionLinks = new JSDOM(item.definition).window.document.querySelectorAll('a[href*="#"]') + item.linkedHashes = Array.from(definitionLinks, a => a.href.substring(a.href.indexOf("#") + 1)) + item.id = item.term.toLowerCase().replace(/\s/g, '-') + item.pattern = new RegExp(`(? [item.id, item])) +const glossaryTermPattern = new RegExp(`(? item.term).join('|')})\\b`, 'i') +const glossaryContentTags = new Set(['p', 'td', 'a', 'h1', 'h2', 'h3', 'h4']) +const htmlParseOptions = { scriptingEnabled: false } +const closeOverlayIcon = '' + +function glossaryTooltipHtml(term) { + const readMoreButton = term.hasMultipleParagraphs + ? `` + : '' + return `

${term.term}

${term.tooltip}

${readMoreButton}
` +} + +function glossaryOverlayHtml(term) { + return `` +} + +function descendantElements(node, tags) { + return (node.childNodes || []).flatMap((child) => + tags.has(child.tagName) ? [child, ...descendantElements(child, tags)] : descendantElements(child, tags)) +} + +function parseChildren(node, html) { + const children = parse5.parseFragment(node, html, htmlParseOptions).childNodes + children.forEach((child) => child.parentNode = node) + return children +} + const globalConfig = { onionLocation: "http://isdb4l77sjqoy2qq7ipum6x3at6hyn3jmxfx4zdhc72ufbmuq4ilwkqd.onion", @@ -118,7 +172,11 @@ fs.readdir(translationsDirectoryPath, (err, files) => { const translations = require("./translations.json") +const outputDir = '_site' + module.exports = function (ty) { + ty.on("eleventy.before", () => permalinks.clear()) + // Add this after your markdownLib definition ty.addShortcode("mdInclude", function (filepath) { const fullPath = path.join(__dirname, 'src/_includes', filepath); @@ -126,14 +184,8 @@ module.exports = function (ty) { return markdownLib.render(content); }); - ty.addGlobalData("links", linksData) - ty.addGlobalData("linkLanguages", [...new Set(linksData.map(e => e.language).filter(Boolean))].sort()) - - const catCounts = {} - linksData.forEach(e => { if (e.category) { const c = e.category.toLowerCase(); catCounts[c] = (catCounts[c] || 0) + 1 } }) - const mediaPills = ["Video", "Audio"].filter(p => linksData.some(e => e.mediaType === p.toLowerCase())) - const catPills = Object.keys(catCounts).sort() - ty.addGlobalData("linkPills", mediaPills.concat(catPills)) + ty.addGlobalData("linksPage", readLinksPage) + ty.addWatchTarget(linksFilePath) ty.addShortcode("cfg", (name) => globalConfig[name]) @@ -160,100 +212,52 @@ module.exports = function (ty) { }) ty.addFilter('applyGlossary', function (content) { - const dom = new JSDOM(content) - const { document } = dom.window - const body = document.querySelector('body') - const allContentNodes = document.querySelectorAll('p, td, a, h1, h2, h3, h4') + if (!glossaryTermPattern.test(content)) return content + const document = parse5.parse(content, htmlParseOptions) + const body = document.childNodes.find((node) => node.tagName === 'html').childNodes.find((node) => node.tagName === 'body') + const allContentNodes = descendantElements(document, glossaryContentTags) + const contentHtml = allContentNodes.map((node) => parse5.serialize(node)) const overlayIds = [] + const matchedIds = new Set() - glossary.forEach((term, index) => { - let changeNoted = false - const id = term.term.toLowerCase().replace(/\s/g, '-') - - allContentNodes.forEach((node) => { - const regex = new RegExp(`(? { + glossary.forEach((term) => { + const id = term.id + allContentNodes.forEach((node, nodeIndex) => { + const beforeContent = contentHtml[nodeIndex] + const afterContent = beforeContent.replace(term.pattern, (match) => { return `${match}` }) - if (beforeContent !== node.innerHTML && !changeNoted) { - changeNoted = true + if (afterContent !== beforeContent) { + node.childNodes = parseChildren(node, afterContent) + contentHtml[nodeIndex] = parse5.serialize(node) + matchedIds.add(id) } }) + }) - if (changeNoted) { - const definitionTooltipDiv = document.createElement('div') - definitionTooltipDiv.id = `tooltip-${id}` - definitionTooltipDiv.className = "glossary-tooltip" - const titleH4 = document.createElement('h4') - titleH4.innerHTML = term.term - titleH4.className = "tooltip-title" - const p = document.createElement('p') - p.innerHTML = term.tooltip - const innerDiv = document.createElement('div') - innerDiv.appendChild(titleH4) - innerDiv.appendChild(p) - if (term.hasMultipleParagraphs) { - const readMoreBtn = document.createElement('button') - readMoreBtn.innerHTML = "Read more" - readMoreBtn.className = "read-more-btn open-overlay-btn" - readMoreBtn.setAttribute('data-show-overlay', id) - innerDiv.appendChild(readMoreBtn) - } - innerDiv.className = "tooltip-content" - definitionTooltipDiv.appendChild(innerDiv) - body.appendChild(definitionTooltipDiv) + const neededIds = new Set(matchedIds) + neededIds.forEach((id) => glossaryById.get(id).linkedHashes.forEach((hash) => { + if (glossaryById.has(hash)) neededIds.add(hash) + })) + + glossary.forEach((term) => { + const id = term.id + + if (matchedIds.has(id)) { + body.childNodes.push(...parseChildren(body, glossaryTooltipHtml(term))) } - let tooltipDom = new JSDOM(term.definition) - let tooltipDocument = tooltipDom.window.document - const hashList = [term.term.toLowerCase().replace(/\s/g, '-')] - tooltipDocument.querySelectorAll('a[href*="#"]').forEach(a => { - let hashIndex = a.href.indexOf("#") - if (hashIndex !== -1) { - let hash = a.href.substring(hashIndex + 1) - hashList.push(hash) - } - }) + const hashList = [id, ...term.linkedHashes] hashList.forEach(hash => { - if (!overlayIds.includes(hash)) { - let termFromHash = glossary.find(term => term.term.toLowerCase().replace(/\s/g, '-') === hash) - if (!termFromHash) return - - const overlayDiv = document.createElement('div') - overlayDiv.id = hash - overlayDiv.className = "overlay glossary-overlay hidden" - const overlayCardDiv = document.createElement('div') - overlayCardDiv.className = "overlay-card" - const overlayTitleH1 = document.createElement('h1') - overlayTitleH1.className = "overlay-title" - overlayTitleH1.innerHTML = termFromHash.term - const overlayContent = document.createElement('div') - overlayContent.className = "overlay-content" - overlayContent.innerHTML = termFromHash.definition - const crossSVG = document.createElementNS("http://www.w3.org/2000/svg", "svg") - crossSVG.setAttribute('class', 'close-overlay-btn') - crossSVG.setAttribute('id', 'cross') - crossSVG.setAttribute('width', '16') - crossSVG.setAttribute('height', '16') - crossSVG.setAttribute('viewBox', '0 0 13 13') - crossSVG.setAttribute('xmlns', 'http://www.w3.org/2000/svg') - const crossPath = document.createElementNS("http://www.w3.org/2000/svg", "path") - crossPath.setAttribute('d', 'M12.7973 11.5525L7.59762 6.49833L12.7947 1.44675C13.055 1.19371 13.0658 0.771991 12.8188 0.505331C12.5718 0.238674 12.1602 0.227644 11.8999 0.480681L6.65343 5.58028L1.09979 0.182228C0.805 0.002228 0.430001 0.002228 0.135211 0.182228C-0.159579 0.362228 -0.159579 0.697228 0.135211 0.877228L5.68885 6.27528L0.4918 11.3295C0.231501 11.5825 0.220703 12.0042 0.467664 12.2709C0.714625 12.5376 1.12625 12.5486 1.38655 12.2956L6.63302 7.196L12.1867 12.5941C12.4815 12.7741 12.8565 12.7741 13.1513 12.5941C13.4461 12.4141 13.4461 12.0791 13.1513 11.8991L12.7973 11.5525Z') - crossSVG.appendChild(crossPath) - - overlayCardDiv.appendChild(overlayTitleH1) - overlayCardDiv.appendChild(overlayContent) - overlayCardDiv.appendChild(crossSVG) - overlayDiv.appendChild(overlayCardDiv) - body.appendChild(overlayDiv) + if (neededIds.has(hash) && !overlayIds.includes(hash)) { + body.childNodes.push(...parseChildren(body, glossaryOverlayHtml(glossaryById.get(hash)))) overlayIds.push(hash) } }) }) - return dom.serialize() + return parse5.serialize(document) }) ty.addFilter('wrapH3s', function (content, page) { @@ -334,14 +338,81 @@ module.exports = function (ty) { } }) + ty.addFilter("markdownUrl", (url) => markdownPages.markdownUrl(url, supportedLangs)) + + ty.addFilter("markdownAlternate", (page) => + markdownPages.markdownKind(page.inputPath, supportedLangs) ? markdownPages.markdownUrl(page.url, supportedLangs) : "") + + ty.addFilter("markdownSource", (inputPath) => markdownPages.markdownFromSource(inputPath, replaceLink)) + + ty.addFilter("markdownTitle", markdownPages.markdownTitle) + + ty.addFilter("plainText", markdownPages.plainText) + + const outputSections = ["pages", "blog pages", "doc pages", "language pages", "other files", "Markdown files"] + const outputCounts = new Map() + + function countOutput(section) { + outputCounts.set(section, (outputCounts.get(section) || 0) + 1) + } + + function outputSection(outputPath) { + const [folder] = path.relative(outputDir, outputPath).split(path.sep) + if (supportedLangs.includes(folder)) return "language pages" + if (folder === "blog") return "blog pages" + if (folder === "docs") return "doc pages" + return outputPath.endsWith(".html") ? "pages" : "other files" + } + + ty.on("eleventy.before", () => outputCounts.clear()) + + ty.on("eleventy.after", () => { + const counts = outputSections.filter((section) => outputCounts.has(section)).map((section) => `${outputCounts.get(section)} ${section}`) + console.log(`[11ty] Wrote ${counts.join(", ")}`) + }) + + ty.addTransform("countOutputs", function (content) { + if (this.outputPath) countOutput(outputSection(this.outputPath)) + return content + }) + + ty.addTransform("markdownPages", function (content) { + const kind = markdownPages.markdownKind(this.inputPath, supportedLangs) + if (kind && this.outputPath) { + const pageUrl = "/" + path.relative(outputDir, this.outputPath).replace(/index\.html$/, "") + const markdown = kind === "source" + ? markdownPages.markdownFromSource(this.inputPath, replaceLink) + : markdownPages.markdownFromHtml(content, pageUrl) + const markdownPath = path.join(outputDir, markdownPages.markdownUrl(pageUrl, supportedLangs)) + fs.mkdirSync(path.dirname(markdownPath), { recursive: true }) + fs.writeFileSync(markdownPath, markdown) + countOutput("Markdown files") + } + return content + }) + ty.addPlugin(pluginRss) - ty.addPlugin(i18n, { - translations, - fallbackLocales: { - '*': 'en' - }, - defaultLocale: 'en', + const unknownStrings = new Set() + + ty.addFilter("i18n", function (key, _data, lang) { + const locale = lang || (this.page || this.ctx.page)?.url?.split("/")[1] + const strings = translations[key] || {} + if (strings.en === undefined) unknownStrings.add(key) + return strings[locale] ?? strings.en ?? key + }) + + ty.on("eleventy.before", () => unknownStrings.clear()) + + ty.on("eleventy.after", () => { + const allStrings = Object.values(translations) + const untranslated = supportedLangs.filter((lang) => lang !== "en").sort() + .map((lang) => [lang, allStrings.filter((strings) => strings[lang] === undefined).length]) + .filter(([, count]) => count > 0) + if (untranslated.length > 0) { + console.warn(`[i18n] Untranslated of ${allStrings.length} strings, English used: ${untranslated.map(([lang, count]) => `${lang} ${count}`).join(", ")}`) + } + if (unknownStrings.size > 0) console.warn(`[i18n] Missing in en.json: ${Array.from(unknownStrings).join(", ")}`) }) // Keeps the same directory structure. @@ -453,6 +524,13 @@ module.exports = function (ty) { return newDocs }) + ty.addCollection("markdownDocs", (collection) => + collection.getFilteredByGlob("src/docs/**/*.md") + .filter((doc) => markdownPages.markdownKind(doc.inputPath, supportedLangs) === "source") + .sort((a, b) => a.url.localeCompare(b.url))) + + ty.setQuietMode(true) + ty.setWatchThrottleWaitTime(100) ty.addWatchTarget("src/css") ty.addWatchTarget("markdown/") ty.addWatchTarget("components/Card.js") @@ -509,7 +587,7 @@ module.exports = function (ty) { dir: { input: 'src', includes: '_includes', - output: '_site', + output: outputDir, }, templateFormats: ['md', 'njk', 'html'], markdownTemplateEngine: 'njk', diff --git a/website/copy_sources.js b/website/copy_sources.js new file mode 100644 index 0000000000..124179ac3c --- /dev/null +++ b/website/copy_sources.js @@ -0,0 +1,81 @@ +const fs = require("fs") +const path = require("path") +const customizeDocs = require("./customize_docs_frontmatter") + +const repoDir = path.resolve(__dirname, "..") + +const docs = { + from: "docs", + to: "src/docs", + exclude: [/^contributing\//, /^rfcs\//, /^dependencies\//, /^LINKS\.md$/, /^lang\/[^/]+\/README\.md$/], +} + +const sources = [ + docs, + { from: "docs/links/images", to: "src/link-images", exclude: [] }, + { from: "blog", to: "src/blog", exclude: [/^README\.md$/, /^new\//] }, + { from: "images", to: "src/images", exclude: [] }, + { from: "PRIVACY.md", to: "src/privacy.md", exclude: [] }, +] + +function sourcePath(source, file = "") { + return path.join(repoDir, source.from, file) +} + +function targetPath(source, file = "") { + return path.join(__dirname, source.to, file) +} + +function relativeFiles(root, dir = "") { + if (!fs.statSync(path.join(root, dir)).isDirectory()) return [dir] + return fs.readdirSync(path.join(root, dir)).flatMap((name) => relativeFiles(root, path.join(dir, name))) +} + +function isExcluded(source, file) { + return source.exclude.some((pattern) => pattern.test(file)) +} + +function isDocsMarkdown(source, file) { + return source === docs && file.endsWith(".md") +} + +function sourceFiles(source) { + return relativeFiles(sourcePath(source)).filter((file) => !isExcluded(source, file)) +} + +function copyFile(source, file) { + fs.mkdirSync(path.dirname(targetPath(source, file)), { recursive: true }) + fs.copyFileSync(sourcePath(source, file), targetPath(source, file)) +} + +function writeDocsMarkdown() { + const markdownFiles = sourceFiles(docs).filter((file) => isDocsMarkdown(docs, file)) + customizeDocs(sourcePath(docs), targetPath(docs), markdownFiles) +} + +function copySources() { + sources.filter((source) => fs.existsSync(sourcePath(source))).forEach((source) => { + fs.rmSync(targetPath(source), { recursive: true, force: true }) + sourceFiles(source).filter((file) => !isDocsMarkdown(source, file)).forEach((file) => copyFile(source, file)) + }) + writeDocsMarkdown() +} + +function syncFile(source, event, file) { + if (event === "unlink" || event === "unlinkDir") fs.rmSync(targetPath(source, file), { recursive: true, force: true }) + if (isDocsMarkdown(source, file) || (source === docs && event === "unlinkDir")) writeDocsMarkdown() + else if (event === "add" || event === "change") copyFile(source, file) +} + +function watchSources() { + const chokidar = require("chokidar") + sources.forEach((source) => { + chokidar.watch(sourcePath(source), { ignoreInitial: true }).on("all", (event, changedPath) => { + const file = path.relative(sourcePath(source), changedPath) + if (!isExcluded(source, file)) syncFile(source, event, file) + }) + }) +} + +if (process.argv.includes("--watch")) watchSources() +else copySources() diff --git a/website/customize_docs_frontmatter.js b/website/customize_docs_frontmatter.js index 10031b5436..e2056c380e 100644 --- a/website/customize_docs_frontmatter.js +++ b/website/customize_docs_frontmatter.js @@ -2,93 +2,62 @@ const fs = require('fs'); const path = require('path'); const matter = require('gray-matter'); -const directoryPath = path.resolve(__dirname, 'src/docs'); -const langFolder = 'lang'; -const enFiles = {}; - -function traverseDirectory(directory, currentLanguage = 'en', result = {}, callback) { - const filesAndDirectories = fs.readdirSync(directory); - - filesAndDirectories.forEach((fileOrDirectoryName) => { - const fullPath = path.join(directory, fileOrDirectoryName); - - if (fs.statSync(fullPath).isDirectory()) { - // If the subdirectory is inside the 'lang' folder, update the current language - if (directory.endsWith('/lang')) { - currentLanguage = fileOrDirectoryName; - } - - // Recursively traverse the subdirectories - traverseDirectory(fullPath, currentLanguage, result, callback); - } else { - // Process the file only if it has the '.md' extension - if (path.extname(fullPath) === '.md') { - // Add the language to the file's language array or create a new array if it doesn't exist - const fileName = path.basename(fullPath, '.md'); - if (!result[fileName]) { - result[fileName] = []; - } - result[fileName].push(currentLanguage); - } - if (callback) { - callback(fullPath, currentLanguage); - } - } - }); - - return result; +function docLanguage(relativePath) { + const [folder, language] = relativePath.split(path.sep); + return folder === 'lang' ? language : 'en'; } -const fileLanguageMapping = traverseDirectory(directoryPath); +function customizeDocs(sourceDir, destDir, relativePaths) { + const docs = relativePaths.map((relativePath) => { + const parsedMatter = matter(fs.readFileSync(path.join(sourceDir, relativePath), 'utf-8')); + return { + relativePath, + fileName: path.basename(relativePath, '.md'), + language: docLanguage(relativePath), + content: parsedMatter.content, + data: { ...parsedMatter.data }, + }; + }); -// Update the frontmatter of each Markdown file -Object.entries(fileLanguageMapping).forEach(([fileName, languages]) => { - // Find and update the frontmatter of each Markdown file - traverseDirectory(directoryPath, null, {}, (fullPath, currentLanguage) => { - if (path.basename(fullPath) === `${fileName}.md`) { - // Read the existing frontmatter - const fileContent = fs.readFileSync(fullPath, 'utf-8'); - const parsedMatter = matter(fileContent); - const relativePath = path.relative(directoryPath, fullPath); + const languagesByFileName = new Map(); + docs.forEach(({ fileName, language }) => { + if (!languagesByFileName.has(fileName)) languagesByFileName.set(fileName, []); + languagesByFileName.get(fileName).push(language); + }); - // Calculate the permalink based on the file's location - const linkPath = path.relative(directoryPath, fullPath).replace(/\.md$/, '.html'); - const permalink = `/docs/${linkPath}`.toLowerCase(); + const enRevisions = new Map(docs + .filter((doc) => doc.language === 'en') + .map((doc) => [doc.relativePath, doc.data.revision])); - if (fileName === 'JOIN_TEAM') { - parsedMatter.data.active_jobs = true; + docs.forEach(({ relativePath, fileName, language, content, data }) => { + const permalink = `/docs/${relativePath.replace(/\.md$/, '.html')}`.toLowerCase(); + + if (fileName === 'JOIN_TEAM') { + data.active_jobs = true; + } + if (!data.permalink) data.permalink = permalink; + + data.supportedLangsForDoc = languagesByFileName.get(fileName); + + if (!data.layout) data.layout = 'layouts/doc.html'; + + if (language === 'en') { + data.version = 'new'; + } else { + const enRelativePath = path.join(...relativePath.split(path.sep).slice(2)); + if (enRevisions.has(enRelativePath)) { + const isOld = new Date(data.revision) < new Date(enRevisions.get(enRelativePath)); + data.version = isOld ? 'old' : 'new'; } - if (!parsedMatter.data.permalink) parsedMatter.data.permalink = permalink; + } - // Update the frontmatter with the new languages list - parsedMatter.data.supportedLangsForDoc = languages; - - // Add the layout value - if (!parsedMatter.data.layout) parsedMatter.data.layout = 'layouts/doc.html'; - - if (fullPath.startsWith(path.join(directoryPath, langFolder))) { - // Non-English files - const [language, ...rest] = relativePath.split(path.sep).slice(1); - const enFilePath = path.join(directoryPath, ...rest); - - if (enFiles[enFilePath]) { - const enRevision = new Date(enFiles[enFilePath].revision); - const currentRevision = new Date(parsedMatter.data.revision); - - const isOld = currentRevision < enRevision; - // Add the version value - parsedMatter.data.version = isOld ? 'old' : 'new'; - } - } else { - // English files - enFiles[fullPath] = { revision: parsedMatter.data.revision }; - // Add the version value - parsedMatter.data.version = 'new'; - } - - // Save the updated frontmatter and content back to the file - const updatedFileContent = matter.stringify(parsedMatter.content, parsedMatter.data); - fs.writeFileSync(fullPath, updatedFileContent, 'utf-8'); + const destPath = path.join(destDir, relativePath); + const updatedFileContent = matter.stringify(content, data); + if (!fs.existsSync(destPath) || fs.readFileSync(destPath, 'utf-8') !== updatedFileContent) { + fs.mkdirSync(path.dirname(destPath), { recursive: true }); + fs.writeFileSync(destPath, updatedFileContent, 'utf-8'); } }); -}); +} + +module.exports = customizeDocs; diff --git a/website/langs/cs.json b/website/langs/cs.json index ef06c53676..90bad228aa 100644 --- a/website/langs/cs.json +++ b/website/langs/cs.json @@ -101,7 +101,7 @@ "simplex-unique-overlay-card-3-p-2": "End-to-end šifrované zprávy jsou dočasně uchovávány na přenosových serverech SimpleX, dokud nejsou přijaty, poté jsou trvale odstraněny.", "simplex-unique-overlay-card-3-p-3": "Na rozdíl od serverů federovaných sítí (e-mail, XMPP nebo Matrix) servery SimpleX neukládají uživatelské účty, pouze předávají zprávy, čímž chrání soukromí obou stran.", "simplex-unique-overlay-card-4-p-1": "Můžete použít SimpleX se svými vlastními servery a přesto komunikovat s lidmi, kteří používají přednastavené servery v aplikacích.", - "simplex-unique-overlay-card-4-p-3": "Pokud uvažujete o vývoji pro SimpleX síť, například chat bot pro uživatele aplikace SimpleX nebo integraci knihovny SimpleX chat do Vasí mobilní aplikace, prosím buďte ve spojení pro jakoukoli radu a podporu.", + "simplex-unique-overlay-card-4-p-3": "Pokud uvažujete o vývoji pro SimpleX síť, například chat bot pro uživatele aplikace SimpleX nebo integraci knihovny SimpleX chat do Vasí mobilní aplikace, prosím buďte ve spojení pro jakoukoli radu a podporu.", "simplex-unique-card-1-p-1": "SimpleX chrání soukromí vašeho profilu, kontaktů a metadat a skrývá je před servery SimpleX sítě a jakýmikoli pozorovateli.", "simplex-unique-card-1-p-2": "Na rozdíl od jakékoli jiné existující síti pro zasílání zpráv nemá SimpleX žádné identifikátory přiřazené uživatelům — ani náhodná čísla.", "simplex-unique-card-3-p-1": "SimpleX Chat ukládá všechna uživatelská data pouze na klientských zařízeních pomocí přenosného šifrovaného databázového formátu, který lze exportovat a přenést na jakékoli podporované zařízení.", diff --git a/website/langs/en.json b/website/langs/en.json index 8e4fba8470..24dde29b39 100644 --- a/website/langs/en.json +++ b/website/langs/en.json @@ -331,6 +331,7 @@ "why-p7": "The oldest human freedom — to speak to another person without being watched — built on infrastructure that cannot betray it.", "why-p8": "Because we destroyed the power to know who you are. So that your power can never be taken.", "why-tagline": "Be free in your network.", + "why-film": "Watch the film", "why-footer-link": "Why we are building it", "send-file": "Send file", "file-desc": "Send files securely with end-to-end encryption — no accounts, no tracking.", diff --git a/website/markdown_pages.js b/website/markdown_pages.js new file mode 100644 index 0000000000..2671de3690 --- /dev/null +++ b/website/markdown_pages.js @@ -0,0 +1,272 @@ +const fs = require("fs") +const path = require("path") +const matter = require("gray-matter") +const parse5 = require("parse5") + +const siteLocation = "https://simplex.chat" + +const htmlPages = new Set([ + "index", + "why", + "messaging", + "fdroid", + "contact", + "invitation", + "file", + "crowdfunding", + "crowdfunding-news", + "news", + "livestream", + "blog", + "links", + "directory", +]) + +const removedTags = new Set([ + "script", + "style", + "noscript", + "svg", + "button", + "select", + "input", + "textarea", +]) + +const removedIds = new Set([ + "navbar", + "mobile-header", +]) + +const removedClasses = new Set([ + "footer", + "glossary-tooltip", + "glossary-overlay", +]) + +const blockTags = new Set([ + "address", + "article", + "aside", + "dd", + "details", + "div", + "dl", + "dt", + "fieldset", + "figcaption", + "figure", + "form", + "header", + "main", + "nav", + "section", + "summary", +]) + +const codeFencePattern = /(^```[\s\S]*?^```[^\n]*$)/gm +const linkTargetPattern = /\]\((<[^>]*>|[^)\s]+)/g +const referenceTargetPattern = /^( {0,3}\[[^\]]+\]:[ \t]*)(<[^>]*>|\S+)/gm +const textEntityPattern = /&(?!lt;|gt;|amp;)(#\d+|#x[0-9a-f]+|[a-z]+\d*);/gi +const headingPattern = /^#\s+(.+)$/m +const invisibleCharacters = /[​⁠]/g + +function markdownKind(inputPath, langs) { + const relativePath = path.relative("src", inputPath) + if (/^docs\/(?!lang\/).+\.md$/.test(relativePath)) return "source" + if (/^blog\/[^/]+\.md$/.test(relativePath)) return "source" + if (relativePath === "privacy.md" || relativePath === "token.md") return "source" + const parts = relativePath.split("/") + const fileName = parts.length === 1 ? parts[0] : parts.length === 2 && langs.includes(parts[0]) ? parts[1] : "" + return fileName.endsWith(".html") && htmlPages.has(fileName.slice(0, -5)) ? "html" : null +} + +function markdownUrl(url, langs) { + if (url.endsWith(".html")) return url.slice(0, -5) + ".md" + const isRoot = url === "/" || langs.some((lang) => url === `/${lang}/`) + return isRoot ? url + "index.md" : url.slice(0, -1) + ".md" +} + +function isMarkdownPagePath(pathname) { + const isDoc = pathname.startsWith("/docs/") && !pathname.startsWith("/docs/lang/") + return pathname.endsWith(".html") && (isDoc || pathname.startsWith("/blog/")) +} + +function absoluteMarkdownLink(link, inputPath, resolveLink) { + if (link.startsWith("<")) return `<${absoluteMarkdownLink(link.slice(1, -1), inputPath, resolveLink)}>` + if (link.startsWith("#")) return link + let resolved + try { + resolved = resolveLink(link, { page: { inputPath } }) + } catch { + return link + } + if (!resolved.startsWith("/")) return resolved + const [pathname, fragment] = resolved.split("#") + const target = isMarkdownPagePath(pathname) ? pathname.slice(0, -5) + ".md" : pathname + return siteLocation + target + (fragment === undefined ? "" : "#" + fragment) +} + +function markdownFromSource(inputPath, resolveLink) { + const { content } = matter(fs.readFileSync(inputPath, "utf8")) + const rewrite = (link) => absoluteMarkdownLink(link, inputPath, resolveLink) + const markdown = content + .split(codeFencePattern) + .map((part, index) => index % 2 === 1 ? part : part + .replace(linkTargetPattern, (_, link) => "](" + rewrite(link)) + .replace(referenceTargetPattern, (_, prefix, link) => prefix + rewrite(link)) + .replace(textEntityPattern, plainText)) + .join("") + return markdown.trim() + "\n" +} + +function markdownTitle(inputPath) { + const { data, content } = matter(fs.readFileSync(inputPath, "utf8")) + return plainText(data.title || content.match(headingPattern)?.[1] || "") +} + +function attribute(node, name) { + return node.attrs.find((attr) => attr.name === name)?.value +} + +function isRemoved(element) { + const classes = (attribute(element, "class") || "").split(/\s+/) + return removedTags.has(element.tagName) + || (element.tagName === "template" && attribute(element, "data-markdown") === undefined) + || attribute(element, "data-markdown-skip") !== undefined + || removedIds.has(attribute(element, "id")) + || classes.some((name) => removedClasses.has(name)) +} + +function childElements(node) { + return node.childNodes.filter((child) => child.tagName && !isRemoved(child)) +} + +function findElements(node, predicate) { + return childElements(node).flatMap((child) => predicate(child) ? [child] : findElements(child, predicate)) +} + +function findElement(node, predicate) { + return findElements(node, predicate)[0] +} + +function textContent(node) { + return node.nodeName === "#text" ? node.value : (node.childNodes || []).map(textContent).join("") +} + +function singleLine(text) { + return text.trim().replace(/\s*\n\s*/g, " ") +} + +function emphasis(marker, content) { + return content.replace(/^(\s*)([\s\S]*?)(\s*)$/, (_, leading, text, trailing) => + text ? leading + marker + text + marker + trailing : leading + trailing) +} + +function linkMarkdown(element, content, pageUrl) { + const href = attribute(element, "href") + if (href && href.startsWith("javascript:")) return "" + const image = findElement(element, (child) => child.tagName === "img" && attribute(child, "alt") !== undefined) + const label = attribute(element, "aria-label") || singleLine(content) || attribute(element, "title") || (image && attribute(image, "alt")) || "" + if (!href) return label + return label ? `[${label}](${new URL(href, siteLocation + pageUrl).href})` : "" +} + +function listMarkdown(element, ordered, pageUrl) { + const items = childElements(element).filter((child) => child.tagName === "li") + const lines = items.map((item, index) => (ordered ? `${index + 1}. ` : "- ") + singleLine(childrenMarkdown(item, pageUrl))) + return `\n\n${lines.join("\n")}\n\n` +} + +function tableMarkdown(element, pageUrl) { + const rows = findElements(element, (child) => child.tagName === "tr").map((row) => + childElements(row).map((cell) => singleLine(childrenMarkdown(cell, pageUrl)).replace(/\|/g, "\\|"))) + if (rows.length === 0) return "" + const width = Math.max(...rows.map((row) => row.length)) + const line = (cells) => "| " + Array.from({ length: width }, (_, index) => cells[index] || "").join(" | ") + " |" + const [header, ...body] = rows + return `\n\n${[line(header), line(Array(width).fill("---")), ...body.map(line)].join("\n")}\n\n` +} + +function nodeMarkdown(node, pageUrl) { + if (node.nodeName === "#text") return node.value.replace(invisibleCharacters, "").replace(/\s+/g, " ") + if (!node.tagName || isRemoved(node)) return "" + const tag = node.tagName + const content = () => childrenMarkdown(node, pageUrl) + switch (tag) { + case "h1": + case "h2": + case "h3": + case "h4": + case "h5": + case "h6": + return `\n\n${"#".repeat(Number(tag[1]))} ${singleLine(content())}\n\n` + case "p": + return `\n\n${content().trim()}\n\n` + case "br": + return "\n" + case "hr": + return "\n\n---\n\n" + case "a": + return linkMarkdown(node, content(), pageUrl) + case "strong": + case "b": + return emphasis("**", content()) + case "em": + case "i": + return emphasis("*", content()) + case "code": + return "`" + textContent(node) + "`" + case "pre": + return "\n\n```\n" + textContent(node).trim() + "\n```\n\n" + case "sup": + return `[${content().trim()}]` + case "img": + return "" + case "ul": + case "ol": + return listMarkdown(node, tag === "ol", pageUrl) + case "table": + return tableMarkdown(node, pageUrl) + case "blockquote": + return "\n\n" + content().trim().split("\n").map((line) => "> " + line).join("\n") + "\n\n" + case "template": + return childrenMarkdown(node.content, pageUrl) + default: + return blockTags.has(tag) ? `\n\n${content()}\n\n` : content() + } +} + +function childrenMarkdown(node, pageUrl) { + return node.childNodes.map((child) => nodeMarkdown(child, pageUrl)).join("") +} + +function normalizedMarkdown(markdown) { + let inFence = false + const lines = markdown.split("\n").map((line) => { + if (line.startsWith("```")) inFence = !inFence + return inFence || line.startsWith("```") ? line : line.trim() + }) + return lines.join("\n").replace(/\n{3,}/g, "\n\n").trim() + "\n" +} + +function markdownFromHtml(html, pageUrl) { + const document = parse5.parse(html) + const body = findElement(document, (node) => node.tagName === "body") + const title = findElement(document, (node) => node.tagName === "title") + const heading = findElement(body, (node) => node.tagName === "h1") ? "" : `# ${singleLine(title ? textContent(title) : "")}\n\n` + return normalizedMarkdown(heading + childrenMarkdown(body, pageUrl)) +} + +function plainText(html) { + return textContent(parse5.parseFragment(html)) +} + +module.exports = { + markdownKind, + markdownUrl, + markdownFromSource, + markdownTitle, + markdownFromHtml, + plainText, +} diff --git a/website/package.json b/website/package.json index 5e4e947bcf..7aa8a1399e 100644 --- a/website/package.json +++ b/website/package.json @@ -31,12 +31,12 @@ "dependencies": { "@plausible-analytics/tracker": "0.4.6", "@simplex-chat/xftp-web": "^0.3.0", - "eleventy-plugin-i18n": "^0.1.3", "fs": "^0.0.1-security", "gray-matter": "^4.0.3", "jsdom": "^22.1.0", "lottie-web": "5.12.2", "markdown-it": "^13.0.1", - "markdown-it-footnote": "^4.0.0" + "markdown-it-footnote": "^4.0.0", + "parse5": "^7.1.2" } } diff --git a/website/plans/2026-10-08-llms.md b/website/plans/2026-10-08-llms.md new file mode 100644 index 0000000000..8d604a79e0 --- /dev/null +++ b/website/plans/2026-10-08-llms.md @@ -0,0 +1,99 @@ +# LLM content for simplex.chat + +## Outputs + +### Index files + +| URL | Content | +|---|---| +| `/llms.txt` | site name, summary, sections of links to Markdown pages | +| `/llms-full.txt` | core pages in one file | +| `/docs.md` | every English documentation page | + +### Markdown versions of site pages + +URL rule: + +| Page URL | Markdown URL | +|---|---| +| `/` | `/index.md` | +| `/xx/` (language root) | `/xx/index.md` | +| `/x/` | `/x.md` | +| `/docs/x.html` | `/docs/x.md` | +| `/blog/x.html` | `/blog/x.md` | + +Sources: + +| Pages | Markdown made from | +|---|---| +| English `docs/**/*.md`, `blog/*.md`, `privacy.md`, `token.md` | source text: front matter removed, links made absolute, HTML entities decoded | +| `index`, `why`, `messaging`, `fdroid`, `contact`, `invitation`, `file` (every language) | rendered HTML | +| `crowdfunding`, `crowdfunding-news`, `news`, `livestream`, `blog`, `links`, `directory` | rendered HTML | + +Translated docs and blog posts: no Markdown version. + +HTML to Markdown markup: + +| Markup | Effect | +|---|---| +| `data-markdown-skip` | element left out | +| `