core: chat relays protocol (#6383)

* core: chat relays protocol wip

* types, notes

* remove file

* removal protocol

* schema

* status

* update

* recovery

* update

* formatting

* rename

* more types

* comment

* more docs

* decrease number of steps

* format

* correct

* update

* update protocol

* update

* typo

* todo

* update doc

* update

* update

* remove added

* update

* update

* XGrpRelayReady

* link to chat relays

* update

* remove from protocol

* update

* json

* wip

* remove comment

* wip

* update

* wip

* wip

* update

* wip

* wip

* plans

* better view

* fix

* fix

* relay acceptance

* rework api

* add relays to link

* comment

* active on con, fix send

* comments

* direct in group plan

* prepare

* member connection wip

* comments

* member connection wip

* fix forwarding

* introduce moderators to new member

* enable relay tests

* plans

* security objectives

* refactor

* add to threat model

* stress test wip

* stress test wip

* Revert "stress test wip"

This reverts commit acde8a1fb3.

* Revert "stress test wip"

This reverts commit 6435808438.

* remove stress test

* improve output

* invert relay fkey

* postgres schema

* comments

* group in progress, remove auto-select relays commented code

* comments

* corrections

* comment

* lint

* redundant import

* core: chat relay request worker (#6509)

* update plans

* strict tables

* core: update group link asynchronously with relay link (#6548)

* update simplexmq

* docs: connection to chat relays rfc (#6554)

* add test for 2 relays (doesn't pass)

* create unknown member in same transaction as checking

* fix relays choosing different memberId (XContactRelay)

* plans, api

* use same incognito profile for relays, connect concurrently, save correct link for plan

* test

* don't duplicate items on group connection

* check relay record exists when joining

* use mapConcurrently when adding relays, update schemas

* fix multi-relay join for postgres (savepoint)

* core: async retry connection to chat relays (#6584)

* update simplexmq

* fix api tests

* prefer throwing temp error on connection

* check group relays when deleting from configuration

* relay_request_err_reason

* relay role

* rename, fix syntax

* plans

* rename, style

---------

Co-authored-by: Evgeny Poberezkin <evgeny@poberezkin.com>

bots/src/API/Docs/Commands.hs

@@ -117,6 +117,7 @@ chatCommandsDocsData =
         ("APILeaveGroup", [], "Leave group.", ["CRLeftMemberUser", "CRChatCmdError"], [], Just UNBackground, "/_leave #" <> Param "groupId"),
         ("APIListMembers", [], "Get group members.", ["CRGroupMembers", "CRChatCmdError"], [], Nothing, "/_members #" <> Param "groupId"),
         ("APINewGroup", [], "Create group.", ["CRGroupCreated", "CRChatCmdError"], [], Nothing, "/_group " <> Param "userId" <> OnOffParam "incognito" "incognito" (Just False) <> " " <> Json "groupProfile"),
+        ("APINewPublicGroup", [], "Create public group.", ["CRPublicGroupCreated", "CRChatCmdError"], [], Just UNInteractive, "/_public group " <> Param "userId" <> OnOffParam "incognito" "incognito" (Just False) <> " " <> Join ',' "relayIds" <> " " <> Json "groupProfile"),
         ("APIUpdateGroupProfile", [], "Update group profile.", ["CRGroupUpdated", "CRChatCmdError"], [], Just UNBackground, "/_group_profile #" <> Param "groupId" <> " " <> Json "groupProfile")
       ]
     ),
@@ -240,6 +241,7 @@ cliCommands =
     "MemberRole",
     "MuteUser",
     "NewGroup",
+    "NewPublicGroup",
     "QuitChat",
     "ReactToMessage",
     "RejectContact",

bots/src/API/Docs/Events.hs

@@ -97,7 +97,8 @@ chatEventsDocsData =
       [ ("CEvtConnectedToGroupMember", "Connected to another group member."),
         ("CEvtMemberAcceptedByOther", "Another group owner, admin or moderator accepted member to the group after review (\"knocking\")."),
         ("CEvtMemberBlockedForAll", "Another member blocked for all members."),
-        ("CEvtGroupMemberUpdated", "Another group member profile updated.")
+        ("CEvtGroupMemberUpdated", "Another group member profile updated."),
+        ("CEvtGroupLinkRelaysUpdated", "Group link relays updated.")
       ]
     ),
     ( "File events",

bots/src/API/Docs/Responses.hs

@@ -68,6 +68,7 @@ chatResponsesDocsData =
     ("CRGroupLinkCreated", ""),
     ("CRGroupLinkDeleted", ""),
     ("CRGroupCreated", ""),
+    ("CRPublicGroupCreated", ""),
     ("CRGroupMembers", ""),
     ("CRGroupUpdated", ""),
     ("CRGroupsList", "Groups"),

bots/src/API/Docs/Types.hs

@@ -275,12 +275,13 @@ chatTypesDocsData =
     (sti @GroupMemberAdmission, STRecord, "", [], "", ""),
     (sti @GroupMemberCategory, (STEnum' $ dropPfxSfx "GC" "Member"), "", [], "", ""),
     (sti @GroupMemberRef, STRecord, "", [], "", ""),
-    (sti @GroupMemberRole, STEnum, "GR", [], "", ""),
+    (sti @GroupMemberRole, (STEnum' $ dropPfxSfx "GR" ""), "", ["GRUnknown"], "", ""),
     (sti @GroupMemberSettings, STRecord, "", [], "", ""),
     (sti @GroupMemberStatus, (STEnum' $ (\case "group_deleted" -> "deleted"; "intro_invited" -> "intro-inv"; s -> s) . consSep "GSMem" '_'), "", [], "", ""),
     (sti @GroupPreference, STRecord, "", [], "", ""),
     (sti @GroupPreferences, STRecord, "", [], "", ""),
     (sti @GroupProfile, STRecord, "", [], "", ""),
+    (sti @GroupRelay, STRecord, "", [], "", ""),
     (sti @GroupShortLinkData, STRecord, "", [], "", ""),
     (sti @GroupSummary, STRecord, "", [], "", ""),
     (sti @GroupSupportChat, STRecord, "", [], "", ""),
@@ -320,6 +321,7 @@ chatTypesDocsData =
     (sti @RcvFileStatus, STUnion, "RFS", [], "", ""),
     (sti @RcvFileTransfer, STRecord, "", [], "", ""),
     (sti @RcvGroupEvent, STUnion, "RGE", [], "", ""),
+    (sti @RelayStatus, STEnum, "RS", [], "", ""),
     (sti @ReportReason, (STEnum' $ dropPfxSfx "RR" ""), "", ["RRUnknown"], "", ""),
     (sti @RoleGroupPreference, STRecord, "", [], "", ""),
     (sti @SecurityCode, STRecord, "", [], "", ""),
@@ -468,6 +470,7 @@ deriving instance Generic GroupMemberStatus
 deriving instance Generic GroupPreference
 deriving instance Generic GroupPreferences
 deriving instance Generic GroupProfile
+deriving instance Generic GroupRelay
 deriving instance Generic GroupShortLinkData
 deriving instance Generic GroupSummary
 deriving instance Generic GroupSupportChat
@@ -513,6 +516,7 @@ deriving instance Generic RcvFileDescr
 deriving instance Generic RcvFileStatus
 deriving instance Generic RcvFileTransfer
 deriving instance Generic RcvGroupEvent
+deriving instance Generic RelayStatus
 deriving instance Generic ReportReason
 deriving instance Generic SecurityCode
 deriving instance Generic SimplexLinkType