review fixes

- core: reject service payloads nested deeper than 32 levels, so a hostile service cannot crash the app's JSON decoder
- core: one parseServiceBody (decompress, decode, depth check) at all three decode points: incoming requests, sendServiceRequestTo, redeemBadgeCode
- directory: '%' and '_' in search text match literally instead of acting as LIKE wildcards
- directory: search text over 100 characters is rejected over RPC
- directory: a new directory creates its address with DR keys, so service requests work without a manual key rotation
- directory: service requests are always processed; the --service-requests switch is no longer needed for the directory
- directory: the in-flight service request cap is a DirectoryOpts field (default 8) instead of a hardcoded constant
- directory: the page-fit check compresses the exact response object that is sent, not a proxy with a different key order and cursor
- directory: search entries omit the full link when a short link exists, saving hundreds of bytes per entry in the envelope
- ios: add DirectorySearch.swift and DirectorySearchView.swift to the Xcode project; the app did not build without them
- ios: in the onboarding state the search bar is at the bottom in one-hand mode, as in the chat list
- ios: in the onboarding state a failed search shows the retry row instead of returning to the cards
- ios: a search with no results shows a "No results" row
- ios: the "No chats found" overlay no longer covers directory results, the retry row or the no-results row
- ios: directory rows are rendered by one builder for both the onboarding and chat-list branches
- ios: retrying after a failed "Show more" resumes from the cursor instead of restarting the search
- ios: cancelling the spinner, or tapping a result mid-search, drops the pending request without clearing results already shown
- ios: typing during a search stops the spinner, and a stale response no longer clears a newer search's spinner
- ios: repeated Search taps for the same text while a search is in flight are ignored
- ios: channel results show the subscriber count and channel icon; group results show the group icon
- ios: remove the unused searchText parameter of SearchInDirectoryRow
- android: a directory search timeout no longer pops the retry alert (the UI will show a retry row)
- tests: RPC paging by the echoed cursor, literal wildcards and the length bound, rejection over the cap, page fitting to the envelope, nested payload rejection
This commit is contained in:
spaced4ndy
2026-09-18 16:45:42 +04:00
parent 125bd0e3dd
commit 54ed1d846a
14 changed files with 280 additions and 130 deletions
@@ -438,39 +438,42 @@ struct ChatListView: View {
parentSheet: $sheet,
directorySearch: directorySearch
)
.scaleEffect(x: 1, y: oneHandUI ? -1 : 1, anchor: .center)
.padding(.horizontal)
if directorySearch.entries.isEmpty && !directorySearch.loading {
ConnectOnboardingView()
if directorySearch.showResults {
List { directoryRows() }.listStyle(.plain)
} else {
directorySearchList()
ConnectOnboardingView()
.scaleEffect(x: 1, y: oneHandUI ? -1 : 1, anchor: .center)
}
}
.scaleEffect(x: 1, y: oneHandUI ? -1 : 1, anchor: .center)
.modifier(ThemedBackground())
} else {
chatListContent
}
}
// the directory results on their own, for the onboarding state where there is no chat list
@ViewBuilder private func directorySearchList() -> some View {
List {
ForEach(directorySearch.entries) { entry in
DirectorySearchRow(entry: entry)
.listRowBackground(Color.clear)
.onTapGesture {
guard let link = entry.connectLink else { return }
searchFocussed = false
planAndConnect(link, theme: theme, dismiss: false, cleanup: nil)
}
}
if directorySearch.failed {
directoryRetryRow()
} else if directorySearch.hasMore {
directoryShowMoreRow()
}
@ViewBuilder private func directoryRows() -> some View {
ForEach(directorySearch.entries) { entry in
DirectorySearchRow(entry: entry)
.scaleEffect(x: 1, y: oneHandUI ? -1 : 1, anchor: .center)
.listRowBackground(Color.clear)
.onTapGesture {
guard let link = entry.connectLink else { return }
searchFocussed = false
planAndConnect(link, theme: theme, dismiss: false, cleanup: nil)
}
}
if directorySearch.failed {
directoryRetryRow()
} else if directorySearch.hasMore {
directoryShowMoreRow()
} else if directorySearch.searched && directorySearch.entries.isEmpty {
Text("No results")
.foregroundColor(theme.colors.secondary)
.scaleEffect(x: 1, y: oneHandUI ? -1 : 1, anchor: .center)
.listRowBackground(Color.clear)
}
.listStyle(.plain)
}
private var chatListContent: some View {
@@ -545,23 +548,9 @@ struct ChatListView: View {
.disabled(chatModel.chatRunning != true || chatModel.deletedChats.contains(chat.chatInfo.id))
}
}
if !directorySearch.entries.isEmpty || directorySearch.loading || directorySearch.failed {
if directorySearch.showResults {
Section {
ForEach(directorySearch.entries) { entry in
DirectorySearchRow(entry: entry)
.scaleEffect(x: 1, y: oneHandUI ? -1 : 1, anchor: .center)
.listRowBackground(Color.clear)
.onTapGesture {
guard let link = entry.connectLink else { return }
searchFocussed = false
planAndConnect(link, theme: theme, dismiss: false, cleanup: nil)
}
}
if directorySearch.failed {
directoryRetryRow()
} else if directorySearch.hasMore {
directoryShowMoreRow()
}
directoryRows()
} header: {
Text("Directory")
.scaleEffect(x: 1, y: oneHandUI ? -1 : 1, anchor: .center)
@@ -593,8 +582,8 @@ struct ChatListView: View {
}
}
}
// the overlay covers the list, so it must not appear while directory results are shown
if cs.isEmpty && !chatModel.chats.isEmpty && directorySearch.entries.isEmpty && !directorySearch.loading {
// the overlay covers the list, so it yields to the directory section and its own empty and retry rows
if cs.isEmpty && !chatModel.chats.isEmpty && !directorySearch.showResults {
noChatsView()
.scaleEffect(x: 1, y: oneHandUI ? -1 : 1, anchor: .center)
.foregroundColor(.secondary)
@@ -616,7 +605,14 @@ struct ChatListView: View {
@ViewBuilder private func directoryRetryRow() -> some View {
Button {
Task { await directorySearch.search(searchText) }
// a failed "Show more" keeps its cursor, so the retry resumes the page rather than starting over
Task {
if directorySearch.hasMore {
await directorySearch.loadMore()
} else {
await directorySearch.search(searchText)
}
}
} label: {
Text("Search failed, tap to retry")
.foregroundColor(theme.colors.primary)
@@ -974,11 +970,7 @@ struct ChatListSearchBar: View {
@ViewBuilder private func searchInDirectoryRow() -> some View {
// a SimpleX link connects on its own, so the directory is not offered for one
if !searchShowingSimplexLink {
SearchInDirectoryRow(
searchText: searchTrimmed,
searchFocussed: $searchFocussed,
onSearch: runDirectorySearch
)
SearchInDirectoryRow(searchFocussed: $searchFocussed, onSearch: runDirectorySearch)
}
}
@@ -27,6 +27,7 @@ class DirectorySearchModel: ObservableObject {
private var generation = 0
var hasMore: Bool { cursor != nil }
var showResults: Bool { loading || searched }
func reset() {
generation += 1
@@ -36,11 +37,18 @@ class DirectorySearchModel: ObservableObject {
failed = false
searched = false
searchedText = ""
ConnectProgressManager.shared.stopConnectProgress(.directorySearch)
}
// a way out of the wait, not out of the results: the page already shown stays
func cancelRequest() {
generation += 1
loading = false
}
func search(_ text: String) async {
let text = text.trimmingCharacters(in: .whitespaces)
guard !text.isEmpty else { return }
guard !text.isEmpty, !(loading && text == searchedText) else { return }
reset()
searchedText = text
await request(append: false)
@@ -59,11 +67,11 @@ class DirectorySearchModel: ObservableObject {
NSLocalizedString("Searching directory…", comment: "in progress text"),
owner: .directorySearch
) { [weak self] in
Task { @MainActor in self?.reset() }
Task { @MainActor in self?.cancelRequest() }
}
let r = await apiSearchDirectory(searchedText, cursor: cursor)
ConnectProgressManager.shared.stopConnectProgress(.directorySearch)
guard gen == generation else { return }
ConnectProgressManager.shared.stopConnectProgress(.directorySearch)
loading = false
searched = true
guard let r else {
@@ -82,7 +90,6 @@ class DirectorySearchModel: ObservableObject {
// text to the directory.
struct SearchInDirectoryRow: View {
@EnvironmentObject var theme: AppTheme
var searchText: String
@FocusState.Binding var searchFocussed: Bool
var onSearch: () -> Void
@@ -109,7 +116,11 @@ struct DirectorySearchRow: View {
var body: some View {
HStack(spacing: 8) {
ProfileImage(imageStr: entry.image, size: 42)
ProfileImage(
imageStr: entry.image,
iconName: isChannel ? "antenna.radiowaves.left.and.right.circle.fill" : "person.2.circle.fill",
size: 42
)
VStack(alignment: .leading, spacing: 2) {
HStack(spacing: 4) {
Text(displayName).fontWeight(.bold).lineLimit(1)
@@ -137,11 +148,14 @@ struct DirectorySearchRow: View {
private var displayName: String { "#" + entry.displayName }
private var isChannel: Bool { entry.entryType.groupType == .channel }
private var membersText: String {
String.localizedStringWithFormat(
NSLocalizedString("%d members", comment: "directory search result"),
Int(entry.entryType.summary.currentMembers)
)
let summary = entry.entryType.summary
let count = summary.publicMemberCount ?? summary.currentMembers
return isChannel
? subscriberCountStr(count)
: String.localizedStringWithFormat(NSLocalizedString("%d members", comment: "directory search result"), Int(count))
}
}
@@ -266,6 +266,8 @@
E5AEC0AF2F91A73500270665 /* ComposeChatLinkView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5AEC0AE2F91A73500270665 /* ComposeChatLinkView.swift */; };
E5C0BBE82F82B45500EA7527 /* SimpleXAssets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = E5C0BBE72F82B45500EA7527 /* SimpleXAssets.xcassets */; };
E5C0BBE92F82B45500EA7527 /* SimpleXAssets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = E5C0BBE72F82B45500EA7527 /* SimpleXAssets.xcassets */; };
E5D5A0012F9B0000AAAA0001 /* DirectorySearch.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5D5A0022F9B0000AAAA0001 /* DirectorySearch.swift */; };
E5D5A0032F9B0000AAAA0001 /* DirectorySearchView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5D5A0042F9B0000AAAA0001 /* DirectorySearchView.swift */; };
E5DBF1932F88169800E1D7FD /* ConnectBannerCard.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5DBF1922F88169800E1D7FD /* ConnectBannerCard.swift */; };
E5DCF8DB2C56FAC1007928CC /* SimpleXChat.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5CE2BA682845308900EC33A6 /* SimpleXChat.framework */; };
E5DCF9712C590272007928CC /* Localizable.strings in Resources */ = {isa = PBXBuildFile; fileRef = E5DCF96F2C590272007928CC /* Localizable.strings */; };
@@ -654,6 +656,8 @@
E5C0BBE72F82B45500EA7527 /* SimpleXAssets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = SimpleXAssets.xcassets; sourceTree = "<group>"; };
E5C0BBFD2F82BBC000EA7527 /* Debug.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Debug.xcconfig; sourceTree = "<group>"; };
E5C0BBFE2F82BBC900EA7527 /* Release.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Release.xcconfig; sourceTree = "<group>"; };
E5D5A0022F9B0000AAAA0001 /* DirectorySearch.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectorySearch.swift; sourceTree = "<group>"; };
E5D5A0042F9B0000AAAA0001 /* DirectorySearchView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectorySearchView.swift; sourceTree = "<group>"; };
E5DBF1922F88169800E1D7FD /* ConnectBannerCard.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectBannerCard.swift; sourceTree = "<group>"; };
E5DCF9702C590272007928CC /* en */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = en; path = en.lproj/Localizable.strings; sourceTree = "<group>"; };
E5DCF9722C590274007928CC /* bg */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = bg; path = bg.lproj/Localizable.strings; sourceTree = "<group>"; };
@@ -867,6 +871,7 @@
children = (
E5DDBE6D2DC4106200A0EFF0 /* AppAPITypes.swift */,
5C764E88279CBCB3000C6508 /* ChatModel.swift */,
E5D5A0022F9B0000AAAA0001 /* DirectorySearch.swift */,
5C2E260627A2941F00F70299 /* SimpleXAPI.swift */,
5C35CFC727B2782E00FB6C6D /* BGManager.swift */,
5C35CFCA27B2E91D00FB6C6D /* NtfManager.swift */,
@@ -1051,6 +1056,7 @@
isa = PBXGroup;
children = (
5C2E260A27A30CFA00F70299 /* ChatListView.swift */,
E5D5A0042F9B0000AAAA0001 /* DirectorySearchView.swift */,
5C5346A727B59A6A004DF848 /* ChatHelp.swift */,
5CB9250C27A9432000ACCCDD /* ChatListNavLink.swift */,
5C063D2627A4564100AEC577 /* ChatPreviewView.swift */,
@@ -1565,6 +1571,8 @@
64A779FE2DC3AFF200FDEF2F /* MemberSupportChatToolbar.swift in Sources */,
5C3F1D58284363C400EC8A82 /* PrivacySettings.swift in Sources */,
E5E418012F83D2CA00252B9E /* OnboardingCards.swift in Sources */,
E5D5A0012F9B0000AAAA0001 /* DirectorySearch.swift in Sources */,
E5D5A0032F9B0000AAAA0001 /* DirectorySearchView.swift in Sources */,
5C55A923283CEDE600C4E99E /* SoundPlayer.swift in Sources */,
64A779F82DBFDBF200FDEF2F /* MemberSupportView.swift in Sources */,
5C93292F29239A170090FFF9 /* ProtocolServersView.swift in Sources */,
@@ -1623,11 +1623,12 @@ object ChatController {
}
// Blocks until the directory replies or the timeout elapses, so callers must use
// withLongRunningApi, not the single-threaded withBGApi.
// withLongRunningApi, not the single-threaded withBGApi. A timeout becomes a retry row,
// not the retry alert sendCmdWithRetry would show.
suspend fun apiSearchDirectory(rh: Long?, text: String, cursor: JsonObject?): DirectorySearchResults? {
val userId = kotlin.runCatching { currentUserId("apiSearchDirectory") }.getOrElse { return null }
val req = directorySearchRequest(text, cursor)
val r = sendCmdWithRetry(rh, CC.APISendServiceRequest(userId, DIRECTORY_SERVICE_LINK, DIRECTORY_SEARCH_TIMEOUT_SEC, req))
val r = sendCmd(rh, CC.APISendServiceRequest(userId, DIRECTORY_SERVICE_LINK, DIRECTORY_SEARCH_TIMEOUT_SEC, req))
if (r is API.Result && r.res is CR.CRServiceResponse) return parseDirectorySearchResponse(r.res.responseData)
Log.e(TAG, "apiSearchDirectory error: $r")
return null
@@ -18,7 +18,7 @@ import qualified Data.Text as T
import Options.Applicative
import Simplex.Chat.Bot.KnownContacts
import Simplex.Chat.Controller (updateStr, versionNumber, versionString)
import Simplex.Chat.Options (ChatCmdLog (..), ChatOpts (..), CoreChatOpts, CreateBotOpts (..), coreChatOptsP)
import Simplex.Chat.Options (ChatCmdLog (..), ChatOpts (..), CoreChatOpts (..), CreateBotOpts (..), coreChatOptsP)
data DirectoryOpts = DirectoryOpts
{ coreOptions :: CoreChatOpts,
@@ -37,6 +37,7 @@ data DirectoryOpts = DirectoryOpts
clientService :: Bool,
runCLI :: Bool,
searchResults :: Int,
maxServiceRequestsInFlight :: Int,
webFolder :: Maybe FilePath,
linkCheckInterval :: Int,
prohibitedToObserver :: Bool,
@@ -199,6 +200,7 @@ directoryOpts appDir defaultDbName = do
clientService,
runCLI,
searchResults = 10,
maxServiceRequestsInFlight = 8,
webFolder,
linkCheckInterval,
prohibitedToObserver,
@@ -222,7 +224,7 @@ getDirectoryOpts appDir defaultDbName =
mkChatOpts :: DirectoryOpts -> ChatOpts
mkChatOpts DirectoryOpts {coreOptions, serviceName, clientService} =
ChatOpts
{ coreOptions,
{ coreOptions = coreOptions {serviceRequests = True},
chatCmd = "",
chatCmdDelay = 3,
chatCmdLog = CCLNone,
@@ -6,8 +6,13 @@
module Directory.Rpc where
import qualified Data.Aeson as J
import qualified Data.Aeson.KeyMap as JM
import qualified Data.Aeson.TH as JQ
import Data.Maybe (fromMaybe)
import qualified Data.ByteString.Lazy.Char8 as LB
import Data.Either (isRight)
import Data.Foldable (foldl')
import Data.Maybe (fromMaybe, isJust)
import Data.Text (Text)
import qualified Data.Text as T
import Data.Time.Clock (UTCTime)
@@ -15,6 +20,7 @@ import Directory.Listing
import Directory.Search
import Directory.Store
import Simplex.Chat.Library.Commands (maxProfileImageSize)
import Simplex.Chat.Protocol (compressServiceBody)
import Simplex.Chat.Types
import Simplex.Messaging.SimplexName (SimplexNameInfo (..), SimplexNameType (..), shortNameInfoStr)
import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, taggedObjectJSON)
@@ -50,18 +56,48 @@ $(JQ.deriveJSON (taggedObjectJSON $ dropPrefix "DR") ''DirectoryRequest)
$(JQ.deriveJSON (taggedObjectJSON $ dropPrefix "DR") ''DirectoryResponse)
-- Entries without any link are dropped: there would be nothing to connect to.
responseObject :: DirectoryResponse -> J.Object
responseObject resp = case J.toJSON resp of
J.Object o -> o
_ -> JM.fromList [("type", J.String "error"), ("errorMessage", J.String "internal error")]
-- The page is shrunk from the end until its compressed encoding fits the envelope, and the cursor is the
-- last row consumed. A lone entry that does not fit is retried without its image, then skipped, or paging would stall.
searchResultsPage :: (row -> SearchCursor) -> Bool -> [(row, Maybe DirectorySearchEntry)] -> DirectoryResponse
searchResultsPage rowCursor storeHasMore rows = fit entryRows
where
entryRows = [(row, e) | (row, Just e) <- rows]
fit sent = case page sent of
Just resp -> resp
Nothing -> case sent of
[(row, e)] -> fromMaybe (skipped row) $ page [(row, dropImage e)]
_ -> fit (init sent)
page sent
| fits resp = Just resp
| otherwise = Nothing
where
fittedAll = length sent == length entryRows
cursorRow = if fittedAll then fst <$> lastMaybe rows else fst <$> lastMaybe sent
more = not fittedAll || storeHasMore
resp = DRSearchResults {entries = map snd sent, searchCursor = if more then rowCursor <$> cursorRow else Nothing}
skipped row = DRSearchResults {entries = [], searchCursor = Just $ rowCursor row}
fits = isRight . compressServiceBody . LB.toStrict . J.encode . responseObject
dropImage :: DirectorySearchEntry -> DirectorySearchEntry
dropImage e = e {image = Nothing}
lastMaybe = foldl' (\_ x -> Just x) Nothing
searchEntry :: UTCTime -> GroupInfo -> Maybe GroupLink -> Maybe DirectorySearchEntry
searchEntry now g@GroupInfo {groupProfile, chatTs, createdAt = groupCreatedAt, groupSummary} gLink_ =
entry <$> groupPublicLink g gLink_
where
GroupProfile {displayName, shortDescr, image, memberAdmission, publicGroup} = groupProfile
entry groupLink =
entry link@PublicLink {connShortLink} =
DirectorySearchEntry
{ entryType = DETGroup ((\PublicGroupProfile {groupType} -> groupType) <$> publicGroup) memberAdmission groupSummary,
displayName,
simplexName = shortNameInfoStr . SimplexNameInfo NTPublicGroup <$> verifiedGroupDomain g,
groupLink,
-- the apps connect through the short link, and a full link is hundreds of bytes of the envelope
groupLink = if isJust connShortLink then link {connFullLink = Nothing} else link,
shortDescr,
-- a profile received from its owner is not size-checked, so bound what is relayed
-- rather than passing an arbitrarily large data URI on to the apps
@@ -28,12 +28,9 @@ import Control.Monad.IO.Class
import Control.Monad.Reader (runReaderT)
import qualified Data.Attoparsec.Text as A
import qualified Data.Aeson as J
import qualified Data.Aeson.KeyMap as JM
import qualified Data.Aeson.Types as JT
import Data.Bifunctor (first)
import qualified Data.ByteString.Lazy.Char8 as LB
import Data.Either (fromRight, isRight)
import Data.Foldable (foldl')
import Data.Either (fromRight)
import Data.Functor (($>))
import Data.List (intercalate)
import Data.List.NonEmpty (NonEmpty (..))
@@ -63,7 +60,7 @@ import Simplex.Chat.Library.Internal (setGroupLinkData)
import Simplex.Chat.Markdown (Format (..), FormattedText (..), SimplexLinkType (..), parseMaybeMarkdownList, viewName)
import Simplex.Chat.Messages
import Simplex.Chat.Options
import Simplex.Chat.Protocol (GroupShortLinkData (..), compressServiceBody, LinkOwnerSig (..), MsgChatLink (..), MsgContent (..), memberSupportVoiceVersion)
import Simplex.Chat.Protocol (GroupShortLinkData (..), LinkOwnerSig (..), MsgChatLink (..), MsgContent (..), memberSupportVoiceVersion)
import Simplex.Chat.Store.Direct (getContact)
import Simplex.Chat.Store.Groups (getGroupLink, getGroupMember, getGroupMemberByMemberId, setGroupCustomData) -- TODO remove setGroupCustomData
import Simplex.Chat.Store.Profiles (GroupLinkInfo (..), getGroupLinkInfo)
@@ -140,11 +137,10 @@ newServiceState opts = do
serviceRequestsInFlight <- newTVarIO 0
pure ServiceState {searchRequests, blockedWordsCfg, pendingCaptchas, serviceCC, eventQ, updateListingsJob, serviceRequestsInFlight}
-- Requests are answered off the event loop, which is shared with registrations and captchas,
-- so the bound has to be here rather than in the loop. Over the bound requests are refused
-- immediately: a caller gets an error rather than waiting out its timeout.
maxServiceRequestsInFlight :: Int
maxServiceRequestsInFlight = 8
-- bounds the LIKE scan an unauthenticated request can demand;
-- no substring of a name or description worth matching is longer
maxSearchTextLength :: Int
maxSearchTextLength = 100
welcomeGetOpts :: IO DirectoryOpts
welcomeGetOpts = do
@@ -232,7 +228,7 @@ directoryPostStartHook opts@DirectoryOpts {noAddress, testing} env cc =
readTVarIO (currentUser cc) >>= \case
Nothing -> putStrLn "No current user" >> exitFailure
Just User {userId, profile = p@LocalProfile {preferences}} -> do
unless noAddress $ initializeBotAddress' (not testing) Nothing True cc
unless noAddress $ initializeBotAddress' (not testing) (Just True) True cc
void $ atomically $ tryPutTMVar (serviceCC env) cc
listingsUpdated env
let cmds = fromMaybe [] $ preferences >>= commands_
@@ -330,7 +326,7 @@ readBlockedWordsConfig DirectoryOpts {blockedFragmentsFile, blockedWordsFile, na
pure BlockedWordsConfig {blockedFragments, blockedWords, extensionRules, spelling}
directoryServiceEvent :: DirectoryOpts -> ServiceState -> User -> ChatController -> DirectoryEvent -> IO ()
directoryServiceEvent opts@DirectoryOpts {adminUsers, superUsers, serviceName, ownersGroup, searchResults, prohibitedToObserver, alwaysCaptcha, alwaysObserver} env@ServiceState {searchRequests, serviceRequestsInFlight} user@User {userId} cc = \case
directoryServiceEvent opts@DirectoryOpts {adminUsers, superUsers, serviceName, ownersGroup, searchResults, maxServiceRequestsInFlight, prohibitedToObserver, alwaysCaptcha, alwaysObserver} env@ServiceState {searchRequests, serviceRequestsInFlight} user@User {userId} cc = \case
DEContactConnected ct -> deContactConnected ct
DEGroupInvitation {contact = ct, groupInfo = g, fromMemberRole, memberRole} -> deGroupInvitation ct g fromMemberRole memberRole
DEServiceJoinedGroup ctId g owner -> deServiceJoinedGroup ctId g owner
@@ -361,6 +357,7 @@ directoryServiceEvent opts@DirectoryOpts {adminUsers, superUsers, serviceName, o
where
deServiceRequest :: AgentInvId -> J.Object -> IO ()
deServiceRequest reqId req = do
-- the loop is shared with registrations and captchas, so the bound is on the forked handlers
accepted <- atomically $ stateTVar serviceRequestsInFlight $ \n ->
if n < maxServiceRequestsInFlight then (True, n + 1) else (False, n)
if accepted
@@ -376,53 +373,21 @@ directoryServiceEvent opts@DirectoryOpts {adminUsers, superUsers, serviceName, o
Left e -> logError $ "service reject error: " <> tshow e
requestResponse = case JT.parseMaybe JT.parseJSON (J.Object req) of
Nothing -> pure $ DRError "unsupported request"
Just DRSearch {searchText, searchCursor} -> directorySearch searchText searchCursor
Just DRSearch {searchText, searchCursor}
| T.length searchText > maxSearchTextLength -> pure $ DRError "search text is too long"
| otherwise -> directorySearch searchText searchCursor
respond resp =
sendChatCmd cc (APISendServiceResponse userId reqId $ responseObject resp) >>= \case
Right _ -> pure ()
Left e -> logError $ "service response error: " <> tshow e
responseObject resp = case J.toJSON resp of
J.Object o -> o
-- unreachable: DirectoryResponse encodes as a tagged object; kept for totality
_ -> JM.fromList [("type", J.String "error"), ("errorMessage", J.String "internal error")]
directorySearch :: Text -> Maybe SearchCursor -> IO DirectoryResponse
directorySearch searchText cursor_ =
searchListedGroups cc user (STSearch searchText) cursor_ searchResults >>= \case
Left e -> logError ("searchListedGroups error: " <> T.pack e) $> DRError "search failed"
Right (gs, n) -> do
now <- getCurrentTime
let rows = map (\row@(g, _, gLink_) -> (row, searchEntry now g gLink_)) gs
-- rows with no link cannot be connected to, so they are not sent
entryRows = [(row, e) | (row, Just e) <- rows]
(sent, lastFitted) = fitPage entryRows
fittedAll = length sent == length entryRows
-- when the whole page fitted, the cursor covers every row read, including
-- rows dropped for having no link; otherwise it stops where sending stopped
cursorRow = if fittedAll then fst <$> lastMaybe rows else lastFitted
more = not fittedAll || n > length gs
pure
DRSearchResults
{ entries = map snd sent,
searchCursor = if more then rowCursor <$> cursorRow else Nothing
}
pure $ searchResultsPage rowCursor (n > length gs) [(row, searchEntry now g gLink_) | row@(g, _, gLink_) <- gs]
where
-- Send as many entries as the padded envelope allows, and report the last row consumed
-- so the cursor can move past rows that were read but not sent. A lone entry that does
-- not fit is retried without its image; if it still does not fit it is skipped rather
-- than sent, or every retry would land on it again and paging would stall there.
fitPage [] = ([], Nothing)
fitPage rows
| fits rows = (rows, fst <$> lastMaybe rows)
| [(gr, _)] <- rows = (if fits noImage then noImage else [], Just gr)
| otherwise = fitPage $ init rows
where
noImage = [(gr, dropImage e) | (gr, e) <- rows]
dropImage :: DirectorySearchEntry -> DirectorySearchEntry
dropImage e = e {image = Nothing}
fits rows = isRight $ compressServiceBody $ LB.toStrict $ J.encode $ page rows
page rows =
DRSearchResults {entries = map snd rows, searchCursor = rowCursor . fst <$> lastMaybe rows}
lastMaybe = foldl' (\_ x -> Just x) Nothing
rowCursor (GroupInfo {groupId, groupSummary = GroupSummary {currentMembers}}, GroupReg {createdAt}, _) =
SearchCursor {lastMembers = currentMembers, lastCreatedAt = createdAt, lastGroupId = groupId}
groupLinkText (CCLink cReq sLnk_) = maybe (strEncodeTxt $ simplexChatContact cReq) strEncodeTxt sLnk_
@@ -376,7 +376,7 @@ searchListedGroups cc user@User {userId, userContactId} searchType cursor_ pageS
n <- count $ DB.query db (countQuery' <> membersCond <> searchCond) ((GRSActive, lastMembers, lastMembers, lastGroupId) :. (s, s, s, s, sDomain))
pure (gs, n)
where
s = T.toLower search
s = likeEscape $ T.toLower search
-- a bare "#"/"@" maps to "#", matching no stored domain (domains are stored unprefixed)
sDomain = case T.uncons s of
Just (c, rest) | c == '#' || c == '@' -> if T.null rest then "#" else rest
@@ -395,14 +395,18 @@ searchListedGroups cc user@User {userId, userContactId} searchType cursor_ pageS
recentCond = " AND (r.created_at < ? OR (r.created_at = ? AND r.group_id > ?)) "
searchCond =
[sql|
AND (LOWER(gp.display_name) LIKE '%' || ? || '%'
OR LOWER(gp.full_name) LIKE '%' || ? || '%'
OR LOWER(gp.short_descr) LIKE '%' || ? || '%'
OR LOWER(gp.description) LIKE '%' || ? || '%'
OR (LOWER(gp.group_domain) LIKE '%' || ? || '%' AND g.group_domain_verified = 1)
AND (LOWER(gp.display_name) LIKE '%' || ? || '%' ESCAPE '\'
OR LOWER(gp.full_name) LIKE '%' || ? || '%' ESCAPE '\'
OR LOWER(gp.short_descr) LIKE '%' || ? || '%' ESCAPE '\'
OR LOWER(gp.description) LIKE '%' || ? || '%' ESCAPE '\'
OR (LOWER(gp.group_domain) LIKE '%' || ? || '%' ESCAPE '\' AND g.group_domain_verified = 1)
)
|]
-- the search text is a substring, not a pattern: its '%' and '_' must match literally
likeEscape :: Text -> Text
likeEscape = T.concatMap $ \c -> if c `elem` ['%', '_', '\\'] then T.pack ['\\', c] else T.singleton c
getAllGroupRegs_ :: DB.Connection -> StoreCxt -> User -> IO [(GroupInfo, GroupReg)]
getAllGroupRegs_ db cxt user@User {userId, userContactId} = do
currentTs <- getCurrentTime
+4 -7
View File
@@ -1500,9 +1500,7 @@ processChatCommand cxt nm = \case
pure $ CRServiceReplyAccepted user (AgentConnId connId)
APIRejectServiceRequest userId requestId reason -> withUserId userId $ \user -> do
let AgentInvId invId = requestId
-- A reason is required for the requester to fail fast; without it the request is dropped
-- silently and the caller waits out its timeout. Async, so a service shedding load does
-- not block on a network round trip per rejected request.
-- without a reason the request is dropped silently and the requester waits out its timeout
withAgent $ \a -> rejectServiceRequestAsync a "" (aUserId user) invId (encodeUtf8 <$> reason)
ok user
APISendCallInvitation contactId callType -> withUser $ \user -> do
@@ -5245,7 +5243,7 @@ redeemBadgeCode nm user@User {userId} codeText = do
maybe (withStore' $ \db -> createBadgeCodeRedemption db g user codeSent now) pure redemption_
let req = BadgeServiceRequest {version = currentBadgeServiceVersion, purchaseKey = Just purchaseKey, request = BSCRedeemBadgeCode {masterKey, code = codeSent}}
respBytes <- sendServiceRequestBytes nm user sendTarget Nothing (Just purchasePrivKey) req
respData <- either (const $ throwRedeemError $ BREInvalidResponse "not JSON") pure $ J.eitherDecodeStrict' respBytes
respData <- either (const $ throwRedeemError $ BREInvalidResponse "not JSON") pure $ parseServiceBody respBytes
case J.fromJSON (J.Object respData) of
J.Error _ -> throwRedeemError $ BREInvalidResponse "not a badge service response"
J.Success BSPError {code = errCode} -> do
@@ -5640,14 +5638,13 @@ applyBadgeStatement db g purchaseId badgeType BadgeStatement {entries} cred_ now
sendServiceRequestTo :: J.ToJSON a => NetworkRequestMode -> User -> ConnectTarget 'CMContact -> Maybe NominalDiffTime -> Maybe C.PrivateKeyEd25519 -> a -> CM J.Object
sendServiceRequestTo nm user sendTarget requestTimeout signKey request =
sendServiceRequestBytes nm user sendTarget requestTimeout signKey request
>>= either (const $ throwCmdError "invalid service response") pure . J.eitherDecodeStrict'
>>= either (const $ throwCmdError "invalid service response") pure . parseServiceBody
sendServiceRequestBytes :: J.ToJSON a => NetworkRequestMode -> User -> ConnectTarget 'CMContact -> Maybe NominalDiffTime -> Maybe C.PrivateKeyEd25519 -> a -> CM ByteString
sendServiceRequestBytes nm user sendTarget requestTimeout signKey request = do
cReq <- resolveServiceTarget sendTarget
reqData <- either throwCmdError pure $ compressServiceBody $ LB.toStrict $ J.encode request
respData <- withAgent $ \a -> sendServiceRequestAsync a (aUserId user) cReq requestTimeout signKey reqData
either (const $ throwCmdError "invalid service response") pure $ decompressServiceBody respData
withAgent $ \a -> sendServiceRequestAsync a (aUserId user) cReq requestTimeout signKey reqData
where
resolveServiceTarget = \case
CTFullContact cReq -> pure cReq
+1 -2
View File
@@ -108,7 +108,6 @@ import Text.Read (readMaybe)
import UnliftIO.Concurrent (ThreadId, forkIO, mkWeakThreadId)
import UnliftIO.Directory
import UnliftIO.STM
import qualified Data.Aeson as J
smallGroupsRcptsMemLimit :: Int
smallGroupsRcptsMemLimit = 20
@@ -1396,7 +1395,7 @@ processAgentMessageConn cxt user@User {userId} entity corrId agentConnId agentMe
_ -> pure ()
SREQ invId sigKey_ payload ->
chatReadVar processServiceRequests >>= \case
True -> case J.eitherDecodeStrict' =<< decompressServiceBody payload of
True -> case parseServiceBody payload of
Right request -> toView $ CEvtServiceRequest user (AgentInvId invId) sigKey_ request
Left e -> logError ("service request dropped, invalid payload: " <> tshow e) >> dropSReq
-- the requester gets no reply and waits out its timeout, so this must be visible
-1
View File
@@ -315,7 +315,6 @@ coreChatOptsP appDir defaultDbName = do
( long "ha"
<> help "Run as a highly available client (this may increase traffic in groups)"
)
-- TODO [directory] default this on for the directory binary, so a deployment cannot omit it
serviceRequests <-
switch
( long "service-requests"
+16
View File
@@ -1063,6 +1063,22 @@ decompressServiceBody body = case B.uncons body of
Right _ -> Left "unexpected compressed batch"
_ -> Right body
-- The apps decode a service payload recursively on a fixed stack, and no service nests deeper
-- than a few levels, so depth is bounded here rather than left to each client.
maxServiceBodyDepth :: Int
maxServiceBodyDepth = 32
parseServiceBody :: ByteString -> Either String J.Object
parseServiceBody body = do
o <- J.eitherDecodeStrict' =<< decompressServiceBody body
when (depth (J.Object o) > maxServiceBodyDepth) $ Left "service payload is nested too deeply"
pure o
where
depth = \case
J.Object kv -> 1 + foldr (max . depth) 0 kv
J.Array vs -> 1 + foldr (max . depth) 0 vs
_ -> 0
justTrue :: Bool -> Maybe Bool
justTrue True = Just True
justTrue False = Nothing
+116 -3
View File
@@ -11,13 +11,20 @@ import ChatTests.DBUtils
import ChatTests.Groups (memberJoinChannel, prepareChannel1Relay)
import ChatTests.Utils
import Control.Concurrent (forkIO, killThread, threadDelay)
import Control.Concurrent.STM (atomically)
import Control.Exception (finally)
import Control.Monad (forM_, when, void)
import Data.Aeson ((.:), (.:?), (.=))
import qualified Data.Aeson as J
import qualified Data.Aeson.Types as JT
import qualified Data.ByteString.Lazy.Char8 as LB
import qualified Data.Text as T
import Data.Time.Clock (getCurrentTime)
import Directory.Captcha
import Directory.Listing
import Directory.Options
import Directory.Rpc
import Directory.Search (SearchCursor (..))
import Directory.Service
import System.Directory (emptyPermissions, setOwnerExecutable, setOwnerReadable, setOwnerWritable, setPermissions)
import Simplex.Chat.Bot.KnownContacts
@@ -26,9 +33,12 @@ import qualified Simplex.Chat.Markdown as MD
import Simplex.Chat.Options (CoreChatOpts (..))
import Simplex.Chat.Options.DB
import Simplex.Chat.Protocol (memberSupportVoiceVersion)
import Simplex.Chat.Types (ChatPeerType (..), Profile (..))
import Simplex.Chat.Types (ChatPeerType (..), GroupSummary (..), ImageData (..), Profile (..))
import Simplex.Chat.Types.Shared (GroupMemberRole (..))
import qualified Simplex.Messaging.Crypto as C
import Simplex.Messaging.Encoding.String (strEncode)
import Simplex.Messaging.SimplexName (SimplexDomain (..), SimplexNameInfo (..), SimplexNameType (..), SimplexTLD (..))
import Simplex.Messaging.Util (safeDecodeUtf8)
import Simplex.Messaging.Version
import NameResolver
import System.FilePath ((</>))
@@ -47,6 +57,10 @@ directoryServiceTests = do
it "should return more groups in search, all and recent groups" testSearchGroups
it "should page from the sort key, not group ID" testSearchGroupsPaging
it "should answer search over service RPC" testDirectorySearchRpc
it "should page search over service RPC by the echoed cursor" testDirectorySearchRpcPaging
it "should match LIKE wildcards in search text literally" testDirectorySearchRpcLiteral
it "should reject service requests over the cap" testDirectorySearchRpcBusy
it "should fit the search page to the envelope" testSearchResultsPage
it "should invite to owners' group if specified" testInviteToOwnersGroup
it "should re-invite owner who left owners' group" testInviteOwnerAfterLeavingOwnersGroup
describe "de-listing the group" $ do
@@ -119,8 +133,7 @@ mkDirectoryOpts TestParams {tmpPath = ps} superUsers ownersGroup webFolder =
DirectoryOpts
{ coreOptions =
testCoreOpts
{ serviceRequests = True,
dbOptions =
{ dbOptions =
(dbOptions testCoreOpts)
#if defined(dbPostgres)
{dbSchemaPrefix = "client_" <> serviceDbPrefix}
@@ -144,6 +157,7 @@ mkDirectoryOpts TestParams {tmpPath = ps} superUsers ownersGroup webFolder =
clientService = True,
runCLI = False,
searchResults = 3,
maxServiceRequestsInFlight = 8,
webFolder,
linkCheckInterval = 0,
prohibitedToObserver = False,
@@ -673,6 +687,105 @@ testDirectorySearchRpc ps =
cath ##> ("/_service_request 1 " <> dsShortLink <> " {\"type\":\"nonsense\"}")
cath <## "service response: {\"errorMessage\":\"unsupported request\",\"type\":\"error\"}"
-- the contract the apps page by: the cursor is opaque, echoed back as received, and continues where the page stopped
testDirectorySearchRpcPaging :: HasCallStack => TestParams -> IO ()
testDirectorySearchRpcPaging ps =
withDirectoryService ps $ \superUser (dsShortLink, _) ->
withNewTestChat ps "bob" bobProfile $ \bob -> do
bob `connectVia` dsShortLink
forM_ [1 .. 4 :: Int] $ \i -> registerGroupId superUser bob ("group" <> show i) "" i i
withNewTestChat ps "cath" cathProfile $ \cath -> do
(page1, cursor) <- searchDirectory cath dsShortLink "group" Nothing
page1 `shouldBe` ["group1", "group2", "group3"]
(page2, cursor') <- searchDirectory cath dsShortLink "group" cursor
page2 `shouldBe` ["group4"]
cursor' `shouldBe` Nothing
-- the search text is a substring: '%' and '_' in it match literally, and its length is bounded
testDirectorySearchRpcLiteral :: HasCallStack => TestParams -> IO ()
testDirectorySearchRpcLiteral ps =
withDirectoryService ps $ \superUser (dsShortLink, _) ->
withNewTestChat ps "bob" bobProfile $ \bob -> do
bob `connectVia` dsShortLink
registerGroupId superUser bob "PrivacyGroup" "" 1 1
let descr = replicate 200 'e'
bob ##> ("/set welcome #PrivacyGroup " <> descr)
bob <## "welcome message changed to:"
bob <## descr
groupUpdatedHidden superUser bob "PrivacyGroup" ""
notifySuperUser_ superUser bob "PrivacyGroup" "" (Just descr) 1 1
void $ approveRegistrationId superUser bob "PrivacyGroup" 1 1
withNewTestChat ps "cath" cathProfile $ \cath -> do
(found, _) <- searchDirectory cath dsShortLink "privacy" Nothing
found `shouldBe` ["PrivacyGroup"]
-- ten wildcard pairs against two hundred e's, then a character the description lacks
(none, _) <- searchDirectory cath dsShortLink (concat (replicate 10 "%e") <> "%q") Nothing
none `shouldBe` []
-- as a wildcard, '_' would match the 'g'
(none', _) <- searchDirectory cath dsShortLink "privacy_roup" Nothing
none' `shouldBe` []
cath ##> ("/_service_request 1 " <> dsShortLink <> " {\"type\":\"search\",\"searchText\":\"" <> replicate 101 'a' <> "\"}")
cath <## "service response: {\"errorMessage\":\"search text is too long\",\"type\":\"error\"}"
-- over the cap a request is refused at once with a reason, so the requester does not wait out its timeout
testDirectorySearchRpcBusy :: HasCallStack => TestParams -> IO ()
testDirectorySearchRpcBusy ps =
withDirectoryServiceOpts ps (\o -> o {maxServiceRequestsInFlight = 0}) $ \_superUser (dsShortLink, _) ->
withNewTestChat ps "cath" cathProfile $ \cath -> do
cath ##> ("/_service_request 1 " <> dsShortLink <> " {\"type\":\"search\",\"searchText\":\"privacy\"}")
cath <## "smp agent error: AGENT {agentErr = A_SERVICE {serviceError = ASERejected {rejectReason = \"service is busy\"}}}"
searchDirectory :: HasCallStack => TestCC -> String -> String -> Maybe J.Value -> IO ([String], Maybe J.Value)
searchDirectory u dsLink text cursor_ = do
let req = J.object $ ["type" .= ("search" :: String), "searchText" .= text] <> maybe [] (\c -> ["searchCursor" .= c]) cursor_
u ##> ("/_service_request 1 " <> dsLink <> " " <> LB.unpack (J.encode req))
resp <- dropStrPrefix "service response: " <$> getTermLine u
maybe (fail $ "unexpected response: " <> resp) pure $ JT.parseMaybe searchResults =<< J.decode (LB.pack resp)
where
searchResults = J.withObject "searchResults" $ \o -> do
entries <- o .: "entries" :: JT.Parser [J.Object]
names <- mapM (.: "displayName") entries
cursor <- o .:? "searchCursor"
pure (names, cursor)
-- the page is bounded by the envelope, not by searchResults: entries are cut from the end, the cursor
-- follows the last row consumed, and a lone oversize entry loses its image or is skipped
testSearchResultsPage :: HasCallStack => TestParams -> IO ()
testSearchResultsPage _ps = do
g <- C.newRandom
now <- getCurrentTime
-- base64 of random bytes does not compress, so the envelope decides what fits
let randomText n = safeDecodeUtf8 . strEncode <$> atomically (C.randomBytes n g)
cursor gId = SearchCursor {lastMembers = 1, lastCreatedAt = now, lastGroupId = gId}
page r = case r of
DRSearchResults {entries, searchCursor} -> (map (\DirectorySearchEntry {displayName} -> displayName) entries, lastGroupId <$> searchCursor)
DRError e -> error $ T.unpack e
entry name descr img =
DirectorySearchEntry
{ entryType = DETGroup {groupType = Nothing, admission = Nothing, summary = GroupSummary {currentMembers = 1, publicMemberCount = Nothing}},
displayName = name,
simplexName = Nothing,
groupLink = PublicLink {connFullLink = Nothing, connShortLink = Nothing},
shortDescr = Just descr,
image = ImageData <$> img,
activeAt = Nothing,
createdAt = Nothing
}
-- about 4.7 KB compressed each: two fit the 10,968-byte envelope, three do not
sized name = entry name <$> randomText 4500 <*> pure Nothing
a <- sized "a"
b <- sized "b"
c <- sized "c"
page (searchResultsPage cursor False [(1, Just a), (2, Just b), (3, Just c)]) `shouldBe` (["a", "b"], Just 2)
page (searchResultsPage cursor False [(1, Just a), (2, Just b)]) `shouldBe` (["a", "b"], Nothing)
page (searchResultsPage cursor True [(1, Just a), (2, Just b)]) `shouldBe` (["a", "b"], Just 2)
-- a row without a link is consumed by the cursor, not sent
page (searchResultsPage cursor True [(1, Just a), (2, Nothing)]) `shouldBe` (["a"], Just 2)
page (searchResultsPage cursor False [(1, Just a), (2, Nothing)]) `shouldBe` (["a"], Nothing)
big <- randomText 12000
page (searchResultsPage cursor False [(1, Just $ entry "d" "" (Just big))]) `shouldBe` (["d"], Nothing)
page (searchResultsPage cursor False [(1, Just $ entry "e" big Nothing)]) `shouldBe` ([], Just 1)
testInviteToOwnersGroup :: HasCallStack => TestParams -> IO ()
testInviteToOwnersGroup ps =
withDirectoryServiceCfgOwnersGroup ps testCfg True Nothing $ \superUser (_, dsLink) ->
+4
View File
@@ -57,6 +57,10 @@ serviceBodyTests = describe "service payload compression" $ do
let bomb = compressedBatchMsgBody_ $ B.replicate (maxDecompressedMsgLength + 1) 'a'
B.length bomb `shouldSatisfy` (< maxCompressedInfoLength)
decompressServiceBody bomb `shouldBe` Left "decompressed size exceeds limit"
it "rejects a payload nested deeper than the bound" $ do
let nested n = "{\"a\":" <> B.replicate n '[' <> B.replicate n ']' <> "}"
parseServiceBody (nested 10) `shouldBe` J.eitherDecodeStrict' (nested 10)
parseServiceBody (nested 100) `shouldBe` Left "service payload is nested too deeply"
batchLimitTests :: Spec
batchLimitTests = describe "Chat message batch limits" $ do