core: send badge ZK-proofs to XFTP servers, add file expiry time (#7430)

* core: send badge ZK-proofs to XFTP servers

* update simplexmq, use keys and header from simplexmq

* core: add file expiry time

* refactor

* remove posix module

* rename migrations

* bot api

* update UI

* import

* show item from loaded information

* ios: update core library

* text

* send entitlements in handshake

* simplexmq

* query plans

* update simplexmq

---------

Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>
This commit is contained in:
Evgeny
2026-09-01 23:16:15 +01:00
committed by GitHub
co-authored by Evgeny @ SimpleX Chat
parent f71b132536
commit f99c474ca6
43 changed files with 328 additions and 133 deletions
+5 -3
View File
@@ -56,7 +56,7 @@ import Data.Type.Equality
import qualified Data.UUID as UUID
import qualified Data.UUID.V4 as V4
import Simplex.Chat.Library.Subscriber
import Simplex.Chat.Badges (BadgeCredential (..), LocalBadge (..), maxXFTPFileSize, mkBadgeStatus, verifyCredential)
import Simplex.Chat.Badges (BadgeCredential (..), LocalBadge (..), badgeServerCredential, maxXFTPFileSize, mkBadgeStatus, verifyCredential)
import Simplex.Chat.Names (SimplexDomainProof (..), SimplexDomainClaim (..), claimDomain, mkDomainClaim)
import Simplex.Chat.Call
import Simplex.Chat.Controller
@@ -5000,7 +5000,7 @@ processChatCommand cxt nm = \case
chunkSize <- asks $ fileChunkSize . config
withFastStore' $ \db -> do
fileId <- createLocalFile CIFSSndStored db user nf createdAt cf fileSize chunkSize
pure CIFile {fileId, fileName = takeFileName filePath, fileSize, fileSource = Just cf, fileStatus = CIFSSndStored, fileProtocol = FPLocal}
pure CIFile {fileId, fileName = takeFileName filePath, fileSize, fileSource = Just cf, fileStatus = CIFSSndStored, fileProtocol = FPLocal, fileExpires = Nothing}
prepareLocalItemsData ::
NonEmpty ComposedMessageReq ->
NonEmpty (Maybe (CIFile 'MDSnd)) ->
@@ -5142,8 +5142,10 @@ addUserBadge user cred@(BadgeCredential keyIdx _ _ info) = do
verified <- liftIO $ verifyCredential key cred
unless verified $ throwCmdError "badge credential does not verify against configured key"
now <- liftIO getCurrentTime
user' <- withFastStore' $ \db -> setUserBadge db user (Just (OwnBadge cred (mkBadgeStatus now (Just True) info)))
let badge = Just (OwnBadge cred (mkBadgeStatus now (Just True) info))
user' <- withFastStore' $ \db -> setUserBadge db user badge
asks currentUser >>= atomically . (`writeTVar` Just user')
lift $ withAgent' $ \a -> setUserEntitlement a (aUserId user') (badgeServerCredential badge)
cxt <- asks $ mkStoreCxt . config
contacts <- withFastStore' $ \db -> getUserContacts db cxt user'
withChatLock "addUserBadge" $ forM_ contacts $ \ct ->
+12 -12
View File
@@ -441,12 +441,12 @@ xftpSndFileTransfer_ user file@(CryptoFile filePath cfArgs) fileSize n contactOr
fInv = xftpFileInvitation fileName fileSize dummyFileDescr
fsFilePath <- lift $ toFSFilePath filePath
let srcFile = CryptoFile fsFilePath cfArgs
aFileId <- withAgent $ \a -> xftpSendFile a (aUserId user) srcFile (roundedFDCount n)
aFileId <- withAgent $ \a -> xftpSendFile a (aUserId user) srcFile (roundedFDCount n) Nothing
-- TODO CRSndFileStart event for XFTP
chSize <- asks $ fileChunkSize . config
ft@FileTransferMeta {fileId} <- withStore' $ \db -> createSndFileTransferXFTP db user contactOrGroup_ file fInv (AgentSndFileId aFileId) Nothing chSize
let fileSource = Just $ CryptoFile filePath cfArgs
ciFile = CIFile {fileId, fileName, fileSize, fileSource, fileStatus = CIFSSndStored, fileProtocol = FPXFTP}
ciFile = CIFile {fileId, fileName, fileSize, fileSource, fileStatus = CIFSSndStored, fileProtocol = FPXFTP, fileExpires = Nothing}
pure (fInv, ciFile, ft)
cryptoFileDigest :: CryptoFile -> CM FD.FileDigest
@@ -1417,7 +1417,7 @@ sendHistory user gInfo@GroupInfo {membership} m@GroupMember {activeConn = Just c
resolveAuthor (Just gmId) = do
cxt <- chatStoreCxt
eitherToMaybe <$> withStore' (\db -> runExceptT $ getGroupMemberById db cxt user gmId)
getRcvFileInvDescr :: CIFile 'MDRcv -> CM (Maybe (FileInvitation, RcvFileDescrText))
getRcvFileInvDescr :: CIFile 'MDRcv -> CM (Maybe (FileInvitation, RcvFileDescrText, Maybe UTCTime))
getRcvFileInvDescr ciFile@CIFile {fileId, fileProtocol, fileStatus} = do
expired <- fileExpired
if fileProtocol /= FPXFTP || fileStatus == CIFSRcvCancelled || expired
@@ -1425,7 +1425,7 @@ sendHistory user gInfo@GroupInfo {membership} m@GroupMember {activeConn = Just c
else do
rfd <- withStore $ \db -> getRcvFileDescrByRcvFileId db fileId
pure $ invCompleteDescr ciFile rfd
getSndFileInvDescr :: CIFile 'MDSnd -> CM (Maybe (FileInvitation, RcvFileDescrText))
getSndFileInvDescr :: CIFile 'MDSnd -> CM (Maybe (FileInvitation, RcvFileDescrText, Maybe UTCTime))
getSndFileInvDescr ciFile@CIFile {fileId, fileProtocol, fileStatus} = do
expired <- fileExpired
if fileProtocol /= FPXFTP || fileStatus == CIFSSndCancelled || expired
@@ -1440,21 +1440,21 @@ sendHistory user gInfo@GroupInfo {membership} m@GroupMember {activeConn = Just c
ttl <- asks $ rcvFilesTTL . agentConfig . config
cutoffTs <- addUTCTime (-ttl) <$> liftIO getCurrentTime
pure $ chatItemTs cci < cutoffTs
invCompleteDescr :: CIFile d -> RcvFileDescr -> Maybe (FileInvitation, RcvFileDescrText)
invCompleteDescr CIFile {fileName, fileSize} RcvFileDescr {fileDescrText, fileDescrComplete}
invCompleteDescr :: CIFile d -> RcvFileDescr -> Maybe (FileInvitation, RcvFileDescrText, Maybe UTCTime)
invCompleteDescr CIFile {fileName, fileSize, fileExpires} RcvFileDescr {fileDescrText, fileDescrComplete}
| fileDescrComplete =
let fInvDescr = FileDescr {fileDescrText = "", fileDescrPartNo = 0, fileDescrComplete = False}
fInv = xftpFileInvitation fileName fileSize fInvDescr
in Just (fInv, fileDescrText)
in Just (fInv, fileDescrText, fileExpires)
| otherwise = Nothing
processContentItem :: Maybe GroupMember -> ChatItem 'CTGroup d -> MsgContent -> Maybe (FileInvitation, RcvFileDescrText) -> CM [(GrpMsgForward, VerifiedMsg 'Json)]
processContentItem :: Maybe GroupMember -> ChatItem 'CTGroup d -> MsgContent -> Maybe (FileInvitation, RcvFileDescrText, Maybe UTCTime) -> CM [(GrpMsgForward, VerifiedMsg 'Json)]
processContentItem member_ ChatItem {formattedText, meta, quotedItem, mentions} mc fInvDescr_ =
if isNothing fInvDescr_ && not (msgContentHasText mc)
then pure []
else do
let CIMeta {itemTs, itemSharedMsgId, itemTimed, showGroupAsSender} = meta
quotedItemId_ = quoteItemId =<< quotedItem
fInv_ = fst <$> fInvDescr_
fInv_ = (\(fInv, _, _) -> fInv) <$> fInvDescr_
(mc', _, mentions') = updatedMentionNames mc formattedText mentions
mentions'' = M.map (\CIMention {memberId} -> MsgMention {memberId}) mentions'
-- for channel messages default chat version range to membership range
@@ -1472,11 +1472,11 @@ sendHistory user gInfo@GroupInfo {membership} m@GroupMember {activeConn = Just c
-- TODO [knocking] send history to other scopes too?
(chatMsgEvent, _) <- withStore $ \db -> prepareGroupMsg db user gInfo Nothing showGroupAsSender mc' mentions'' quotedItemId_ Nothing fInv_ itemTimed False
pure $ VMUnsigned ChatMessage {chatVRange = senderVRange, msgId = itemSharedMsgId, chatMsgEvent}
fileDescrEvents <- case (snd <$> fInvDescr_, itemSharedMsgId) of
(Just fileDescrText, Just msgId) -> do
fileDescrEvents <- case (fInvDescr_, itemSharedMsgId) of
(Just (_, fileDescrText, fileExpires), Just msgId) -> do
partSize <- asks $ xftpDescrPartSize . config
let parts = splitFileDescr partSize fileDescrText
pure . L.toList $ L.map (XMsgFileDescr msgId) parts
pure . L.toList $ L.map (\fd -> XMsgFileDescr msgId fd fileExpires) parts
_ -> pure []
let fileDescrVMs = map (VMUnsigned . ChatMessage senderVRange Nothing) fileDescrEvents
pure $ map ((,) fwd) (contentVM : fileDescrVMs)
+25 -22
View File
@@ -41,6 +41,7 @@ import Data.Text (Text)
import qualified Data.Text as T
import Data.Text.Encoding (decodeLatin1)
import Data.Time.Clock (NominalDiffTime, UTCTime, addUTCTime, diffUTCTime, getCurrentTime)
import Data.Time.Clock.System (SystemTime (..), systemToUTCTime)
import Data.Time.Format (defaultTimeLocale, formatTime)
import qualified Data.UUID as UUID
import qualified Data.UUID.V4 as V4
@@ -75,7 +76,7 @@ import Simplex.Chat.Types.Preferences
import Simplex.Chat.Types.Shared
import Simplex.FileTransfer.Description (ValidFileDescription)
import qualified Simplex.FileTransfer.Description as FD
import Simplex.FileTransfer.Protocol (FilePartyI)
import Simplex.FileTransfer.Protocol (FilePartyI, GrantedStorageTime (..))
import qualified Simplex.FileTransfer.Transport as XFTP
import Simplex.FileTransfer.Types (FileErrorType (..), RcvFileId, SndFileId)
import Simplex.Messaging.Agent
@@ -205,8 +206,9 @@ processAgentMsgSndFile _corrId aFileId msg = do
liftIO $ updateCIFileStatus db user fileId status
lookupChatItemByFileId db cxt user fileId
toView $ CEvtSndFileProgressXFTP user ci ft sndProgress sndTotal
SFDONE sndDescr rfds -> do
withStore' $ \db -> setSndFTPrivateSndDescr db user fileId (fileDescrText sndDescr)
SFDONE sndDescr rfds granted -> do
let fileExpires = (\GSTExpires {epochSeconds} -> systemToUTCTime $ MkSystemTime epochSeconds 0) <$> granted
withStore' $ \db -> setSndFTPrivateSndDescr db user fileId (fileDescrText sndDescr) fileExpires
ci <- withStore $ \db -> lookupChatItemByFileId db cxt user fileId
case ci of
Nothing -> do
@@ -232,7 +234,7 @@ processAgentMsgSndFile _corrId aFileId msg = do
(rfd : extraRFDs, sft : _, SMDSnd, DirectChat ct) -> do
withStore' $ \db -> createExtraSndFTDescrs db user fileId (map fileDescrText extraRFDs)
conn@Connection {connId} <- liftEither $ contactSendConn_ ct
sendFileDescriptions (ConnectionId connId) ((conn, sft, fileDescrText rfd) :| []) sharedMsgId >>= \case
sendFileDescriptions (ConnectionId connId) ((conn, sft, fileDescrText rfd) :| []) sharedMsgId fileExpires >>= \case
Just rs -> case L.last rs of
Right ([msgDeliveryId], _) ->
withStore' $ \db -> updateSndFTDeliveryXFTP db sft msgDeliveryId
@@ -247,7 +249,7 @@ processAgentMsgSndFile _corrId aFileId msg = do
extraRFDs = drop (length rfdsMemberFTs) rfds
withStore' $ \db -> createExtraSndFTDescrs db user fileId (map fileDescrText extraRFDs)
forM_ (L.nonEmpty rfdsMemberFTs) $ \rfdsMemberFTs' ->
sendFileDescriptions (GroupId groupId) rfdsMemberFTs' sharedMsgId
sendFileDescriptions (GroupId groupId) rfdsMemberFTs' sharedMsgId fileExpires
ci' <- withStore $ \db -> do
liftIO $ updateCIFileStatus db user fileId CIFSSndComplete
getChatItemByFileId db cxt user fileId
@@ -276,8 +278,8 @@ processAgentMsgSndFile _corrId aFileId msg = do
where
fileDescrText :: FilePartyI p => ValidFileDescription p -> T.Text
fileDescrText = safeDecodeUtf8 . strEncode
sendFileDescriptions :: ConnOrGroupId -> NonEmpty (Connection, SndFileTransfer, RcvFileDescrText) -> SharedMsgId -> CM (Maybe (NonEmpty (Either ChatError ([Int64], PQEncryption))))
sendFileDescriptions connOrGroupId connsTransfersDescrs sharedMsgId = do
sendFileDescriptions :: ConnOrGroupId -> NonEmpty (Connection, SndFileTransfer, RcvFileDescrText) -> SharedMsgId -> Maybe UTCTime -> CM (Maybe (NonEmpty (Either ChatError ([Int64], PQEncryption))))
sendFileDescriptions connOrGroupId connsTransfersDescrs sharedMsgId fileExpires = do
lift . void . withStoreBatch' $ \db -> L.map (\(_, sft, rfdText) -> updateSndFTDescrXFTP db user sft rfdText) connsTransfersDescrs
partSize <- asks $ xftpDescrPartSize . config
let connsIdsEvts = connDescrEvents partSize
@@ -293,7 +295,7 @@ processAgentMsgSndFile _corrId aFileId msg = do
where
splitText :: (Connection, SndFileTransfer, RcvFileDescrText) -> [(Connection, (ConnOrGroupId, Maybe MsgSigning, ChatMsgEvent 'Json))]
splitText (conn, _, rfdText) =
map (\fileDescr -> (conn, (connOrGroupId, Nothing, XMsgFileDescr {msgId = sharedMsgId, fileDescr}))) (L.toList $ splitFileDescr partSize rfdText)
map (\fileDescr -> (conn, (connOrGroupId, Nothing, XMsgFileDescr {msgId = sharedMsgId, fileDescr, fileExpires}))) (L.toList $ splitFileDescr partSize rfdText)
toMsgReq :: (Connection, (ConnOrGroupId, Maybe MsgSigning, ChatMsgEvent 'Json)) -> SndMessage -> ChatMsgReq
toMsgReq (conn, _) SndMessage {msgId, msgBody} =
(conn, MsgFlags {notification = hasNotification XMsgFileDescr_}, (vrValue msgBody, [msgId]))
@@ -548,7 +550,7 @@ processAgentMessageConn cxt user@User {userId} corrId agentConnId agentMessage =
let ct'' = ct' {activeConn = Just conn''} :: Contact
case event of
XMsgNew mc -> newContentMessage ct'' mc msg msgMeta
XMsgFileDescr sharedMsgId fileDescr -> messageFileDescription ct'' sharedMsgId fileDescr
XMsgFileDescr sharedMsgId fileDescr fileExpires -> messageFileDescription ct'' sharedMsgId fileDescr fileExpires
XMsgUpdate sharedMsgId mContent _ ttl live _msgScope _ -> messageUpdate ct'' sharedMsgId mContent msg msgMeta ttl live
XMsgDel sharedMsgId _ _ _ -> messageDelete ct'' sharedMsgId msg msgMeta
XMsgReact sharedMsgId _ _ reaction add -> directMsgReaction ct'' sharedMsgId reaction add msg msgMeta
@@ -1033,7 +1035,7 @@ processAgentMessageConn cxt user@User {userId} corrId agentConnId agentMessage =
where
MsgContainer {scope, asGroup} = mc
-- file description is always allowed, to allow sending files to support scope
XMsgFileDescr sharedMsgId fileDescr -> groupMessageFileDescription gInfo' (Just m'') sharedMsgId fileDescr
XMsgFileDescr sharedMsgId fileDescr fileExpires -> groupMessageFileDescription gInfo' (Just m'') sharedMsgId fileDescr fileExpires
XMsgUpdate sharedMsgId mContent mentions ttl live msgScope asGroup_ ->
checkSendAsGroup asGroup_ $
memberCanSend (Just m'') msgScope $
@@ -1905,16 +1907,16 @@ processAgentMessageConn cxt user@User {userId} corrId agentConnId agentMessage =
ChatConfig {autoAcceptFileSize = sz} <- asks config
when (sz > fileSize) $ receiveFileEvt' user ft False Nothing Nothing >>= toView
messageFileDescription :: Contact -> SharedMsgId -> FileDescr -> CM ()
messageFileDescription Contact {contactId} sharedMsgId fileDescr = do
messageFileDescription :: Contact -> SharedMsgId -> FileDescr -> Maybe UTCTime -> CM ()
messageFileDescription Contact {contactId} sharedMsgId fileDescr fileExpires = do
(fileId, aci) <- withStore $ \db -> do
fileId <- getFileIdBySharedMsgId db userId contactId sharedMsgId
aci <- getChatItemByFileId db cxt user fileId
pure (fileId, aci)
processFDMessage fileId aci fileDescr
processFDMessage fileId aci fileDescr fileExpires
groupMessageFileDescription :: GroupInfo -> Maybe GroupMember -> SharedMsgId -> FileDescr -> CM (Maybe DeliveryTaskContext)
groupMessageFileDescription g@GroupInfo {groupId} m_ sharedMsgId fileDescr = do
groupMessageFileDescription :: GroupInfo -> Maybe GroupMember -> SharedMsgId -> FileDescr -> Maybe UTCTime -> CM (Maybe DeliveryTaskContext)
groupMessageFileDescription g@GroupInfo {groupId} m_ sharedMsgId fileDescr fileExpires = do
(fileId, aci) <- withStore $ \db -> do
fileId <- getGroupFileIdBySharedMsgId db userId groupId sharedMsgId
aci <- getChatItemByFileId db cxt user fileId
@@ -1925,17 +1927,18 @@ processAgentMessageConn cxt user@User {userId} corrId agentConnId agentMessage =
-- in processFDMessage some paths are programmed as errors,
-- for example failure on not approved relays (CEFileNotApproved).
-- we catch error, so that even if processFDMessage fails, message can still be forwarded.
processFDMessage fileId aci fileDescr `catchAllErrors` \_ -> pure ()
processFDMessage fileId aci fileDescr fileExpires `catchAllErrors` \_ -> pure ()
pure $ Just $ infoToDeliveryContext g scopeInfo (isChannelDir chatDir)
| otherwise -> messageError "x.msg.file.descr: file/sender mismatch" $> Nothing
_ -> messageError "x.msg.file.descr: invalid file description part" $> Nothing
processFDMessage :: FileTransferId -> AChatItem -> FileDescr -> CM ()
processFDMessage fileId aci fileDescr = do
processFDMessage :: FileTransferId -> AChatItem -> FileDescr -> Maybe UTCTime -> CM ()
processFDMessage fileId aci fileDescr fileExpires = do
ft <- withStore $ \db -> getRcvFileTransfer db user fileId
unless (rcvFileCompleteOrCancelled ft) $ do
(rfd@RcvFileDescr {fileDescrComplete}, ft'@RcvFileTransfer {fileStatus, xftpRcvFile, cryptoArgs, fileInvitation = FileInvitation {fileSize}}) <- withStore $ \db -> do
rfd <- appendRcvFD db userId fileId fileDescr
forM_ fileExpires $ liftIO . setFileExpiration db user fileId
-- reading second time in the same transaction as appending description
-- to prevent race condition with accept
ft' <- getRcvFileTransfer db user fileId
@@ -1962,7 +1965,7 @@ processAgentMessageConn cxt user@User {userId} corrId agentConnId agentMessage =
_ -> pure (Nothing, CIFSRcvInvitation, ft)
let RcvFileTransfer {cryptoArgs} = ft'
fileSource = (`CryptoFile` cryptoArgs) <$> filePath
pure (ft', CIFile {fileId, fileName, fileSize, fileSource, fileStatus, fileProtocol})
pure (ft', CIFile {fileId, fileName, fileSize, fileSource, fileStatus, fileProtocol, fileExpires = Nothing})
mkValidFileInvitation :: FileInvitation -> FileInvitation
mkValidFileInvitation fInv@FileInvitation {fileName} = fInv {fileName = safeFileNameStr fileName}
@@ -2418,7 +2421,7 @@ processAgentMessageConn cxt user@User {userId} corrId agentConnId agentMessage =
inline <- receiveInlineMode fInv' Nothing fileChunkSize
RcvFileTransfer {fileId, xftpRcvFile} <- withStore $ \db -> createRcvFileTransfer db userId ct fInv' inline fileChunkSize
let fileProtocol = if isJust xftpRcvFile then FPXFTP else FPSMP
ciFile = Just $ CIFile {fileId, fileName, fileSize, fileSource = Nothing, fileStatus = CIFSRcvInvitation, fileProtocol}
ciFile = Just $ CIFile {fileId, fileName, fileSize, fileSource = Nothing, fileStatus = CIFSRcvInvitation, fileProtocol, fileExpires = Nothing}
content = ciContentNoParse $ CIRcvMsgContent $ MCFile ""
(ci, cInfo) <- saveRcvChatItem' user (CDDirectRcv ct) msg sharedMsgId_ brokerTs content ciFile Nothing False M.empty
toView $ CEvtNewChatItems user [AChatItem SCTDirect SMDRcv cInfo ci]
@@ -2433,7 +2436,7 @@ processAgentMessageConn cxt user@User {userId} corrId agentConnId agentMessage =
inline <- receiveInlineMode fInv' Nothing fileChunkSize
RcvFileTransfer {fileId, xftpRcvFile} <- withStore $ \db -> createRcvGroupFileTransfer db userId gInfo (Just m) FTNormal sharedMsgId_ fInv' inline fileChunkSize
let fileProtocol = if isJust xftpRcvFile then FPXFTP else FPSMP
ciFile = Just $ CIFile {fileId, fileName, fileSize, fileSource = Nothing, fileStatus = CIFSRcvInvitation, fileProtocol}
ciFile = Just $ CIFile {fileId, fileName, fileSize, fileSource = Nothing, fileStatus = CIFSRcvInvitation, fileProtocol, fileExpires = Nothing}
content = ciContentNoParse $ CIRcvMsgContent $ MCFile ""
(ci, cInfo) <- saveRcvChatItem' user (CDGroupRcv gInfo Nothing m) msg sharedMsgId_ brokerTs content ciFile Nothing False M.empty
ci' <- blockedMemberCI gInfo m ci
@@ -3875,7 +3878,7 @@ processAgentMessageConn cxt user@User {userId} corrId agentConnId agentMessage =
where
MsgContainer {scope} = mc
-- file description is always allowed, to allow sending files to support scope
XMsgFileDescr sharedMsgId fileDescr -> void $ groupMessageFileDescription gInfo author_ sharedMsgId fileDescr
XMsgFileDescr sharedMsgId fileDescr fileExpires -> void $ groupMessageFileDescription gInfo author_ sharedMsgId fileDescr fileExpires
XMsgUpdate sharedMsgId mContent mentions ttl live msgScope asGroup_ ->
void $ memberCanSend author_ msgScope $ groupMessageUpdate gInfo author_ sharedMsgId mContent mentions msgScope rcvMsg msgTs ttl live asGroup_
XMsgDel sharedMsgId memId scope_ _ -> void $ groupMessageDelete gInfo author_ sharedMsgId memId scope_ False rcvMsg msgTs