agent: complete file uploads when the app leaves the foreground (#2010)

* agent: continue XFTP uploads started in foreground while suspending

* tests: verify XFTP upload stage of agent suspension

* docs: plan for XFTP uploads while suspending

* docs: revise plan for XFTP uploads while suspending

* docs: revise plan wording for XFTP uploads while suspending

* agent: exclude ended and stopped XFTP uploads from suspension wait

* tests: verify suspension wait for ended and stopped XFTP uploads

* docs: plan decisions for ended XFTP uploads while suspending

* agent: simplify tryBeginAgentOperation with orElse

* agent: rename upload wait functions by the conditions they check

* tests: verify database check and deleted rows at suspension start

* docs: plan residual case, conditions and names for XFTP suspension

* docs: plan tests for the remaining XFTP suspension paths

* tests: verify the remaining XFTP suspension paths

* docs: update plan line references and the query error test

* tests: foreground agent before checking the query error result

* docs: plan line references after merging master and stage diagrams

* agent: one check of the begun database operation when removing deleted or failed counted files

* agent: suspendAgent returns before the database query of counted files

* tests: closed database store tests excluded from PostgreSQL agent builds

* tests: suspension timeout test waits for the timeout and bounds SUSPENDED

* tests: one setup helper for suspension tests with a small sent file, one for the bounded SUSPENDED wait

* tests: suspendAgent returns while the agent database is in use

* docs: plan states the forked removal at suspension start, uncounted redirects, offline files and the changed tests

* docs: plan links the simplex-chat pull request

* agent: suspend timeout starts when suspendAgent returns, not after the database query of counted files

* docs: plan states the separate removal thread and the TTL cleanup interval
This commit is contained in:
Evgeny @ SimpleX Chat authored and GitHub committed 2026-11-07 15:23:45 +00:00
1 parent 0424f667ee
commit 7ca1f4e508
7 files changed
+569 -25

No files matched your search

+14 -8
View File
@@ -404,6 +404,7 @@ xftpPrepareSendDescription' c userId (ValidFileDescription fdDirect@FileDescript
xftpStartSendFile' :: AgentClient -> SndFileId -> AM ()
xftpStartSendFile' c sndFileEntityId = do
withStore c (`startPreparedSndFile` sndFileEntityId)
atomically . whenM ((ASForeground ==) <$> readTVar (agentState c)) $ addSndFileInProgress c sndFileEntityId
lift . void $ getXFTPSndWorker True c Nothing
resumeXFTPSndWork :: AgentClient -> Maybe XFTPServer -> AM' ()
@@ -420,7 +421,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
cfg <- asks config
forever $ do
lift $ waitForWork doWork
liftIO $ assertAgentForeground c
liftIO $ assertUploadsAllowed c
runXFTPOperation cfg
where
runXFTPOperation :: AgentConfig -> AM ()
@@ -483,7 +484,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
SndFileChunkReplica {server} : _ -> Right server
createChunk :: Int -> Maybe Word32 -> SndFileChunk -> AM (ProtocolServer 'PXFTP)
createChunk numRecipients' storageHours ch = do
liftIO $ assertAgentForeground c
liftIO $ assertUploadsAllowed c
(replica, ProtoServerWithAuth srv _) <- tryCreate
withStore' c $ \db -> createSndFileReplica db ch replica
pure srv
@@ -503,7 +504,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
retryLoop loop triedAllSrvs e = do
flip catchAllErrors (\_ -> pure ()) $ do
when (triedAllSrvs && serverHostError e) $ notify c sndFileEntityId $ SFWARN e
liftIO $ assertAgentForeground c
liftIO $ assertUploadsAllowed c
loop
createWithNextSrv triedHosts = do
deleted <- withStore' c $ \db -> getSndFileDeleted db sndFileId
@@ -524,7 +525,7 @@ runXFTPSndWorker c srv Worker {doWork} = do
cfg <- asks config
forever $ do
lift $ waitForWork doWork
liftIO $ assertAgentForeground c
liftIO $ assertUploadsAllowed c
runXFTPOperation cfg
where
runXFTPOperation :: AgentConfig -> AM ()
@@ -545,7 +546,7 @@ runXFTPSndWorker c srv Worker {doWork} = do
when (serverHostError e) $ notify c sndFileEntityId $ SFWARN e
liftIO $ closeXFTPServerClient c userId server digest
withStore' c $ \db -> updateSndChunkReplicaDelay db sndChunkReplicaId replicaDelay
liftIO $ assertAgentForeground c
liftIO $ assertUploadsAllowed c
loop
retryDone e = do
incXFTPServerStat c userId srv uploadErrs
@@ -556,9 +557,9 @@ runXFTPSndWorker c srv Worker {doWork} = do
fsFilePath <- lift $ toFSFilePath filePath
unlessM (doesFileExist fsFilePath) $ throwE $ FILE NO_FILE
let chunkSpec' = chunkSpec {filePath = fsFilePath} :: XFTPChunkSpec
liftIO $ assertAgentForeground c
liftIO $ assertUploadsAllowed c
agentXFTPUploadChunk c userId chunkDigest replica' chunkSpec'
liftIO $ waitUntilForeground c
liftIO $ waitUntilUploadsAllowed c
sf@SndFile {sndFileEntityId, prefixPath, chunks} <- withStore c $ \db -> do
lockSndFileForUpdate db sndFileId
updateSndChunkReplicaStatus db sndChunkReplicaId SFRSUploaded
@@ -659,7 +660,7 @@ deleteSndFileInternal :: AgentClient -> SndFileId -> AM' ()
deleteSndFileInternal c sndFileEntityId = deleteSndFilesInternal c [sndFileEntityId]
deleteSndFilesInternal :: AgentClient -> [SndFileId] -> AM' ()
deleteSndFilesInternal c sndFileEntityIds = do
deleteSndFilesInternal c sndFileEntityIds = flip E.finally (atomically $ mapM_ (removeSndFileInProgress c) sndFileEntityIds) $ do
sndFiles <- rights <$> withStoreBatch c (\db -> map (fmap (first storeError) . getSndFileByEntityId db) sndFileEntityIds)
let (toDelete, toMarkDeleted) = partition fileComplete sndFiles
workPath <- getXFTPWorkPath
@@ -744,3 +745,8 @@ assertAgentForeground :: AgentClient -> IO ()
assertAgentForeground c = do
throwWhenInactive c
waitUntilForeground c
assertUploadsAllowed :: AgentClient -> IO ()
assertUploadsAllowed c = do
throwWhenInactive c
waitUntilUploadsAllowed c
+11 -8
View File
@@ -3444,15 +3444,17 @@ suspendAgent c@AgentClient {agentState = as} maxDelay = do
writeTVar as ASSuspending
suspendOperation c AONtfNetwork $ pure ()
suspendOperation c AORcvNetwork $
suspendOperation c AOMsgDelivery $
suspendSendingAndDatabase c
suspendOperation c AOSndFiles $
suspendMsgDelivery c
readTVar as
when (state == ASSuspending) . void . forkIO $ do
threadDelay maxDelay
-- liftIO $ putStrLn "suspendAgent after timeout"
atomically . whenSuspending c $ do
-- unsafeIOToSTM $ putStrLn $ "in timeout: suspendSendingAndDatabase"
suspendSendingAndDatabase c
when (state == ASSuspending) $ do
void . forkIO $ removeDeletedOrFailedSndFilesInProgress c
void . forkIO $ do
threadDelay maxDelay
-- liftIO $ putStrLn "suspendAgent after timeout"
atomically . whenSuspending c $ do
-- unsafeIOToSTM $ putStrLn $ "in timeout: suspendSendingAndDatabase"
suspendSendingAndDatabase c
execAgentStoreSQL :: AgentClient -> Text -> AE [Text]
execAgentStoreSQL c sql = withAgentEnv c $ withStore' c (`execSQL` sql)
@@ -3544,6 +3546,7 @@ cleanupManager c@AgentClient {subQ} = do
sndFilesTTL <- asks $ sndFilesTTL . config
sndExpired <- withStore' c (`getSndFilesExpired` sndFilesTTL)
forM_ sndExpired $ \(dbId, entId, p) -> flip catchAllErrors (notify entId . SFERR) $ do
atomically $ removeSndFileInProgress c entId
lift . forM_ p $ removePath <=< toFSFilePath
withStore' c (`deleteSndFile'` dbId)
deleteSndFilesDeleted = do
+63 -5
View File
@@ -156,9 +156,14 @@ module Simplex.Messaging.Agent.Client
waitUntilForeground,
waitWhileSuspended,
suspendSendingAndDatabase,
suspendMsgDelivery,
suspendOperation,
notifySuspended,
whenSuspending,
addSndFileInProgress,
removeSndFileInProgress,
removeDeletedOrFailedSndFilesInProgress,
waitUntilUploadsAllowed,
withStore,
withStore',
withStoreBatch,
@@ -235,7 +240,7 @@ import Simplex.FileTransfer.Description (ChunkReplicaId (..), FileDigest (..), k
import Simplex.FileTransfer.Protocol (FileInfo (..), FileResponse)
import Simplex.FileTransfer.Transport (XFTPErrorType (DIGEST), XFTPRcvChunkSpec (..), XFTPVersion)
import qualified Simplex.FileTransfer.Transport as XFTP
import Simplex.FileTransfer.Types (DeletedSndChunkReplica (..), NewSndChunkReplica (..), RcvFileChunkReplica (..), SndFileChunk (..), SndFileChunkReplica (..))
import Simplex.FileTransfer.Types (DeletedSndChunkReplica (..), NewSndChunkReplica (..), RcvFileChunkReplica (..), SndFileChunk (..), SndFileChunkReplica (..), SndFileId)
import Simplex.FileTransfer.Util (uniqueCombine)
import Simplex.Messaging.Agent.Env.SQLite
import Simplex.Messaging.Agent.Lock
@@ -371,6 +376,8 @@ data AgentClient = AgentClient
asyncCmdWorkers :: TMap (ConnId, Maybe SMPServer) Worker,
ntfNetworkOp :: TVar AgentOpState,
rcvNetworkOp :: TVar AgentOpState,
sndFilesOp :: TVar AgentOpState,
sndFilesInProgress :: TVar (Set SndFileId),
msgDeliveryOp :: TVar AgentOpState,
sndNetworkOp :: TVar AgentOpState,
databaseOp :: TVar AgentOpState,
@@ -466,19 +473,20 @@ runWorkerAsync Worker {action} work =
where
start = atomically . putTMVar action . Just =<< mkWeakThreadId =<< forkIO work
data AgentOperation = AONtfNetwork | AORcvNetwork | AOMsgDelivery | AOSndNetwork | AODatabase
data AgentOperation = AONtfNetwork | AORcvNetwork | AOSndFiles | AOMsgDelivery | AOSndNetwork | AODatabase
deriving (Eq, Show)
agentOpSel :: AgentOperation -> (AgentClient -> TVar AgentOpState)
agentOpSel = \case
AONtfNetwork -> ntfNetworkOp
AORcvNetwork -> rcvNetworkOp
AOSndFiles -> sndFilesOp
AOMsgDelivery -> msgDeliveryOp
AOSndNetwork -> sndNetworkOp
AODatabase -> databaseOp
agentOperations :: [AgentClient -> TVar AgentOpState]
agentOperations = [ntfNetworkOp, rcvNetworkOp, msgDeliveryOp, sndNetworkOp, databaseOp]
agentOperations = [ntfNetworkOp, rcvNetworkOp, sndFilesOp, msgDeliveryOp, sndNetworkOp, databaseOp]
data AgentOpState = AgentOpState {opSuspended :: !Bool, opsInProgress :: !Int}
@@ -539,6 +547,8 @@ newAgentClient clientId InitialAgentServers {smp, ntf, xftp, entitlements, netCf
asyncCmdWorkers <- TM.emptyIO
ntfNetworkOp <- newTVarIO $ AgentOpState False 0
rcvNetworkOp <- newTVarIO $ AgentOpState False 0
sndFilesOp <- newTVarIO $ AgentOpState False 0
sndFilesInProgress <- newTVarIO S.empty
msgDeliveryOp <- newTVarIO $ AgentOpState False 0
sndNetworkOp <- newTVarIO $ AgentOpState False 0
databaseOp <- newTVarIO $ AgentOpState False 0
@@ -583,6 +593,8 @@ newAgentClient clientId InitialAgentServers {smp, ntf, xftp, entitlements, netCf
asyncCmdWorkers,
ntfNetworkOp,
rcvNetworkOp,
sndFilesOp,
sndFilesInProgress,
msgDeliveryOp,
sndNetworkOp,
databaseOp,
@@ -1009,6 +1021,7 @@ closeAgentClient c = do
atomically $ SS.clear $ currentSubs c
clear subscrConns
clear getMsgLocks
atomically $ readTVar (sndFilesInProgress c) >>= mapM_ (removeSndFileInProgress c)
where
clearWorkers :: Ord k => (AgentClient -> TMap k a) -> IO (Map k a)
clearWorkers workers = atomically $ swapTVar (workers c) mempty
@@ -2352,8 +2365,10 @@ endAgentOperation :: AgentClient -> AgentOperation -> STM ()
endAgentOperation c op = endOperation c op $ case op of
AONtfNetwork -> pure ()
AORcvNetwork ->
suspendOperation c AOMsgDelivery $
suspendSendingAndDatabase c
suspendOperation c AOSndFiles $
suspendMsgDelivery c
AOSndFiles ->
suspendMsgDelivery c
AOMsgDelivery ->
suspendSendingAndDatabase c
AOSndNetwork ->
@@ -2362,6 +2377,11 @@ endAgentOperation c op = endOperation c op $ case op of
AODatabase ->
notifySuspended c
suspendMsgDelivery :: AgentClient -> STM ()
suspendMsgDelivery c =
suspendOperation c AOMsgDelivery $
suspendSendingAndDatabase c
suspendSendingAndDatabase :: AgentClient -> STM ()
suspendSendingAndDatabase c =
suspendOperation c AOSndNetwork $
@@ -2392,6 +2412,29 @@ whenSuspending :: AgentClient -> STM () -> STM ()
whenSuspending c = whenM ((== ASSuspending) <$> readTVar (agentState c))
{-# INLINE whenSuspending #-}
addSndFileInProgress :: AgentClient -> SndFileId -> STM ()
addSndFileInProgress c fId = do
added <- stateTVar (sndFilesInProgress c) $ \fs -> (S.notMember fId fs, S.insert fId fs)
when added $ modifyTVar' (sndFilesOp c) $ \s -> s {opsInProgress = opsInProgress s + 1}
removeSndFileInProgress :: AgentClient -> SndFileId -> STM ()
removeSndFileInProgress c fId = do
removed <- stateTVar (sndFilesInProgress c) $ \fs -> (S.member fId fs, S.delete fId fs)
when removed $ endAgentOperation c AOSndFiles
removeDeletedOrFailedSndFilesInProgress :: AgentClient -> IO ()
removeDeletedOrFailedSndFilesInProgress c = do
fIds <- readTVarIO $ sndFilesInProgress c
unless (S.null fIds) $
E.mask $ \restore ->
whenM (atomically $ tryBeginAgentOperation c AODatabase) $
restore (removeDeletedOrFailed fIds) `E.finally` atomically (endAgentOperation c AODatabase)
where
removeDeletedOrFailed fIds =
E.tryAny (withTransaction (agentClientStore c) getNotDeletedOrFailedSndFileIds) >>= \case
Right fIds' -> atomically $ mapM_ (removeSndFileInProgress c) $ fIds `S.difference` S.fromList fIds'
Left e -> logError $ "removeDeletedOrFailedSndFilesInProgress error: " <> tshow e
beginAgentOperation :: AgentClient -> AgentOperation -> STM ()
beginAgentOperation c op = do
let opVar = agentOpSel op c
@@ -2401,6 +2444,9 @@ beginAgentOperation c op = do
-- unsafeIOToSTM $ putStrLn $ "beginOperation! " <> show op <> " " <> show (opsInProgress s + 1)
writeTVar opVar $! s {opsInProgress = opsInProgress s + 1}
tryBeginAgentOperation :: AgentClient -> AgentOperation -> STM Bool
tryBeginAgentOperation c op = (True <$ beginAgentOperation c op) `orElse` pure False
agentOperationBracket :: MonadUnliftIO m => AgentClient -> AgentOperation -> (AgentClient -> IO ()) -> m a -> m a
agentOperationBracket c op check action =
E.bracket
@@ -2424,6 +2470,18 @@ waitWhileSuspended c =
suspended :: Monad m => (TVar AgentState -> m AgentState) -> m Bool
suspended rd = (ASSuspended ==) <$> rd (agentState c)
waitUntilUploadsAllowed :: AgentClient -> IO ()
waitUntilUploadsAllowed c = atomically $ unlessM uploadsAllowed retry
where
uploadsAllowed =
readTVar (agentState c) >>= \case
ASForeground -> pure True
ASSuspending -> do
AgentOpState {opsInProgress} <- readTVar $ sndFilesOp c
AgentOpState {opSuspended} <- readTVar $ sndNetworkOp c
pure $ opsInProgress > 0 && not opSuspended
ASSuspended -> pure False
withStore' :: AgentClient -> (DB.Connection -> IO a) -> AM a
withStore' c action = withStore c $ fmap Right . action
{-# INLINE withStore' #-}
@@ -262,6 +262,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
getCleanupSndFilesPrefixPaths,
getCleanupSndFilesDeleted,
getSndFilesExpired,
getNotDeletedOrFailedSndFileIds,
createDeletedSndChunkReplica,
getNextDeletedSndChunkReplica,
updateDeletedSndChunkReplicaDelay,
@@ -3894,6 +3895,10 @@ getSndFilesExpired db ttl = do
|]
(Only cutoffTs)
getNotDeletedOrFailedSndFileIds :: DB.Connection -> IO [SndFileId]
getNotDeletedOrFailedSndFileIds db =
map fromOnly <$> DB.query_ db "SELECT snd_file_entity_id FROM snd_files WHERE deleted = 0 AND failed = 0"
createDeletedSndChunkReplica :: DB.Connection -> UserId -> FileChunkReplica -> FileDigest -> IO ()
createDeletedSndChunkReplica db userId FileChunkReplica {server, replicaId, replicaKey} chunkDigest = do
srvId <- createXFTPServer_ db server