mirror of
https://github.com/simplex-chat/simplexmq.git
synced 2026-11-22 22:08:26 +00:00
agent: complete file uploads when the app leaves the foreground (#2010)
* agent: continue XFTP uploads started in foreground while suspending * tests: verify XFTP upload stage of agent suspension * docs: plan for XFTP uploads while suspending * docs: revise plan for XFTP uploads while suspending * docs: revise plan wording for XFTP uploads while suspending * agent: exclude ended and stopped XFTP uploads from suspension wait * tests: verify suspension wait for ended and stopped XFTP uploads * docs: plan decisions for ended XFTP uploads while suspending * agent: simplify tryBeginAgentOperation with orElse * agent: rename upload wait functions by the conditions they check * tests: verify database check and deleted rows at suspension start * docs: plan residual case, conditions and names for XFTP suspension * docs: plan tests for the remaining XFTP suspension paths * tests: verify the remaining XFTP suspension paths * docs: update plan line references and the query error test * tests: foreground agent before checking the query error result * docs: plan line references after merging master and stage diagrams * agent: one check of the begun database operation when removing deleted or failed counted files * agent: suspendAgent returns before the database query of counted files * tests: closed database store tests excluded from PostgreSQL agent builds * tests: suspension timeout test waits for the timeout and bounds SUSPENDED * tests: one setup helper for suspension tests with a small sent file, one for the bounded SUSPENDED wait * tests: suspendAgent returns while the agent database is in use * docs: plan states the forked removal at suspension start, uncounted redirects, offline files and the changed tests * docs: plan links the simplex-chat pull request * agent: suspend timeout starts when suspendAgent returns, not after the database query of counted files * docs: plan states the separate removal thread and the TTL cleanup interval
This commit is contained in:
1 parent
0424f667ee
commit
7ca1f4e508
7 files changed
+569
-25
No files matched your search
@@ -404,6 +404,7 @@ xftpPrepareSendDescription' c userId (ValidFileDescription fdDirect@FileDescript
|
||||
xftpStartSendFile' :: AgentClient -> SndFileId -> AM ()
|
||||
xftpStartSendFile' c sndFileEntityId = do
|
||||
withStore c (`startPreparedSndFile` sndFileEntityId)
|
||||
atomically . whenM ((ASForeground ==) <$> readTVar (agentState c)) $ addSndFileInProgress c sndFileEntityId
|
||||
lift . void $ getXFTPSndWorker True c Nothing
|
||||
|
||||
resumeXFTPSndWork :: AgentClient -> Maybe XFTPServer -> AM' ()
|
||||
@@ -420,7 +421,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
|
||||
cfg <- asks config
|
||||
forever $ do
|
||||
lift $ waitForWork doWork
|
||||
liftIO $ assertAgentForeground c
|
||||
liftIO $ assertUploadsAllowed c
|
||||
runXFTPOperation cfg
|
||||
where
|
||||
runXFTPOperation :: AgentConfig -> AM ()
|
||||
@@ -483,7 +484,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
|
||||
SndFileChunkReplica {server} : _ -> Right server
|
||||
createChunk :: Int -> Maybe Word32 -> SndFileChunk -> AM (ProtocolServer 'PXFTP)
|
||||
createChunk numRecipients' storageHours ch = do
|
||||
liftIO $ assertAgentForeground c
|
||||
liftIO $ assertUploadsAllowed c
|
||||
(replica, ProtoServerWithAuth srv _) <- tryCreate
|
||||
withStore' c $ \db -> createSndFileReplica db ch replica
|
||||
pure srv
|
||||
@@ -503,7 +504,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
|
||||
retryLoop loop triedAllSrvs e = do
|
||||
flip catchAllErrors (\_ -> pure ()) $ do
|
||||
when (triedAllSrvs && serverHostError e) $ notify c sndFileEntityId $ SFWARN e
|
||||
liftIO $ assertAgentForeground c
|
||||
liftIO $ assertUploadsAllowed c
|
||||
loop
|
||||
createWithNextSrv triedHosts = do
|
||||
deleted <- withStore' c $ \db -> getSndFileDeleted db sndFileId
|
||||
@@ -524,7 +525,7 @@ runXFTPSndWorker c srv Worker {doWork} = do
|
||||
cfg <- asks config
|
||||
forever $ do
|
||||
lift $ waitForWork doWork
|
||||
liftIO $ assertAgentForeground c
|
||||
liftIO $ assertUploadsAllowed c
|
||||
runXFTPOperation cfg
|
||||
where
|
||||
runXFTPOperation :: AgentConfig -> AM ()
|
||||
@@ -545,7 +546,7 @@ runXFTPSndWorker c srv Worker {doWork} = do
|
||||
when (serverHostError e) $ notify c sndFileEntityId $ SFWARN e
|
||||
liftIO $ closeXFTPServerClient c userId server digest
|
||||
withStore' c $ \db -> updateSndChunkReplicaDelay db sndChunkReplicaId replicaDelay
|
||||
liftIO $ assertAgentForeground c
|
||||
liftIO $ assertUploadsAllowed c
|
||||
loop
|
||||
retryDone e = do
|
||||
incXFTPServerStat c userId srv uploadErrs
|
||||
@@ -556,9 +557,9 @@ runXFTPSndWorker c srv Worker {doWork} = do
|
||||
fsFilePath <- lift $ toFSFilePath filePath
|
||||
unlessM (doesFileExist fsFilePath) $ throwE $ FILE NO_FILE
|
||||
let chunkSpec' = chunkSpec {filePath = fsFilePath} :: XFTPChunkSpec
|
||||
liftIO $ assertAgentForeground c
|
||||
liftIO $ assertUploadsAllowed c
|
||||
agentXFTPUploadChunk c userId chunkDigest replica' chunkSpec'
|
||||
liftIO $ waitUntilForeground c
|
||||
liftIO $ waitUntilUploadsAllowed c
|
||||
sf@SndFile {sndFileEntityId, prefixPath, chunks} <- withStore c $ \db -> do
|
||||
lockSndFileForUpdate db sndFileId
|
||||
updateSndChunkReplicaStatus db sndChunkReplicaId SFRSUploaded
|
||||
@@ -659,7 +660,7 @@ deleteSndFileInternal :: AgentClient -> SndFileId -> AM' ()
|
||||
deleteSndFileInternal c sndFileEntityId = deleteSndFilesInternal c [sndFileEntityId]
|
||||
|
||||
deleteSndFilesInternal :: AgentClient -> [SndFileId] -> AM' ()
|
||||
deleteSndFilesInternal c sndFileEntityIds = do
|
||||
deleteSndFilesInternal c sndFileEntityIds = flip E.finally (atomically $ mapM_ (removeSndFileInProgress c) sndFileEntityIds) $ do
|
||||
sndFiles <- rights <$> withStoreBatch c (\db -> map (fmap (first storeError) . getSndFileByEntityId db) sndFileEntityIds)
|
||||
let (toDelete, toMarkDeleted) = partition fileComplete sndFiles
|
||||
workPath <- getXFTPWorkPath
|
||||
@@ -744,3 +745,8 @@ assertAgentForeground :: AgentClient -> IO ()
|
||||
assertAgentForeground c = do
|
||||
throwWhenInactive c
|
||||
waitUntilForeground c
|
||||
|
||||
assertUploadsAllowed :: AgentClient -> IO ()
|
||||
assertUploadsAllowed c = do
|
||||
throwWhenInactive c
|
||||
waitUntilUploadsAllowed c
|
||||
@@ -3444,15 +3444,17 @@ suspendAgent c@AgentClient {agentState = as} maxDelay = do
|
||||
writeTVar as ASSuspending
|
||||
suspendOperation c AONtfNetwork $ pure ()
|
||||
suspendOperation c AORcvNetwork $
|
||||
suspendOperation c AOMsgDelivery $
|
||||
suspendSendingAndDatabase c
|
||||
suspendOperation c AOSndFiles $
|
||||
suspendMsgDelivery c
|
||||
readTVar as
|
||||
when (state == ASSuspending) . void . forkIO $ do
|
||||
threadDelay maxDelay
|
||||
-- liftIO $ putStrLn "suspendAgent after timeout"
|
||||
atomically . whenSuspending c $ do
|
||||
-- unsafeIOToSTM $ putStrLn $ "in timeout: suspendSendingAndDatabase"
|
||||
suspendSendingAndDatabase c
|
||||
when (state == ASSuspending) $ do
|
||||
void . forkIO $ removeDeletedOrFailedSndFilesInProgress c
|
||||
void . forkIO $ do
|
||||
threadDelay maxDelay
|
||||
-- liftIO $ putStrLn "suspendAgent after timeout"
|
||||
atomically . whenSuspending c $ do
|
||||
-- unsafeIOToSTM $ putStrLn $ "in timeout: suspendSendingAndDatabase"
|
||||
suspendSendingAndDatabase c
|
||||
|
||||
execAgentStoreSQL :: AgentClient -> Text -> AE [Text]
|
||||
execAgentStoreSQL c sql = withAgentEnv c $ withStore' c (`execSQL` sql)
|
||||
@@ -3544,6 +3546,7 @@ cleanupManager c@AgentClient {subQ} = do
|
||||
sndFilesTTL <- asks $ sndFilesTTL . config
|
||||
sndExpired <- withStore' c (`getSndFilesExpired` sndFilesTTL)
|
||||
forM_ sndExpired $ \(dbId, entId, p) -> flip catchAllErrors (notify entId . SFERR) $ do
|
||||
atomically $ removeSndFileInProgress c entId
|
||||
lift . forM_ p $ removePath <=< toFSFilePath
|
||||
withStore' c (`deleteSndFile'` dbId)
|
||||
deleteSndFilesDeleted = do
|
||||
|
||||
@@ -156,9 +156,14 @@ module Simplex.Messaging.Agent.Client
|
||||
waitUntilForeground,
|
||||
waitWhileSuspended,
|
||||
suspendSendingAndDatabase,
|
||||
suspendMsgDelivery,
|
||||
suspendOperation,
|
||||
notifySuspended,
|
||||
whenSuspending,
|
||||
addSndFileInProgress,
|
||||
removeSndFileInProgress,
|
||||
removeDeletedOrFailedSndFilesInProgress,
|
||||
waitUntilUploadsAllowed,
|
||||
withStore,
|
||||
withStore',
|
||||
withStoreBatch,
|
||||
@@ -235,7 +240,7 @@ import Simplex.FileTransfer.Description (ChunkReplicaId (..), FileDigest (..), k
|
||||
import Simplex.FileTransfer.Protocol (FileInfo (..), FileResponse)
|
||||
import Simplex.FileTransfer.Transport (XFTPErrorType (DIGEST), XFTPRcvChunkSpec (..), XFTPVersion)
|
||||
import qualified Simplex.FileTransfer.Transport as XFTP
|
||||
import Simplex.FileTransfer.Types (DeletedSndChunkReplica (..), NewSndChunkReplica (..), RcvFileChunkReplica (..), SndFileChunk (..), SndFileChunkReplica (..))
|
||||
import Simplex.FileTransfer.Types (DeletedSndChunkReplica (..), NewSndChunkReplica (..), RcvFileChunkReplica (..), SndFileChunk (..), SndFileChunkReplica (..), SndFileId)
|
||||
import Simplex.FileTransfer.Util (uniqueCombine)
|
||||
import Simplex.Messaging.Agent.Env.SQLite
|
||||
import Simplex.Messaging.Agent.Lock
|
||||
@@ -371,6 +376,8 @@ data AgentClient = AgentClient
|
||||
asyncCmdWorkers :: TMap (ConnId, Maybe SMPServer) Worker,
|
||||
ntfNetworkOp :: TVar AgentOpState,
|
||||
rcvNetworkOp :: TVar AgentOpState,
|
||||
sndFilesOp :: TVar AgentOpState,
|
||||
sndFilesInProgress :: TVar (Set SndFileId),
|
||||
msgDeliveryOp :: TVar AgentOpState,
|
||||
sndNetworkOp :: TVar AgentOpState,
|
||||
databaseOp :: TVar AgentOpState,
|
||||
@@ -466,19 +473,20 @@ runWorkerAsync Worker {action} work =
|
||||
where
|
||||
start = atomically . putTMVar action . Just =<< mkWeakThreadId =<< forkIO work
|
||||
|
||||
data AgentOperation = AONtfNetwork | AORcvNetwork | AOMsgDelivery | AOSndNetwork | AODatabase
|
||||
data AgentOperation = AONtfNetwork | AORcvNetwork | AOSndFiles | AOMsgDelivery | AOSndNetwork | AODatabase
|
||||
deriving (Eq, Show)
|
||||
|
||||
agentOpSel :: AgentOperation -> (AgentClient -> TVar AgentOpState)
|
||||
agentOpSel = \case
|
||||
AONtfNetwork -> ntfNetworkOp
|
||||
AORcvNetwork -> rcvNetworkOp
|
||||
AOSndFiles -> sndFilesOp
|
||||
AOMsgDelivery -> msgDeliveryOp
|
||||
AOSndNetwork -> sndNetworkOp
|
||||
AODatabase -> databaseOp
|
||||
|
||||
agentOperations :: [AgentClient -> TVar AgentOpState]
|
||||
agentOperations = [ntfNetworkOp, rcvNetworkOp, msgDeliveryOp, sndNetworkOp, databaseOp]
|
||||
agentOperations = [ntfNetworkOp, rcvNetworkOp, sndFilesOp, msgDeliveryOp, sndNetworkOp, databaseOp]
|
||||
|
||||
data AgentOpState = AgentOpState {opSuspended :: !Bool, opsInProgress :: !Int}
|
||||
|
||||
@@ -539,6 +547,8 @@ newAgentClient clientId InitialAgentServers {smp, ntf, xftp, entitlements, netCf
|
||||
asyncCmdWorkers <- TM.emptyIO
|
||||
ntfNetworkOp <- newTVarIO $ AgentOpState False 0
|
||||
rcvNetworkOp <- newTVarIO $ AgentOpState False 0
|
||||
sndFilesOp <- newTVarIO $ AgentOpState False 0
|
||||
sndFilesInProgress <- newTVarIO S.empty
|
||||
msgDeliveryOp <- newTVarIO $ AgentOpState False 0
|
||||
sndNetworkOp <- newTVarIO $ AgentOpState False 0
|
||||
databaseOp <- newTVarIO $ AgentOpState False 0
|
||||
@@ -583,6 +593,8 @@ newAgentClient clientId InitialAgentServers {smp, ntf, xftp, entitlements, netCf
|
||||
asyncCmdWorkers,
|
||||
ntfNetworkOp,
|
||||
rcvNetworkOp,
|
||||
sndFilesOp,
|
||||
sndFilesInProgress,
|
||||
msgDeliveryOp,
|
||||
sndNetworkOp,
|
||||
databaseOp,
|
||||
@@ -1009,6 +1021,7 @@ closeAgentClient c = do
|
||||
atomically $ SS.clear $ currentSubs c
|
||||
clear subscrConns
|
||||
clear getMsgLocks
|
||||
atomically $ readTVar (sndFilesInProgress c) >>= mapM_ (removeSndFileInProgress c)
|
||||
where
|
||||
clearWorkers :: Ord k => (AgentClient -> TMap k a) -> IO (Map k a)
|
||||
clearWorkers workers = atomically $ swapTVar (workers c) mempty
|
||||
@@ -2352,8 +2365,10 @@ endAgentOperation :: AgentClient -> AgentOperation -> STM ()
|
||||
endAgentOperation c op = endOperation c op $ case op of
|
||||
AONtfNetwork -> pure ()
|
||||
AORcvNetwork ->
|
||||
suspendOperation c AOMsgDelivery $
|
||||
suspendSendingAndDatabase c
|
||||
suspendOperation c AOSndFiles $
|
||||
suspendMsgDelivery c
|
||||
AOSndFiles ->
|
||||
suspendMsgDelivery c
|
||||
AOMsgDelivery ->
|
||||
suspendSendingAndDatabase c
|
||||
AOSndNetwork ->
|
||||
@@ -2362,6 +2377,11 @@ endAgentOperation c op = endOperation c op $ case op of
|
||||
AODatabase ->
|
||||
notifySuspended c
|
||||
|
||||
suspendMsgDelivery :: AgentClient -> STM ()
|
||||
suspendMsgDelivery c =
|
||||
suspendOperation c AOMsgDelivery $
|
||||
suspendSendingAndDatabase c
|
||||
|
||||
suspendSendingAndDatabase :: AgentClient -> STM ()
|
||||
suspendSendingAndDatabase c =
|
||||
suspendOperation c AOSndNetwork $
|
||||
@@ -2392,6 +2412,29 @@ whenSuspending :: AgentClient -> STM () -> STM ()
|
||||
whenSuspending c = whenM ((== ASSuspending) <$> readTVar (agentState c))
|
||||
{-# INLINE whenSuspending #-}
|
||||
|
||||
addSndFileInProgress :: AgentClient -> SndFileId -> STM ()
|
||||
addSndFileInProgress c fId = do
|
||||
added <- stateTVar (sndFilesInProgress c) $ \fs -> (S.notMember fId fs, S.insert fId fs)
|
||||
when added $ modifyTVar' (sndFilesOp c) $ \s -> s {opsInProgress = opsInProgress s + 1}
|
||||
|
||||
removeSndFileInProgress :: AgentClient -> SndFileId -> STM ()
|
||||
removeSndFileInProgress c fId = do
|
||||
removed <- stateTVar (sndFilesInProgress c) $ \fs -> (S.member fId fs, S.delete fId fs)
|
||||
when removed $ endAgentOperation c AOSndFiles
|
||||
|
||||
removeDeletedOrFailedSndFilesInProgress :: AgentClient -> IO ()
|
||||
removeDeletedOrFailedSndFilesInProgress c = do
|
||||
fIds <- readTVarIO $ sndFilesInProgress c
|
||||
unless (S.null fIds) $
|
||||
E.mask $ \restore ->
|
||||
whenM (atomically $ tryBeginAgentOperation c AODatabase) $
|
||||
restore (removeDeletedOrFailed fIds) `E.finally` atomically (endAgentOperation c AODatabase)
|
||||
where
|
||||
removeDeletedOrFailed fIds =
|
||||
E.tryAny (withTransaction (agentClientStore c) getNotDeletedOrFailedSndFileIds) >>= \case
|
||||
Right fIds' -> atomically $ mapM_ (removeSndFileInProgress c) $ fIds `S.difference` S.fromList fIds'
|
||||
Left e -> logError $ "removeDeletedOrFailedSndFilesInProgress error: " <> tshow e
|
||||
|
||||
beginAgentOperation :: AgentClient -> AgentOperation -> STM ()
|
||||
beginAgentOperation c op = do
|
||||
let opVar = agentOpSel op c
|
||||
@@ -2401,6 +2444,9 @@ beginAgentOperation c op = do
|
||||
-- unsafeIOToSTM $ putStrLn $ "beginOperation! " <> show op <> " " <> show (opsInProgress s + 1)
|
||||
writeTVar opVar $! s {opsInProgress = opsInProgress s + 1}
|
||||
|
||||
tryBeginAgentOperation :: AgentClient -> AgentOperation -> STM Bool
|
||||
tryBeginAgentOperation c op = (True <$ beginAgentOperation c op) `orElse` pure False
|
||||
|
||||
agentOperationBracket :: MonadUnliftIO m => AgentClient -> AgentOperation -> (AgentClient -> IO ()) -> m a -> m a
|
||||
agentOperationBracket c op check action =
|
||||
E.bracket
|
||||
@@ -2424,6 +2470,18 @@ waitWhileSuspended c =
|
||||
suspended :: Monad m => (TVar AgentState -> m AgentState) -> m Bool
|
||||
suspended rd = (ASSuspended ==) <$> rd (agentState c)
|
||||
|
||||
waitUntilUploadsAllowed :: AgentClient -> IO ()
|
||||
waitUntilUploadsAllowed c = atomically $ unlessM uploadsAllowed retry
|
||||
where
|
||||
uploadsAllowed =
|
||||
readTVar (agentState c) >>= \case
|
||||
ASForeground -> pure True
|
||||
ASSuspending -> do
|
||||
AgentOpState {opsInProgress} <- readTVar $ sndFilesOp c
|
||||
AgentOpState {opSuspended} <- readTVar $ sndNetworkOp c
|
||||
pure $ opsInProgress > 0 && not opSuspended
|
||||
ASSuspended -> pure False
|
||||
|
||||
withStore' :: AgentClient -> (DB.Connection -> IO a) -> AM a
|
||||
withStore' c action = withStore c $ fmap Right . action
|
||||
{-# INLINE withStore' #-}
|
||||
|
||||
@@ -262,6 +262,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
|
||||
getCleanupSndFilesPrefixPaths,
|
||||
getCleanupSndFilesDeleted,
|
||||
getSndFilesExpired,
|
||||
getNotDeletedOrFailedSndFileIds,
|
||||
createDeletedSndChunkReplica,
|
||||
getNextDeletedSndChunkReplica,
|
||||
updateDeletedSndChunkReplicaDelay,
|
||||
@@ -3894,6 +3895,10 @@ getSndFilesExpired db ttl = do
|
||||
|]
|
||||
(Only cutoffTs)
|
||||
|
||||
getNotDeletedOrFailedSndFileIds :: DB.Connection -> IO [SndFileId]
|
||||
getNotDeletedOrFailedSndFileIds db =
|
||||
map fromOnly <$> DB.query_ db "SELECT snd_file_entity_id FROM snd_files WHERE deleted = 0 AND failed = 0"
|
||||
|
||||
createDeletedSndChunkReplica :: DB.Connection -> UserId -> FileChunkReplica -> FileDigest -> IO ()
|
||||
createDeletedSndChunkReplica db userId FileChunkReplica {server, replicaId, replicaKey} chunkDigest = do
|
||||
srvId <- createXFTPServer_ db server
|
||||
|
||||
Reference in new issue
Block a user