mirror of
https://github.com/simplex-chat/simplexmq.git
synced 2026-11-22 19:58:39 +00:00
xftp: prevent work directory conflicts of concurrent file transfers (#2062)
* tests: cover XFTP files prepared concurrently * xftp: create unique work directories for concurrent file transfers * xftp: test whether a candidate path is taken * xftp: compute candidate names apart from checking and creating paths * xftp: create unique directories in a separate function * xftp: share the candidate path between unique names and directories
This commit is contained in:
1 parent
e8c7b01d3f
commit
f9f3bc9208
3 files changed
+49
-15
No files matched your search
@@ -65,7 +65,7 @@ import Simplex.FileTransfer.Transport (XFTPRcvChunkSpec (..))
|
||||
import qualified Simplex.FileTransfer.Transport as XFTP
|
||||
import Simplex.FileTransfer.Types
|
||||
import qualified Simplex.FileTransfer.Types as FT
|
||||
import Simplex.FileTransfer.Util (removePath, uniqueCombine)
|
||||
import Simplex.FileTransfer.Util (createUniqueDirectory, removePath, uniqueCombine)
|
||||
import Simplex.Messaging.Agent.Client
|
||||
import Simplex.Messaging.Agent.Env.SQLite
|
||||
import Simplex.Messaging.Agent.Protocol
|
||||
@@ -139,8 +139,7 @@ xftpReceiveFile' c userId vfd cfArgs approvedRelays = do
|
||||
xftpPrepareReceiveFile' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Maybe CryptoFileArgs -> Bool -> AM RcvFileId
|
||||
xftpPrepareReceiveFile' c userId (ValidFileDescription fd@FileDescription {redirect}) cfArgs approvedRelays = do
|
||||
g <- asks random
|
||||
prefixPath <- lift $ getPrefixPath "rcv.xftp"
|
||||
createDirectory prefixPath
|
||||
prefixPath <- lift $ createPrefixDirectory "rcv.xftp"
|
||||
let relPrefixPath = takeFileName prefixPath
|
||||
relTmpPath = relPrefixPath </> "xftp.encrypted"
|
||||
relSavePath = relPrefixPath </> "xftp.decrypted"
|
||||
@@ -170,12 +169,12 @@ downloadChunk c FileChunk {replicas = (FileChunkReplica {server} : _)} = do
|
||||
lift . void $ getXFTPRcvWorker True c (Just server)
|
||||
downloadChunk _ _ = throwE $ INTERNAL "no replicas"
|
||||
|
||||
getPrefixPath :: String -> AM' FilePath
|
||||
getPrefixPath suffix = do
|
||||
createPrefixDirectory :: String -> AM' FilePath
|
||||
createPrefixDirectory suffix = do
|
||||
workPath <- getXFTPWorkPath
|
||||
ts <- liftIO getCurrentTime
|
||||
let isoTime = formatTime defaultTimeLocale "%Y%m%d_%H%M%S_%6q" ts
|
||||
uniqueCombine workPath (isoTime <> "_" <> suffix)
|
||||
createUniqueDirectory workPath (isoTime <> "_" <> suffix)
|
||||
|
||||
toFSFilePath :: FilePath -> AM' FilePath
|
||||
toFSFilePath f = (</> f) <$> getXFTPWorkPath
|
||||
@@ -375,8 +374,7 @@ xftpSendFile' c userId file numRecipients storageHours = do
|
||||
xftpPrepareSendFile' :: AgentClient -> UserId -> CryptoFile -> Int -> Maybe Word32 -> AM SndFileId
|
||||
xftpPrepareSendFile' c userId file numRecipients storageHours = do
|
||||
g <- asks random
|
||||
prefixPath <- lift $ getPrefixPath "snd.xftp"
|
||||
createDirectory prefixPath
|
||||
prefixPath <- lift $ createPrefixDirectory "snd.xftp"
|
||||
let relPrefixPath = takeFileName prefixPath
|
||||
key <- atomically $ C.randomSbKey g
|
||||
nonce <- atomically $ C.randomCbNonce g
|
||||
@@ -391,8 +389,7 @@ xftpSendDescription' c userId vfd numRecipients = do
|
||||
xftpPrepareSendDescription' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> AM SndFileId
|
||||
xftpPrepareSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) numRecipients = do
|
||||
g <- asks random
|
||||
prefixPath <- lift $ getPrefixPath "snd.xftp"
|
||||
createDirectory prefixPath
|
||||
prefixPath <- lift $ createPrefixDirectory "snd.xftp"
|
||||
let relPrefixPath = takeFileName prefixPath
|
||||
let directYaml = prefixPath </> "direct.yaml"
|
||||
cfArgs <- atomically $ CF.randomArgs g
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
module Simplex.FileTransfer.Util
|
||||
( uniqueCombine,
|
||||
createUniqueDirectory,
|
||||
safeFileNameStr,
|
||||
removePath,
|
||||
)
|
||||
@@ -7,6 +8,7 @@ where
|
||||
|
||||
import Simplex.Messaging.Util (ifM, whenM)
|
||||
import System.FilePath (makeValid, splitExtensions, takeFileName, (</>))
|
||||
import System.IO.Error (isAlreadyExistsError)
|
||||
import UnliftIO
|
||||
import UnliftIO.Directory
|
||||
|
||||
@@ -20,11 +22,22 @@ uniqueCombine :: MonadIO m => FilePath -> String -> m FilePath
|
||||
uniqueCombine filePath fileName = tryCombine (0 :: Int)
|
||||
where
|
||||
tryCombine n =
|
||||
let (name, ext) = splitExtensions $ safeFileNameStr fileName
|
||||
suffix = if n == 0 then "" else "_" <> show n
|
||||
f = filePath </> (name <> suffix <> ext)
|
||||
let f = candidatePath filePath fileName n
|
||||
in ifM (doesPathExist f) (tryCombine $ n + 1) (pure f)
|
||||
|
||||
createUniqueDirectory :: MonadUnliftIO m => FilePath -> String -> m FilePath
|
||||
createUniqueDirectory filePath dirName = tryCreate (0 :: Int)
|
||||
where
|
||||
tryCreate n =
|
||||
let d = candidatePath filePath dirName n
|
||||
in (d <$ createDirectory d) `catch` \e -> if isAlreadyExistsError e then tryCreate (n + 1) else throwIO e
|
||||
|
||||
candidatePath :: FilePath -> String -> Int -> FilePath
|
||||
candidatePath filePath fileName n = filePath </> (name <> suffix <> ext)
|
||||
where
|
||||
(name, ext) = splitExtensions $ safeFileNameStr fileName
|
||||
suffix = if n == 0 then "" else "_" <> show n
|
||||
|
||||
removePath :: MonadIO m => FilePath -> m ()
|
||||
removePath p = do
|
||||
ifM (doesDirectoryExist p) (removeDirectoryRecursive p) $
|
||||
|
||||
+26
-2
@@ -57,8 +57,8 @@ import Simplex.Messaging.Protocol (BasicAuth, NetworkError (..), ProtoServerWith
|
||||
import Simplex.Messaging.Server.Expiration (ExpirationConfig (..))
|
||||
import Simplex.Messaging.Server.Information (ServerPublicInfo)
|
||||
import Simplex.Messaging.Transport (EntitlementConfig (..))
|
||||
import Simplex.Messaging.Util (tshow)
|
||||
import System.Directory (Permissions (writable), doesDirectoryExist, doesFileExist, getFileSize, getPermissions, listDirectory, removeFile, setPermissions)
|
||||
import Simplex.Messaging.Util (tshow, unlessM)
|
||||
import System.Directory (Permissions (writable), createDirectory, doesDirectoryExist, doesFileExist, getFileSize, getPermissions, listDirectory, removeFile, setPermissions)
|
||||
import System.FilePath ((</>))
|
||||
import Test.Hspec hiding (fit, it, xit)
|
||||
import UnliftIO
|
||||
@@ -82,6 +82,7 @@ xftpAgentTests =
|
||||
it "should send and receive small file without a redirect" testXFTPAgentSendReceiveNoRedirect
|
||||
it "should receive prepared file only after it is started" testXFTPAgentPrepareReceive
|
||||
it "should send prepared file only after it is started" testXFTPAgentPrepareSend
|
||||
it "should prepare files concurrently in distinct work directories" $ \_ -> testXFTPAgentPrepareSendConcurrently
|
||||
it "should extend storage time with an entitlement proof and report the granted expiry" $ \_ -> testXFTPAgentEntitlement
|
||||
describe "sending and receiving with version negotiation" $ beforeWith (const (pure ())) testXFTPAgentSendReceiveMatrix
|
||||
it "should resume receiving file after restart" $ \_ -> testXFTPAgentReceiveRestore
|
||||
@@ -358,6 +359,29 @@ testXFTPAgentPrepareSend = withXFTPServer $ do
|
||||
withAgent 3 agentCfg initAgentServers testDB2 $ \rcp ->
|
||||
runRight_ . void $ testReceive rcp rfd filePath2
|
||||
|
||||
testXFTPAgentPrepareSendConcurrently :: (HasCallStack, HasTestEnv) => IO ()
|
||||
testXFTPAgentPrepareSendConcurrently = do
|
||||
filePath <- createRandomFile_ (kb 1 :: Integer) "testfile"
|
||||
let workPath = testPath "xftp-work-files"
|
||||
rounds = 100
|
||||
filesPerRound = 8
|
||||
prepareAtOnce sndr = do
|
||||
ready <- newTVarIO 0
|
||||
started <- newTVarIO False
|
||||
let waitStarted = unlessM (readTVarIO started) $ yield >> waitStarted
|
||||
prepare = do
|
||||
atomically $ modifyTVar' ready (+ 1)
|
||||
waitStarted
|
||||
runRight $ xftpPrepareSendFile sndr 1 (CF.plain filePath) 1 Nothing
|
||||
concurrently_
|
||||
(replicateConcurrently_ filesPerRound prepare)
|
||||
(atomically $ readTVar ready >>= checkSTM . (== filesPerRound) >> writeTVar started True)
|
||||
createDirectory workPath
|
||||
withAgent 1 agentCfg initAgentServers testDB $ \sndr -> do
|
||||
runRight_ $ xftpStartWorkers sndr (Just workPath)
|
||||
replicateM_ rounds $ prepareAtOnce sndr
|
||||
length <$> listDirectory workPath `shouldReturn` rounds * filesPerRound
|
||||
|
||||
testXFTPAgentSendReceiveMatrix :: Spec
|
||||
testXFTPAgentSendReceiveMatrix = do
|
||||
describe "old server" $ do
|
||||
|
||||
Reference in new issue
Block a user