diff --git a/src/Simplex/FileTransfer/Agent.hs b/src/Simplex/FileTransfer/Agent.hs index 095e0b4db..9855fb313 100644 --- a/src/Simplex/FileTransfer/Agent.hs +++ b/src/Simplex/FileTransfer/Agent.hs @@ -65,7 +65,7 @@ import Simplex.FileTransfer.Transport (XFTPRcvChunkSpec (..)) import qualified Simplex.FileTransfer.Transport as XFTP import Simplex.FileTransfer.Types import qualified Simplex.FileTransfer.Types as FT -import Simplex.FileTransfer.Util (removePath, uniqueCombine) +import Simplex.FileTransfer.Util (createUniqueDirectory, removePath, uniqueCombine) import Simplex.Messaging.Agent.Client import Simplex.Messaging.Agent.Env.SQLite import Simplex.Messaging.Agent.Protocol @@ -139,8 +139,7 @@ xftpReceiveFile' c userId vfd cfArgs approvedRelays = do xftpPrepareReceiveFile' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Maybe CryptoFileArgs -> Bool -> AM RcvFileId xftpPrepareReceiveFile' c userId (ValidFileDescription fd@FileDescription {redirect}) cfArgs approvedRelays = do g <- asks random - prefixPath <- lift $ getPrefixPath "rcv.xftp" - createDirectory prefixPath + prefixPath <- lift $ createPrefixDirectory "rcv.xftp" let relPrefixPath = takeFileName prefixPath relTmpPath = relPrefixPath "xftp.encrypted" relSavePath = relPrefixPath "xftp.decrypted" @@ -170,12 +169,12 @@ downloadChunk c FileChunk {replicas = (FileChunkReplica {server} : _)} = do lift . void $ getXFTPRcvWorker True c (Just server) downloadChunk _ _ = throwE $ INTERNAL "no replicas" -getPrefixPath :: String -> AM' FilePath -getPrefixPath suffix = do +createPrefixDirectory :: String -> AM' FilePath +createPrefixDirectory suffix = do workPath <- getXFTPWorkPath ts <- liftIO getCurrentTime let isoTime = formatTime defaultTimeLocale "%Y%m%d_%H%M%S_%6q" ts - uniqueCombine workPath (isoTime <> "_" <> suffix) + createUniqueDirectory workPath (isoTime <> "_" <> suffix) toFSFilePath :: FilePath -> AM' FilePath toFSFilePath f = ( f) <$> getXFTPWorkPath @@ -375,8 +374,7 @@ xftpSendFile' c userId file numRecipients storageHours = do xftpPrepareSendFile' :: AgentClient -> UserId -> CryptoFile -> Int -> Maybe Word32 -> AM SndFileId xftpPrepareSendFile' c userId file numRecipients storageHours = do g <- asks random - prefixPath <- lift $ getPrefixPath "snd.xftp" - createDirectory prefixPath + prefixPath <- lift $ createPrefixDirectory "snd.xftp" let relPrefixPath = takeFileName prefixPath key <- atomically $ C.randomSbKey g nonce <- atomically $ C.randomCbNonce g @@ -391,8 +389,7 @@ xftpSendDescription' c userId vfd numRecipients = do xftpPrepareSendDescription' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> AM SndFileId xftpPrepareSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) numRecipients = do g <- asks random - prefixPath <- lift $ getPrefixPath "snd.xftp" - createDirectory prefixPath + prefixPath <- lift $ createPrefixDirectory "snd.xftp" let relPrefixPath = takeFileName prefixPath let directYaml = prefixPath "direct.yaml" cfArgs <- atomically $ CF.randomArgs g diff --git a/src/Simplex/FileTransfer/Util.hs b/src/Simplex/FileTransfer/Util.hs index 0714f090c..661a2fd5c 100644 --- a/src/Simplex/FileTransfer/Util.hs +++ b/src/Simplex/FileTransfer/Util.hs @@ -1,5 +1,6 @@ module Simplex.FileTransfer.Util ( uniqueCombine, + createUniqueDirectory, safeFileNameStr, removePath, ) @@ -7,6 +8,7 @@ where import Simplex.Messaging.Util (ifM, whenM) import System.FilePath (makeValid, splitExtensions, takeFileName, ()) +import System.IO.Error (isAlreadyExistsError) import UnliftIO import UnliftIO.Directory @@ -20,11 +22,22 @@ uniqueCombine :: MonadIO m => FilePath -> String -> m FilePath uniqueCombine filePath fileName = tryCombine (0 :: Int) where tryCombine n = - let (name, ext) = splitExtensions $ safeFileNameStr fileName - suffix = if n == 0 then "" else "_" <> show n - f = filePath (name <> suffix <> ext) + let f = candidatePath filePath fileName n in ifM (doesPathExist f) (tryCombine $ n + 1) (pure f) +createUniqueDirectory :: MonadUnliftIO m => FilePath -> String -> m FilePath +createUniqueDirectory filePath dirName = tryCreate (0 :: Int) + where + tryCreate n = + let d = candidatePath filePath dirName n + in (d <$ createDirectory d) `catch` \e -> if isAlreadyExistsError e then tryCreate (n + 1) else throwIO e + +candidatePath :: FilePath -> String -> Int -> FilePath +candidatePath filePath fileName n = filePath (name <> suffix <> ext) + where + (name, ext) = splitExtensions $ safeFileNameStr fileName + suffix = if n == 0 then "" else "_" <> show n + removePath :: MonadIO m => FilePath -> m () removePath p = do ifM (doesDirectoryExist p) (removeDirectoryRecursive p) $ diff --git a/tests/XFTPAgent.hs b/tests/XFTPAgent.hs index 032f23bda..d76d6ad1b 100644 --- a/tests/XFTPAgent.hs +++ b/tests/XFTPAgent.hs @@ -57,8 +57,8 @@ import Simplex.Messaging.Protocol (BasicAuth, NetworkError (..), ProtoServerWith import Simplex.Messaging.Server.Expiration (ExpirationConfig (..)) import Simplex.Messaging.Server.Information (ServerPublicInfo) import Simplex.Messaging.Transport (EntitlementConfig (..)) -import Simplex.Messaging.Util (tshow) -import System.Directory (Permissions (writable), doesDirectoryExist, doesFileExist, getFileSize, getPermissions, listDirectory, removeFile, setPermissions) +import Simplex.Messaging.Util (tshow, unlessM) +import System.Directory (Permissions (writable), createDirectory, doesDirectoryExist, doesFileExist, getFileSize, getPermissions, listDirectory, removeFile, setPermissions) import System.FilePath (()) import Test.Hspec hiding (fit, it, xit) import UnliftIO @@ -82,6 +82,7 @@ xftpAgentTests = it "should send and receive small file without a redirect" testXFTPAgentSendReceiveNoRedirect it "should receive prepared file only after it is started" testXFTPAgentPrepareReceive it "should send prepared file only after it is started" testXFTPAgentPrepareSend + it "should prepare files concurrently in distinct work directories" $ \_ -> testXFTPAgentPrepareSendConcurrently it "should extend storage time with an entitlement proof and report the granted expiry" $ \_ -> testXFTPAgentEntitlement describe "sending and receiving with version negotiation" $ beforeWith (const (pure ())) testXFTPAgentSendReceiveMatrix it "should resume receiving file after restart" $ \_ -> testXFTPAgentReceiveRestore @@ -358,6 +359,29 @@ testXFTPAgentPrepareSend = withXFTPServer $ do withAgent 3 agentCfg initAgentServers testDB2 $ \rcp -> runRight_ . void $ testReceive rcp rfd filePath2 +testXFTPAgentPrepareSendConcurrently :: (HasCallStack, HasTestEnv) => IO () +testXFTPAgentPrepareSendConcurrently = do + filePath <- createRandomFile_ (kb 1 :: Integer) "testfile" + let workPath = testPath "xftp-work-files" + rounds = 100 + filesPerRound = 8 + prepareAtOnce sndr = do + ready <- newTVarIO 0 + started <- newTVarIO False + let waitStarted = unlessM (readTVarIO started) $ yield >> waitStarted + prepare = do + atomically $ modifyTVar' ready (+ 1) + waitStarted + runRight $ xftpPrepareSendFile sndr 1 (CF.plain filePath) 1 Nothing + concurrently_ + (replicateConcurrently_ filesPerRound prepare) + (atomically $ readTVar ready >>= checkSTM . (== filesPerRound) >> writeTVar started True) + createDirectory workPath + withAgent 1 agentCfg initAgentServers testDB $ \sndr -> do + runRight_ $ xftpStartWorkers sndr (Just workPath) + replicateM_ rounds $ prepareAtOnce sndr + length <$> listDirectory workPath `shouldReturn` rounds * filesPerRound + testXFTPAgentSendReceiveMatrix :: Spec testXFTPAgentSendReceiveMatrix = do describe "old server" $ do