diff --git a/src/Simplex/FileTransfer/Agent.hs b/src/Simplex/FileTransfer/Agent.hs index f3976c47a..095e0b4db 100644 --- a/src/Simplex/FileTransfer/Agent.hs +++ b/src/Simplex/FileTransfer/Agent.hs @@ -280,7 +280,8 @@ retryOnError name loop done e = do rcvWorkerInternalError :: AgentClient -> DBRcvFileId -> RcvFileId -> Maybe RcvFileId -> Maybe FilePath -> AgentErrorType -> AM () rcvWorkerInternalError c rcvFileId rcvFileEntityId redirectEntityId_ tmpPath err = do liftIO $ throwWhenInactive c - lift $ forM_ tmpPath (removePath <=< toFSFilePath) + flip catchAllErrors (\e -> logError $ "XFTP rcv worker error: " <> tshow e) $ + lift $ forM_ tmpPath (removePath <=< toFSFilePath) withStore' c $ \db -> updateRcvFileError db rcvFileId (show err) notify c (fromMaybe rcvFileEntityId redirectEntityId_) (RFERR err) @@ -307,7 +308,7 @@ runXFTPRcvLocalWorker c Worker {doWork} = do chunkPaths <- getChunkPaths chunks encSize <- liftIO $ foldM (\s path -> (s +) . fromIntegral <$> getFileSize path) 0 chunkPaths when (FileSize encSize /= size) $ throwE $ XFTP "" XFTP.SIZE - encDigest <- liftIO $ LC.sha512Hash <$> readChunks chunkPaths + encDigest <- liftIO $ sha512HashChunks chunkPaths when (FileDigest encDigest /= digest) $ throwE $ XFTP "" XFTP.DIGEST let destFile = CryptoFile fsSavePath cfArgs void $ liftError (FILE . FILE_IO . show) $ decryptChunks encSize chunkPaths key nonce $ \_ -> pure destFile diff --git a/src/Simplex/FileTransfer/Client/Main.hs b/src/Simplex/FileTransfer/Client/Main.hs index fb1ce452d..4049cde56 100644 --- a/src/Simplex/FileTransfer/Client/Main.hs +++ b/src/Simplex/FileTransfer/Client/Main.hs @@ -432,7 +432,7 @@ cliReceiveFile ReceiveOptions {fileDescription, filePath, retryCount, tempPath, (errs, rs) <- partitionEithers . concat <$> liftIO (pooledForConcurrentlyN 16 srvChunks $ mapM $ runExceptT . downloadFileChunk g a encPath size downloadedChunks) mapM_ throwE errs let chunkPaths = map snd $ sortOn fst rs - encDigest <- liftIO $ LC.sha512Hash <$> readChunks chunkPaths + encDigest <- liftIO $ sha512HashChunks chunkPaths when (encDigest /= unFileDigest digest) $ throwE $ CLIError "File digest mismatch" encSize <- liftIO $ foldM (\s path -> (s +) . fromIntegral <$> getFileSize path) 0 chunkPaths when (FileSize encSize /= size) $ throwE $ CLIError "File size mismatch" diff --git a/src/Simplex/FileTransfer/Crypto.hs b/src/Simplex/FileTransfer/Crypto.hs index 7014ff4b9..dd0b2d9a3 100644 --- a/src/Simplex/FileTransfer/Crypto.hs +++ b/src/Simplex/FileTransfer/Crypto.hs @@ -7,12 +7,14 @@ module Simplex.FileTransfer.Crypto ( encryptFile, decryptChunks, - readChunks, + sha512HashChunks, ) where import Control.Monad import Control.Monad.Except import Control.Monad.Trans.Except +import Crypto.Hash (Context, hashFinalize, hashInit, hashUpdate) +import Crypto.Hash.Algorithms (SHA512) import qualified Data.Attoparsec.ByteString.Char8 as A import Data.Bifunctor (first) import qualified Data.ByteArray as BA @@ -115,5 +117,10 @@ decryptChunks encSize (chPath : chPaths) key nonce getDestFile = case reverse ch A.Partial _ -> throwE $ FTCEInvalidHeader "incomplete" A.Done rest hdr -> pure (hdr, LB.fromStrict rest <> s') -readChunks :: [FilePath] -> IO LB.ByteString -readChunks = foldM (\s path -> (s <>) <$> LB.readFile path) "" +sha512HashChunks :: [FilePath] -> IO ByteString +sha512HashChunks = fmap (BA.convert . hashFinalize) . foldM hashChunk (hashInit :: Context SHA512) + where + hashChunk ctx path = withFile path ReadMode $ hashHandle ctx + hashHandle !ctx h = do + s <- B.hGet h 65536 + if B.null s then pure ctx else hashHandle (hashUpdate ctx s) h diff --git a/tests/FileDescriptionTests.hs b/tests/FileDescriptionTests.hs index 83b86abc1..519d14496 100644 --- a/tests/FileDescriptionTests.hs +++ b/tests/FileDescriptionTests.hs @@ -7,14 +7,18 @@ module FileDescriptionTests where import Control.Exception (bracket_) +import Control.Monad (zipWithM_) import qualified Data.ByteString.Char8 as B +import qualified Data.ByteString.Lazy.Char8 as LB import Data.Either (isRight) import qualified Data.Yaml as Y import Simplex.FileTransfer.Chunks (chunkSize3, serverChunkSizes) import Simplex.FileTransfer.Client (prepareChunkSizes) +import Simplex.FileTransfer.Crypto (sha512HashChunks) import Simplex.FileTransfer.Description import Simplex.FileTransfer.Protocol import qualified Simplex.Messaging.Crypto as C +import qualified Simplex.Messaging.Crypto.Lazy as LC import Simplex.Messaging.Encoding.String (StrEncoding (..)) import Simplex.Messaging.Protocol (EntityId (..)) import Simplex.Messaging.ServiceScheme (ServiceScheme (..)) @@ -33,6 +37,7 @@ fileDescriptionTests = do it "accept file description with a number at the length limit" testAcceptMaxNumber it "limit chunk sizes and replicas per chunk and per file" testFileDescriptionLimits it "serialize file description" testSerializeFileDescription + it "hash chunk files one at a time" testHashChunks describe "file description URIs" $ do it "round trip file description URI" testFileDescriptionURI it "round trip file description URI with extra JSON" testFileDescriptionURIExtras @@ -236,6 +241,15 @@ testFileDescriptionURIExtras = do let descr = FileDescriptionURI SSSimplex vfd $ Just "{\"something\":\"extra\",\"more\":true}" strDecode (strEncode descr) `shouldBe` Right descr +testHashChunks :: HasTestEnv => IO () +testHashChunks = do + let sizes = [0, 65535, 65536, 100000, 200000] + paths = map (testPath . ("chunk" <>) . show) [1 .. length sizes] + chunks = zipWith B.replicate sizes ['a' ..] + zipWithM_ B.writeFile paths chunks + digest <- sha512HashChunks paths + digest `shouldBe` LC.sha512Hash (LB.fromStrict $ B.concat chunks) + withRemoveTmpFile :: HasTestEnv => IO () -> IO () withRemoveTmpFile = bracket_ diff --git a/tests/XFTPAgent.hs b/tests/XFTPAgent.hs index a74bbe444..032f23bda 100644 --- a/tests/XFTPAgent.hs +++ b/tests/XFTPAgent.hs @@ -58,7 +58,7 @@ import Simplex.Messaging.Server.Expiration (ExpirationConfig (..)) import Simplex.Messaging.Server.Information (ServerPublicInfo) import Simplex.Messaging.Transport (EntitlementConfig (..)) import Simplex.Messaging.Util (tshow) -import System.Directory (doesDirectoryExist, doesFileExist, getFileSize, listDirectory, removeFile) +import System.Directory (Permissions (writable), doesDirectoryExist, doesFileExist, getFileSize, getPermissions, listDirectory, removeFile, setPermissions) import System.FilePath (()) import Test.Hspec hiding (fit, it, xit) import UnliftIO @@ -86,6 +86,7 @@ xftpAgentTests = describe "sending and receiving with version negotiation" $ beforeWith (const (pure ())) testXFTPAgentSendReceiveMatrix it "should resume receiving file after restart" $ \_ -> testXFTPAgentReceiveRestore it "should cleanup rcv tmp path after permanent error" $ \_ -> testXFTPAgentReceiveCleanup + it "should report permanent error even when rcv tmp path cannot be removed" $ \_ -> testXFTPAgentReceiveCleanupError it "should resume sending file after restart" $ \_ -> testXFTPAgentSendRestore xit'' "should cleanup snd prefix path after permanent error" $ \_ -> testXFTPAgentSendCleanup it "should delete sent file on server" testXFTPAgentDelete @@ -559,6 +560,43 @@ testXFTPAgentReceiveCleanup = withGlobalLogging logCfgNoLogs $ do -- tmp path should be removed after permanent error doesDirectoryExist tmpPath `shouldReturn` False +testXFTPAgentReceiveCleanupError :: (HasCallStack, HasTestEnv) => IO () +testXFTPAgentReceiveCleanupError = withGlobalLogging logCfgNoLogs $ do + filePath <- createRandomFile + + rfd <- withXFTPServerStoreLogOn $ \_ -> + withAgent 1 agentCfg initAgentServers testDB $ \sndr -> runRight $ do + (_, _, rfd, _) <- testSend sndr filePath + pure rfd + + rfId <- withAgent 2 agentCfg initAgentServers testDB2 $ \rcp -> runRight $ do + xftpStartWorkers rcp (Just recipientFiles) + rfId <- xftpReceiveFile rcp 1 rfd Nothing True + liftIO $ timeout 300000 (get rcp) `shouldReturn` Nothing + pure rfId + + [prefixDir] <- listDirectory recipientFiles + let prefixPath = recipientFiles prefixDir + tmpPath = prefixPath "xftp.encrypted" + doesDirectoryExist tmpPath `shouldReturn` True + + perms <- getPermissions prefixPath + setPermissions prefixPath perms {writable = False} + let probe = prefixPath ".probe" + removalBlocked <- (writeFile probe "" >> removeFile probe >> pure False) `catchAny` \_ -> pure True + if not removalBlocked + then setPermissions prefixPath perms >> pendingWith "root may remove files from a read-only directory" + else do + ( withXFTPServerThreadOn $ \_ -> + withAgent 3 agentCfg initAgentServers testDB2 $ \rcp' -> do + runRight_ $ xftpStartWorkers rcp' (Just recipientFiles) + ("", rfId', RFERR (XFTP srv AUTH)) <- rfGet rcp' + rfId' `shouldBe` rfId + srv `shouldBe` testXFTPServerStr + ) + `finally` setPermissions prefixPath perms + doesDirectoryExist tmpPath `shouldReturn` True + testXFTPAgentSendRestore :: (HasCallStack, HasTestEnv) => IO () testXFTPAgentSendRestore = withGlobalLogging logCfgNoLogs $ do filePath <- createRandomFile