xftp: receive files with many chunks, always report receive errors (#2025)

* xftp: receive files with many chunks, always report receive errors

* xftp: hash received file chunks one at a time
This commit is contained in:
Evgeny @ SimpleX Chat authored and GitHub committed 2026-11-07 15:57:09 +00:00
1 parent 7ca1f4e508
commit 30d9808f16
5 files changed
+67 -7

No files matched your search

+3 -2
View File
@@ -280,7 +280,8 @@ retryOnError name loop done e = do
rcvWorkerInternalError :: AgentClient -> DBRcvFileId -> RcvFileId -> Maybe RcvFileId -> Maybe FilePath -> AgentErrorType -> AM ()
rcvWorkerInternalError c rcvFileId rcvFileEntityId redirectEntityId_ tmpPath err = do
liftIO $ throwWhenInactive c
lift $ forM_ tmpPath (removePath <=< toFSFilePath)
flip catchAllErrors (\e -> logError $ "XFTP rcv worker error: " <> tshow e) $
lift $ forM_ tmpPath (removePath <=< toFSFilePath)
withStore' c $ \db -> updateRcvFileError db rcvFileId (show err)
notify c (fromMaybe rcvFileEntityId redirectEntityId_) (RFERR err)
@@ -307,7 +308,7 @@ runXFTPRcvLocalWorker c Worker {doWork} = do
chunkPaths <- getChunkPaths chunks
encSize <- liftIO $ foldM (\s path -> (s +) . fromIntegral <$> getFileSize path) 0 chunkPaths
when (FileSize encSize /= size) $ throwE $ XFTP "" XFTP.SIZE
encDigest <- liftIO $ LC.sha512Hash <$> readChunks chunkPaths
encDigest <- liftIO $ sha512HashChunks chunkPaths
when (FileDigest encDigest /= digest) $ throwE $ XFTP "" XFTP.DIGEST
let destFile = CryptoFile fsSavePath cfArgs
void $ liftError (FILE . FILE_IO . show) $ decryptChunks encSize chunkPaths key nonce $ \_ -> pure destFile
+1 -1
View File
@@ -432,7 +432,7 @@ cliReceiveFile ReceiveOptions {fileDescription, filePath, retryCount, tempPath,
(errs, rs) <- partitionEithers . concat <$> liftIO (pooledForConcurrentlyN 16 srvChunks $ mapM $ runExceptT . downloadFileChunk g a encPath size downloadedChunks)
mapM_ throwE errs
let chunkPaths = map snd $ sortOn fst rs
encDigest <- liftIO $ LC.sha512Hash <$> readChunks chunkPaths
encDigest <- liftIO $ sha512HashChunks chunkPaths
when (encDigest /= unFileDigest digest) $ throwE $ CLIError "File digest mismatch"
encSize <- liftIO $ foldM (\s path -> (s +) . fromIntegral <$> getFileSize path) 0 chunkPaths
when (FileSize encSize /= size) $ throwE $ CLIError "File size mismatch"
+10 -3
View File
@@ -7,12 +7,14 @@
module Simplex.FileTransfer.Crypto
( encryptFile,
decryptChunks,
readChunks,
sha512HashChunks,
) where
import Control.Monad
import Control.Monad.Except
import Control.Monad.Trans.Except
import Crypto.Hash (Context, hashFinalize, hashInit, hashUpdate)
import Crypto.Hash.Algorithms (SHA512)
import qualified Data.Attoparsec.ByteString.Char8 as A
import Data.Bifunctor (first)
import qualified Data.ByteArray as BA
@@ -115,5 +117,10 @@ decryptChunks encSize (chPath : chPaths) key nonce getDestFile = case reverse ch
A.Partial _ -> throwE $ FTCEInvalidHeader "incomplete"
A.Done rest hdr -> pure (hdr, LB.fromStrict rest <> s')
readChunks :: [FilePath] -> IO LB.ByteString
readChunks = foldM (\s path -> (s <>) <$> LB.readFile path) ""
sha512HashChunks :: [FilePath] -> IO ByteString
sha512HashChunks = fmap (BA.convert . hashFinalize) . foldM hashChunk (hashInit :: Context SHA512)
where
hashChunk ctx path = withFile path ReadMode $ hashHandle ctx
hashHandle !ctx h = do
s <- B.hGet h 65536
if B.null s then pure ctx else hashHandle (hashUpdate ctx s) h
+14
View File
@@ -7,14 +7,18 @@
module FileDescriptionTests where
import Control.Exception (bracket_)
import Control.Monad (zipWithM_)
import qualified Data.ByteString.Char8 as B
import qualified Data.ByteString.Lazy.Char8 as LB
import Data.Either (isRight)
import qualified Data.Yaml as Y
import Simplex.FileTransfer.Chunks (chunkSize3, serverChunkSizes)
import Simplex.FileTransfer.Client (prepareChunkSizes)
import Simplex.FileTransfer.Crypto (sha512HashChunks)
import Simplex.FileTransfer.Description
import Simplex.FileTransfer.Protocol
import qualified Simplex.Messaging.Crypto as C
import qualified Simplex.Messaging.Crypto.Lazy as LC
import Simplex.Messaging.Encoding.String (StrEncoding (..))
import Simplex.Messaging.Protocol (EntityId (..))
import Simplex.Messaging.ServiceScheme (ServiceScheme (..))
@@ -33,6 +37,7 @@ fileDescriptionTests = do
it "accept file description with a number at the length limit" testAcceptMaxNumber
it "limit chunk sizes and replicas per chunk and per file" testFileDescriptionLimits
it "serialize file description" testSerializeFileDescription
it "hash chunk files one at a time" testHashChunks
describe "file description URIs" $ do
it "round trip file description URI" testFileDescriptionURI
it "round trip file description URI with extra JSON" testFileDescriptionURIExtras
@@ -236,6 +241,15 @@ testFileDescriptionURIExtras = do
let descr = FileDescriptionURI SSSimplex vfd $ Just "{\"something\":\"extra\",\"more\":true}"
strDecode (strEncode descr) `shouldBe` Right descr
testHashChunks :: HasTestEnv => IO ()
testHashChunks = do
let sizes = [0, 65535, 65536, 100000, 200000]
paths = map (testPath . ("chunk" <>) . show) [1 .. length sizes]
chunks = zipWith B.replicate sizes ['a' ..]
zipWithM_ B.writeFile paths chunks
digest <- sha512HashChunks paths
digest `shouldBe` LC.sha512Hash (LB.fromStrict $ B.concat chunks)
withRemoveTmpFile :: HasTestEnv => IO () -> IO ()
withRemoveTmpFile =
bracket_
+39 -1
View File
@@ -58,7 +58,7 @@ import Simplex.Messaging.Server.Expiration (ExpirationConfig (..))
import Simplex.Messaging.Server.Information (ServerPublicInfo)
import Simplex.Messaging.Transport (EntitlementConfig (..))
import Simplex.Messaging.Util (tshow)
import System.Directory (doesDirectoryExist, doesFileExist, getFileSize, listDirectory, removeFile)
import System.Directory (Permissions (writable), doesDirectoryExist, doesFileExist, getFileSize, getPermissions, listDirectory, removeFile, setPermissions)
import System.FilePath ((</>))
import Test.Hspec hiding (fit, it, xit)
import UnliftIO
@@ -86,6 +86,7 @@ xftpAgentTests =
describe "sending and receiving with version negotiation" $ beforeWith (const (pure ())) testXFTPAgentSendReceiveMatrix
it "should resume receiving file after restart" $ \_ -> testXFTPAgentReceiveRestore
it "should cleanup rcv tmp path after permanent error" $ \_ -> testXFTPAgentReceiveCleanup
it "should report permanent error even when rcv tmp path cannot be removed" $ \_ -> testXFTPAgentReceiveCleanupError
it "should resume sending file after restart" $ \_ -> testXFTPAgentSendRestore
xit'' "should cleanup snd prefix path after permanent error" $ \_ -> testXFTPAgentSendCleanup
it "should delete sent file on server" testXFTPAgentDelete
@@ -559,6 +560,43 @@ testXFTPAgentReceiveCleanup = withGlobalLogging logCfgNoLogs $ do
-- tmp path should be removed after permanent error
doesDirectoryExist tmpPath `shouldReturn` False
testXFTPAgentReceiveCleanupError :: (HasCallStack, HasTestEnv) => IO ()
testXFTPAgentReceiveCleanupError = withGlobalLogging logCfgNoLogs $ do
filePath <- createRandomFile
rfd <- withXFTPServerStoreLogOn $ \_ ->
withAgent 1 agentCfg initAgentServers testDB $ \sndr -> runRight $ do
(_, _, rfd, _) <- testSend sndr filePath
pure rfd
rfId <- withAgent 2 agentCfg initAgentServers testDB2 $ \rcp -> runRight $ do
xftpStartWorkers rcp (Just recipientFiles)
rfId <- xftpReceiveFile rcp 1 rfd Nothing True
liftIO $ timeout 300000 (get rcp) `shouldReturn` Nothing
pure rfId
[prefixDir] <- listDirectory recipientFiles
let prefixPath = recipientFiles </> prefixDir
tmpPath = prefixPath </> "xftp.encrypted"
doesDirectoryExist tmpPath `shouldReturn` True
perms <- getPermissions prefixPath
setPermissions prefixPath perms {writable = False}
let probe = prefixPath </> ".probe"
removalBlocked <- (writeFile probe "" >> removeFile probe >> pure False) `catchAny` \_ -> pure True
if not removalBlocked
then setPermissions prefixPath perms >> pendingWith "root may remove files from a read-only directory"
else do
( withXFTPServerThreadOn $ \_ ->
withAgent 3 agentCfg initAgentServers testDB2 $ \rcp' -> do
runRight_ $ xftpStartWorkers rcp' (Just recipientFiles)
("", rfId', RFERR (XFTP srv AUTH)) <- rfGet rcp'
rfId' `shouldBe` rfId
srv `shouldBe` testXFTPServerStr
)
`finally` setPermissions prefixPath perms
doesDirectoryExist tmpPath `shouldReturn` True
testXFTPAgentSendRestore :: (HasCallStack, HasTestEnv) => IO ()
testXFTPAgentSendRestore = withGlobalLogging logCfgNoLogs $ do
filePath <- createRandomFile