mirror of
https://github.com/simplex-chat/simplexmq.git
synced 2026-11-22 19:58:39 +00:00
xftp: receive files with many chunks, always report receive errors (#2025)
* xftp: receive files with many chunks, always report receive errors * xftp: hash received file chunks one at a time
This commit is contained in:
1 parent
7ca1f4e508
commit
30d9808f16
5 files changed
+67
-7
No files matched your search
@@ -280,7 +280,8 @@ retryOnError name loop done e = do
|
||||
rcvWorkerInternalError :: AgentClient -> DBRcvFileId -> RcvFileId -> Maybe RcvFileId -> Maybe FilePath -> AgentErrorType -> AM ()
|
||||
rcvWorkerInternalError c rcvFileId rcvFileEntityId redirectEntityId_ tmpPath err = do
|
||||
liftIO $ throwWhenInactive c
|
||||
lift $ forM_ tmpPath (removePath <=< toFSFilePath)
|
||||
flip catchAllErrors (\e -> logError $ "XFTP rcv worker error: " <> tshow e) $
|
||||
lift $ forM_ tmpPath (removePath <=< toFSFilePath)
|
||||
withStore' c $ \db -> updateRcvFileError db rcvFileId (show err)
|
||||
notify c (fromMaybe rcvFileEntityId redirectEntityId_) (RFERR err)
|
||||
|
||||
@@ -307,7 +308,7 @@ runXFTPRcvLocalWorker c Worker {doWork} = do
|
||||
chunkPaths <- getChunkPaths chunks
|
||||
encSize <- liftIO $ foldM (\s path -> (s +) . fromIntegral <$> getFileSize path) 0 chunkPaths
|
||||
when (FileSize encSize /= size) $ throwE $ XFTP "" XFTP.SIZE
|
||||
encDigest <- liftIO $ LC.sha512Hash <$> readChunks chunkPaths
|
||||
encDigest <- liftIO $ sha512HashChunks chunkPaths
|
||||
when (FileDigest encDigest /= digest) $ throwE $ XFTP "" XFTP.DIGEST
|
||||
let destFile = CryptoFile fsSavePath cfArgs
|
||||
void $ liftError (FILE . FILE_IO . show) $ decryptChunks encSize chunkPaths key nonce $ \_ -> pure destFile
|
||||
|
||||
@@ -432,7 +432,7 @@ cliReceiveFile ReceiveOptions {fileDescription, filePath, retryCount, tempPath,
|
||||
(errs, rs) <- partitionEithers . concat <$> liftIO (pooledForConcurrentlyN 16 srvChunks $ mapM $ runExceptT . downloadFileChunk g a encPath size downloadedChunks)
|
||||
mapM_ throwE errs
|
||||
let chunkPaths = map snd $ sortOn fst rs
|
||||
encDigest <- liftIO $ LC.sha512Hash <$> readChunks chunkPaths
|
||||
encDigest <- liftIO $ sha512HashChunks chunkPaths
|
||||
when (encDigest /= unFileDigest digest) $ throwE $ CLIError "File digest mismatch"
|
||||
encSize <- liftIO $ foldM (\s path -> (s +) . fromIntegral <$> getFileSize path) 0 chunkPaths
|
||||
when (FileSize encSize /= size) $ throwE $ CLIError "File size mismatch"
|
||||
|
||||
@@ -7,12 +7,14 @@
|
||||
module Simplex.FileTransfer.Crypto
|
||||
( encryptFile,
|
||||
decryptChunks,
|
||||
readChunks,
|
||||
sha512HashChunks,
|
||||
) where
|
||||
|
||||
import Control.Monad
|
||||
import Control.Monad.Except
|
||||
import Control.Monad.Trans.Except
|
||||
import Crypto.Hash (Context, hashFinalize, hashInit, hashUpdate)
|
||||
import Crypto.Hash.Algorithms (SHA512)
|
||||
import qualified Data.Attoparsec.ByteString.Char8 as A
|
||||
import Data.Bifunctor (first)
|
||||
import qualified Data.ByteArray as BA
|
||||
@@ -115,5 +117,10 @@ decryptChunks encSize (chPath : chPaths) key nonce getDestFile = case reverse ch
|
||||
A.Partial _ -> throwE $ FTCEInvalidHeader "incomplete"
|
||||
A.Done rest hdr -> pure (hdr, LB.fromStrict rest <> s')
|
||||
|
||||
readChunks :: [FilePath] -> IO LB.ByteString
|
||||
readChunks = foldM (\s path -> (s <>) <$> LB.readFile path) ""
|
||||
sha512HashChunks :: [FilePath] -> IO ByteString
|
||||
sha512HashChunks = fmap (BA.convert . hashFinalize) . foldM hashChunk (hashInit :: Context SHA512)
|
||||
where
|
||||
hashChunk ctx path = withFile path ReadMode $ hashHandle ctx
|
||||
hashHandle !ctx h = do
|
||||
s <- B.hGet h 65536
|
||||
if B.null s then pure ctx else hashHandle (hashUpdate ctx s) h
|
||||
@@ -7,14 +7,18 @@
|
||||
module FileDescriptionTests where
|
||||
|
||||
import Control.Exception (bracket_)
|
||||
import Control.Monad (zipWithM_)
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import qualified Data.ByteString.Lazy.Char8 as LB
|
||||
import Data.Either (isRight)
|
||||
import qualified Data.Yaml as Y
|
||||
import Simplex.FileTransfer.Chunks (chunkSize3, serverChunkSizes)
|
||||
import Simplex.FileTransfer.Client (prepareChunkSizes)
|
||||
import Simplex.FileTransfer.Crypto (sha512HashChunks)
|
||||
import Simplex.FileTransfer.Description
|
||||
import Simplex.FileTransfer.Protocol
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import qualified Simplex.Messaging.Crypto.Lazy as LC
|
||||
import Simplex.Messaging.Encoding.String (StrEncoding (..))
|
||||
import Simplex.Messaging.Protocol (EntityId (..))
|
||||
import Simplex.Messaging.ServiceScheme (ServiceScheme (..))
|
||||
@@ -33,6 +37,7 @@ fileDescriptionTests = do
|
||||
it "accept file description with a number at the length limit" testAcceptMaxNumber
|
||||
it "limit chunk sizes and replicas per chunk and per file" testFileDescriptionLimits
|
||||
it "serialize file description" testSerializeFileDescription
|
||||
it "hash chunk files one at a time" testHashChunks
|
||||
describe "file description URIs" $ do
|
||||
it "round trip file description URI" testFileDescriptionURI
|
||||
it "round trip file description URI with extra JSON" testFileDescriptionURIExtras
|
||||
@@ -236,6 +241,15 @@ testFileDescriptionURIExtras = do
|
||||
let descr = FileDescriptionURI SSSimplex vfd $ Just "{\"something\":\"extra\",\"more\":true}"
|
||||
strDecode (strEncode descr) `shouldBe` Right descr
|
||||
|
||||
testHashChunks :: HasTestEnv => IO ()
|
||||
testHashChunks = do
|
||||
let sizes = [0, 65535, 65536, 100000, 200000]
|
||||
paths = map (testPath . ("chunk" <>) . show) [1 .. length sizes]
|
||||
chunks = zipWith B.replicate sizes ['a' ..]
|
||||
zipWithM_ B.writeFile paths chunks
|
||||
digest <- sha512HashChunks paths
|
||||
digest `shouldBe` LC.sha512Hash (LB.fromStrict $ B.concat chunks)
|
||||
|
||||
withRemoveTmpFile :: HasTestEnv => IO () -> IO ()
|
||||
withRemoveTmpFile =
|
||||
bracket_
|
||||
|
||||
+39
-1
@@ -58,7 +58,7 @@ import Simplex.Messaging.Server.Expiration (ExpirationConfig (..))
|
||||
import Simplex.Messaging.Server.Information (ServerPublicInfo)
|
||||
import Simplex.Messaging.Transport (EntitlementConfig (..))
|
||||
import Simplex.Messaging.Util (tshow)
|
||||
import System.Directory (doesDirectoryExist, doesFileExist, getFileSize, listDirectory, removeFile)
|
||||
import System.Directory (Permissions (writable), doesDirectoryExist, doesFileExist, getFileSize, getPermissions, listDirectory, removeFile, setPermissions)
|
||||
import System.FilePath ((</>))
|
||||
import Test.Hspec hiding (fit, it, xit)
|
||||
import UnliftIO
|
||||
@@ -86,6 +86,7 @@ xftpAgentTests =
|
||||
describe "sending and receiving with version negotiation" $ beforeWith (const (pure ())) testXFTPAgentSendReceiveMatrix
|
||||
it "should resume receiving file after restart" $ \_ -> testXFTPAgentReceiveRestore
|
||||
it "should cleanup rcv tmp path after permanent error" $ \_ -> testXFTPAgentReceiveCleanup
|
||||
it "should report permanent error even when rcv tmp path cannot be removed" $ \_ -> testXFTPAgentReceiveCleanupError
|
||||
it "should resume sending file after restart" $ \_ -> testXFTPAgentSendRestore
|
||||
xit'' "should cleanup snd prefix path after permanent error" $ \_ -> testXFTPAgentSendCleanup
|
||||
it "should delete sent file on server" testXFTPAgentDelete
|
||||
@@ -559,6 +560,43 @@ testXFTPAgentReceiveCleanup = withGlobalLogging logCfgNoLogs $ do
|
||||
-- tmp path should be removed after permanent error
|
||||
doesDirectoryExist tmpPath `shouldReturn` False
|
||||
|
||||
testXFTPAgentReceiveCleanupError :: (HasCallStack, HasTestEnv) => IO ()
|
||||
testXFTPAgentReceiveCleanupError = withGlobalLogging logCfgNoLogs $ do
|
||||
filePath <- createRandomFile
|
||||
|
||||
rfd <- withXFTPServerStoreLogOn $ \_ ->
|
||||
withAgent 1 agentCfg initAgentServers testDB $ \sndr -> runRight $ do
|
||||
(_, _, rfd, _) <- testSend sndr filePath
|
||||
pure rfd
|
||||
|
||||
rfId <- withAgent 2 agentCfg initAgentServers testDB2 $ \rcp -> runRight $ do
|
||||
xftpStartWorkers rcp (Just recipientFiles)
|
||||
rfId <- xftpReceiveFile rcp 1 rfd Nothing True
|
||||
liftIO $ timeout 300000 (get rcp) `shouldReturn` Nothing
|
||||
pure rfId
|
||||
|
||||
[prefixDir] <- listDirectory recipientFiles
|
||||
let prefixPath = recipientFiles </> prefixDir
|
||||
tmpPath = prefixPath </> "xftp.encrypted"
|
||||
doesDirectoryExist tmpPath `shouldReturn` True
|
||||
|
||||
perms <- getPermissions prefixPath
|
||||
setPermissions prefixPath perms {writable = False}
|
||||
let probe = prefixPath </> ".probe"
|
||||
removalBlocked <- (writeFile probe "" >> removeFile probe >> pure False) `catchAny` \_ -> pure True
|
||||
if not removalBlocked
|
||||
then setPermissions prefixPath perms >> pendingWith "root may remove files from a read-only directory"
|
||||
else do
|
||||
( withXFTPServerThreadOn $ \_ ->
|
||||
withAgent 3 agentCfg initAgentServers testDB2 $ \rcp' -> do
|
||||
runRight_ $ xftpStartWorkers rcp' (Just recipientFiles)
|
||||
("", rfId', RFERR (XFTP srv AUTH)) <- rfGet rcp'
|
||||
rfId' `shouldBe` rfId
|
||||
srv `shouldBe` testXFTPServerStr
|
||||
)
|
||||
`finally` setPermissions prefixPath perms
|
||||
doesDirectoryExist tmpPath `shouldReturn` True
|
||||
|
||||
testXFTPAgentSendRestore :: (HasCallStack, HasTestEnv) => IO ()
|
||||
testXFTPAgentSendRestore = withGlobalLogging logCfgNoLogs $ do
|
||||
filePath <- createRandomFile
|
||||
|
||||
Reference in new issue
Block a user