Files
simplexmq/tests/FileDescriptionTests.hs
T
Evgeny @ SimpleX Chat 239290ca98 xftp: limit received files to the size of the sent file (#2067)
* tests: decrypt chunk files into the original file

* xftp: limit received files to the size of the sent file
2026-11-13 20:43:43 +00:00

288 lines
12 KiB
Haskell

{-# LANGUAGE DataKinds #-}
{-# LANGUAGE DuplicateRecordFields #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE ScopedTypeVariables #-}
module FileDescriptionTests where
import Control.Exception (bracket_)
import Control.Monad (forM_, zipWithM_)
import Control.Monad.Trans.Except (runExceptT)
import qualified Data.ByteString.Char8 as B
import qualified Data.ByteString.Lazy.Char8 as LB
import Data.Either (isRight)
import qualified Data.Yaml as Y
import Simplex.FileTransfer.Chunks (chunkSize0, chunkSize3, serverChunkSizes)
import Simplex.FileTransfer.Client (prepareChunkSizes)
import Simplex.FileTransfer.Crypto (decryptChunks, encryptFile, sha512HashChunks)
import Simplex.FileTransfer.Description
import Simplex.FileTransfer.Protocol
import Simplex.FileTransfer.Types (FileHeader (..), authTagSize)
import qualified Simplex.Messaging.Crypto as C
import qualified Simplex.Messaging.Crypto.File as CF
import qualified Simplex.Messaging.Crypto.Lazy as LC
import Simplex.Messaging.Encoding (smpEncode)
import Simplex.Messaging.Encoding.String (StrEncoding (..))
import Simplex.Messaging.Protocol (EntityId (..))
import Simplex.Messaging.ServiceScheme (ServiceScheme (..))
import System.Directory (getFileSize, removeFile)
import System.IO (IOMode (..), withFile)
import Test.Hspec hiding (fit, it)
import Util
fileDescriptionTests :: Spec
fileDescriptionTests = do
describe "file description parsing / serializing" $ do
it "parse YAML file description" testParseYAMLFileDescription
it "serialize YAML file description" testSerializeYAMLFileDescription
it "parse file description" testParseFileDescription
it "reject file description with YAML aliases" testRejectYAMLAliases
it "reject file description with a long number" testRejectLongNumber
it "accept file description with a number at the length limit" testAcceptMaxNumber
it "limit chunk sizes, replicas and redirect description size" testFileDescriptionLimits
it "serialize file description" testSerializeFileDescription
it "hash chunk files one at a time" testHashChunks
it "decrypt chunk files into the original file" testDecryptChunks
describe "file description URIs" $ do
it "round trip file description URI" testFileDescriptionURI
it "round trip file description URI with extra JSON" testFileDescriptionURIExtras
fileDescPath :: FilePath
fileDescPath = "tests/fixtures/file_description.yaml"
tmpFileDescPath :: HasTestEnv => FilePath
tmpFileDescPath = testPath "file_description.yaml"
testSbKey :: C.SbKey
testSbKey = either error id $ strDecode "00n8p1tJq5E-SGnHcYTOrS4A9I07gTA_WFD6MTFFFOY="
testCbNonce :: C.CbNonce
testCbNonce = either error id $ strDecode "dPSF-wrQpDiK_K6sYv0BDBZ9S4dg-jmu"
fileDesc :: FileDescription 'FRecipient
fileDesc =
FileDescription
{ party = SFRecipient,
size = FileSize $ mb 13,
digest = FileDigest "abc",
key = testSbKey,
nonce = testCbNonce,
chunkSize = defaultChunkSize,
chunks =
[ FileChunk
{ chunkNo = 1,
digest = chunkDigest,
chunkSize = defaultChunkSize,
replicas =
[ FileChunkReplica {server = "xftp://abc=@example1.com", replicaId, replicaKey},
FileChunkReplica {server = "xftp://abc=@example3.com", replicaId, replicaKey}
]
},
FileChunk
{ chunkNo = 2,
digest = chunkDigest,
chunkSize = defaultChunkSize,
replicas =
[ FileChunkReplica {server = "xftp://abc=@example2.com", replicaId, replicaKey},
FileChunkReplica {server = "xftp://abc=@example4.com", replicaId, replicaKey}
]
},
FileChunk
{ chunkNo = 3,
digest = chunkDigest,
chunkSize = defaultChunkSize,
replicas =
[ FileChunkReplica {server = "xftp://abc=@example1.com", replicaId, replicaKey},
FileChunkReplica {server = "xftp://abc=@example4.com", replicaId, replicaKey}
]
},
FileChunk
{ chunkNo = 4,
digest = chunkDigest,
chunkSize = FileSize $ mb 1,
replicas =
[ FileChunkReplica {server = "xftp://abc=@example2.com", replicaId, replicaKey},
FileChunkReplica {server = "xftp://abc=@example3.com", replicaId, replicaKey}
]
}
],
redirect = Nothing
}
where
defaultChunkSize = FileSize $ mb 4
replicaId = ChunkReplicaId $ EntityId "abc"
replicaKey = C.APrivateAuthKey C.SEd25519 "MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe"
chunkDigest = FileDigest "ghi"
yamlFileDesc :: YAMLFileDescription
yamlFileDesc =
YAMLFileDescription
{ party = FRecipient,
size = "13mb",
chunkSize = "4mb",
digest = FileDigest "abc",
key = testSbKey,
nonce = testCbNonce,
replicas =
[ YAMLServerReplicas
{ server = "xftp://abc=@example1.com",
chunks =
[ "1:YWJj:MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe:Z2hp",
"3:YWJj:MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe:Z2hp"
]
},
YAMLServerReplicas
{ server = "xftp://abc=@example2.com",
chunks =
[ "2:YWJj:MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe:Z2hp",
"4:YWJj:MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe:Z2hp:1mb"
]
},
YAMLServerReplicas
{ server = "xftp://abc=@example3.com",
chunks =
[ "1:YWJj:MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe",
"4:YWJj:MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe"
]
},
YAMLServerReplicas
{ server = "xftp://abc=@example4.com",
chunks =
[ "2:YWJj:MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe",
"3:YWJj:MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe"
]
}
],
redirect = Nothing
}
testParseYAMLFileDescription :: IO ()
testParseYAMLFileDescription = do
yfd <- Y.decodeFileThrow fileDescPath
yfd `shouldBe` yamlFileDesc
testSerializeYAMLFileDescription :: HasTestEnv => IO ()
testSerializeYAMLFileDescription = withRemoveTmpFile $ do
Y.encodeFile tmpFileDescPath yamlFileDesc
fdSer <- B.readFile tmpFileDescPath
fdExp <- B.readFile fileDescPath
fdSer `shouldBe` fdExp
testParseFileDescription :: IO ()
testParseFileDescription = do
r <- strDecode <$> B.readFile fileDescPath
case r of
Left e -> expectationFailure $ show e
Right fd -> fd `shouldBe` fileDesc
testRejectYAMLAliases :: IO ()
testRejectYAMLAliases = do
(yaml, sizeLine) <- B.breakSubstring "size: " <$> B.readFile fileDescPath
let (header, groups) = B.breakSubstring "- chunks:" yaml
aliased = header <> "- &g\n " <> B.drop 2 groups <> "- *g\n" <> sizeLine
(strDecode aliased :: Either String (FileDescription 'FRecipient)) `shouldBe` Left "YAML aliases are not allowed"
testRejectLongNumber :: IO ()
testRejectLongNumber = do
fd <- B.readFile fileDescPath
let withLongNumber = fd <> "x: " <> B.replicate 65 '7' <> "\n"
(strDecode withLongNumber :: Either String (FileDescription 'FRecipient))
`shouldBe` Left "number too long in file description"
testAcceptMaxNumber :: IO ()
testAcceptMaxNumber = do
fd <- B.readFile fileDescPath
let withMaxNumber = fd <> "x: " <> B.replicate 64 '7' <> "\n"
(strDecode withMaxNumber :: Either String (FileDescription 'FRecipient))
`shouldSatisfy` isRight
testFileDescriptionLimits :: IO ()
testFileDescriptionLimits = do
let largestSplit = prepareChunkSizes (maxFileSizeHard - mb 1)
split64GB = prepareChunkSizes (gb 64 - mb 1)
length largestSplit `shouldBe` 1282
length split64GB `shouldBe` 16386
validateFileDescription (descriptionOf largestSplit 16) `shouldSatisfy` isRight
validateFileDescription (descriptionOf largestSplit 17) `shouldBe` Left "too many chunk replicas"
validateFileDescription (descriptionOf split64GB 4) `shouldSatisfy` isRight
validateFileDescription (descriptionOf (replicate 8192 chunkSize3) 16) `shouldSatisfy` isRight
validateFileDescription (descriptionOf (replicate 8193 chunkSize3) 16) `shouldBe` Left "too many file replicas"
validateFileDescription (descriptionOf serverChunkSizes 1) `shouldSatisfy` isRight
validateFileDescription (descriptionOf [chunkSize3, mb 2] 1) `shouldBe` Left "incorrect chunk size"
validateFileDescription (descriptionOf [chunkSize3, mb 8] 1) `shouldBe` Left "incorrect chunk size"
validateFileDescription (redirectDesc [chunkSize3]) `shouldSatisfy` isRight
validateFileDescription (redirectDesc [chunkSize3, chunkSize0]) `shouldBe` Left "redirect file description is too large"
where
FileDescription {chunks = fixtureChunks} = fileDesc
fixtureReplicas = concatMap (\FileChunk {replicas} -> replicas) fixtureChunks
descriptionOf sizes n =
FileDescription
{ party = SFRecipient,
size = FileSize $ sum $ map fromIntegral sizes,
digest = FileDigest "abc",
key = testSbKey,
nonce = testCbNonce,
chunkSize = FileSize chunkSize3,
chunks = zipWith chunk [1 ..] sizes,
redirect = Nothing
}
where
chunk chunkNo sz = FileChunk {chunkNo, chunkSize = FileSize sz, digest = FileDigest "ghi", replicas = take n $ cycle fixtureReplicas}
redirectDesc sizes = (descriptionOf sizes 1 :: FileDescription 'FRecipient) {redirect = Just RedirectFileInfo {size = FileSize 1024, digest = FileDigest "abc"}}
testSerializeFileDescription :: HasTestEnv => IO ()
testSerializeFileDescription = withRemoveTmpFile $ do
B.writeFile tmpFileDescPath $ strEncode fileDesc
fdSer <- B.readFile tmpFileDescPath
fdExp <- B.readFile fileDescPath
fdSer `shouldBe` fdExp
testFileDescriptionURI :: IO ()
testFileDescriptionURI = do
vfd <- either fail pure $ validateFileDescription fileDesc
let descr = FileDescriptionURI SSSimplex vfd mempty
strDecode (strEncode descr) `shouldBe` Right descr
testFileDescriptionURIExtras :: IO ()
testFileDescriptionURIExtras = do
vfd <- either fail pure $ validateFileDescription fileDesc
let descr = FileDescriptionURI SSSimplex vfd $ Just "{\"something\":\"extra\",\"more\":true}"
strDecode (strEncode descr) `shouldBe` Right descr
testHashChunks :: HasTestEnv => IO ()
testHashChunks = do
let sizes = [0, 65535, 65536, 100000, 200000]
paths = map (testPath . ("chunk" <>) . show) [1 .. length sizes]
chunks = zipWith B.replicate sizes ['a' ..]
zipWithM_ B.writeFile paths chunks
digest <- sha512HashChunks paths
digest `shouldBe` LC.sha512Hash (LB.fromStrict $ B.concat chunks)
testDecryptChunks :: HasTestEnv => IO ()
testDecryptChunks =
forM_ ([kb 128 .. kb 128 + 16] <> [mb 2 + 1, mb 2 + kb 512 + 15]) $ \payloadSize -> do
let fileHdr = smpEncode FileHeader {fileName = "testfile", fileExtra = Nothing}
fileSize' = payloadSize - fileSizeLen - authTagSize
content = B.replicate (fromIntegral fileSize' - B.length fileHdr) 'a'
chunkSizes = prepareChunkSizes payloadSize
encSize = sum $ map fromIntegral chunkSizes
chunkPaths = map (testPath . ("chunk" <>) . show) [1 .. length chunkSizes]
B.writeFile srcPath content
runExceptT (encryptFile (CF.plain srcPath) fileHdr testSbKey testCbNonce fileSize' encSize encPath) `shouldReturn` Right ()
withFile encPath ReadMode $ \h ->
zipWithM_ (\path sz -> B.writeFile path =<< B.hGet h (fromIntegral sz)) chunkPaths chunkSizes
runExceptT (decryptChunks encSize chunkPaths testSbKey testCbNonce $ \_ -> pure $ CF.plain dstPath) `shouldReturn` Right (CF.plain dstPath)
getFileSize dstPath `shouldReturn` fromIntegral (B.length content)
B.readFile dstPath `shouldReturn` content
where
srcPath = testPath "source"
encPath = testPath "encrypted"
dstPath = testPath "decrypted"
withRemoveTmpFile :: HasTestEnv => IO () -> IO ()
withRemoveTmpFile =
bracket_
(pure ())
(removeFile tmpFileDescPath)