diff --git a/CHANGELOG.md b/CHANGELOG.md index a5f37cf73..3311f2681 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +# 5.5.3 + +Agent: +- notification token API also returns active notifications server. +- support file descriptions with redirection and file URIs. + +Servers: +- CLI commands for online key and certificate rotation. +- Configure config and log paths via environment variables. + # 5.5.2 Extensible handshake for clients and SMP/NTF servers (ignore extra data). diff --git a/apps/ntf-server/Main.hs b/apps/ntf-server/Main.hs index aa62d35d8..31dbaec07 100644 --- a/apps/ntf-server/Main.hs +++ b/apps/ntf-server/Main.hs @@ -1,13 +1,14 @@ module Main where import Control.Logger.Simple +import Simplex.Messaging.Server.CLI (getEnvPath) import Simplex.Messaging.Notifications.Server.Main -cfgPath :: FilePath -cfgPath = "/etc/opt/simplex-notifications" +defaultCfgPath :: FilePath +defaultCfgPath = "/etc/opt/simplex-notifications" -logPath :: FilePath -logPath = "/var/opt/simplex-notifications" +defaultLogPath :: FilePath +defaultLogPath = "/var/opt/simplex-notifications" logCfg :: LogConfig logCfg = LogConfig {lc_file = Nothing, lc_stderr = True} @@ -15,4 +16,6 @@ logCfg = LogConfig {lc_file = Nothing, lc_stderr = True} main :: IO () main = do setLogLevel LogDebug -- change to LogError in production + cfgPath <- getEnvPath "NTF_SERVER_CFG_PATH" defaultCfgPath + logPath <- getEnvPath "NTF_SERVER_LOG_PATH" defaultLogPath withGlobalLogging logCfg $ ntfServerCLI cfgPath logPath diff --git a/apps/smp-server/Main.hs b/apps/smp-server/Main.hs index 92bdfded4..d5cc5e732 100644 --- a/apps/smp-server/Main.hs +++ b/apps/smp-server/Main.hs @@ -3,8 +3,8 @@ module Main where import Control.Logger.Simple +import Simplex.Messaging.Server.CLI (getEnvPath) import Simplex.Messaging.Server.Main -import System.Environment defaultCfgPath :: FilePath defaultCfgPath = "/etc/opt/simplex" @@ -21,6 +21,3 @@ main = do cfgPath <- getEnvPath "SMP_SERVER_CFG_PATH" defaultCfgPath logPath <- getEnvPath "SMP_SERVER_LOG_PATH" defaultLogPath withGlobalLogging logCfg $ smpServerCLI cfgPath logPath - -getEnvPath :: String -> FilePath -> IO FilePath -getEnvPath name def = maybe def (\case "" -> def; f -> f) <$> lookupEnv name diff --git a/apps/xftp-server/Main.hs b/apps/xftp-server/Main.hs index ca3528872..269156c93 100644 --- a/apps/xftp-server/Main.hs +++ b/apps/xftp-server/Main.hs @@ -1,13 +1,14 @@ module Main where import Control.Logger.Simple +import Simplex.Messaging.Server.CLI (getEnvPath) import Simplex.FileTransfer.Server.Main -cfgPath :: FilePath -cfgPath = "/etc/opt/simplex-xftp" +defaultCfgPath :: FilePath +defaultCfgPath = "/etc/opt/simplex-xftp" -logPath :: FilePath -logPath = "/var/opt/simplex-xftp" +defaultLogPath :: FilePath +defaultLogPath = "/var/opt/simplex-xftp" logCfg :: LogConfig logCfg = LogConfig {lc_file = Nothing, lc_stderr = True} @@ -15,4 +16,6 @@ logCfg = LogConfig {lc_file = Nothing, lc_stderr = True} main :: IO () main = do setLogLevel LogDebug -- change to LogError in production + cfgPath <- getEnvPath "XFTP_SERVER_CFG_PATH" defaultCfgPath + logPath <- getEnvPath "XFTP_SERVER_LOG_PATH" defaultLogPath withGlobalLogging logCfg $ xftpServerCLI cfgPath logPath diff --git a/package.yaml b/package.yaml index 90ab76092..13109241f 100644 --- a/package.yaml +++ b/package.yaml @@ -1,5 +1,5 @@ name: simplexmq -version: 5.5.2.1 +version: 5.5.3.0 synopsis: SimpleXMQ message broker description: | This package includes <./docs/Simplex-Messaging-Server.html server>, diff --git a/rfcs/2024-01-26-file-links.md b/rfcs/2024-01-26-file-links.md new file mode 100644 index 000000000..3ff2f430e --- /dev/null +++ b/rfcs/2024-01-26-file-links.md @@ -0,0 +1,73 @@ +# Sending large file descriptions + +It is desirable to provide a QR code/URI from which a file can be downloaded. This way files may be addressed outside a chat client. +Currently the `xftp` CLI tool can generate YAML file descriptions that can be used to receive a file. +It is possible to pass such a description as an URI, but descriptions for files larger than ~8 MBs (two 4 MB chunks) would give QR codes that are difficult to process. +A user can manually upload description and get a shorter one. Typically descriptions for files that are up to ~20 GBs would still be small enough to not require another pass, and that is way beyond any current (or, reasonable, fwiw) limitations. + +It is possible to streamline this process, so any application using simplexmq agent can easily send file descriptions and follow redirects. +A file description with a redirect contains an extra field with final file size and digest so it can be followed automatically. + +The flow would be like this: + +- Sending: + 1. Upload file as usual with `xftpSendFile`, get recipient file descriptions in `SFDONE` message. + 2. Upload one of the file descriptions with `xftpSendDescription`, get its redirect-description in its `SFDONE` message. + 3. Wrap in `FileDescriptionURI` and use `strEncode` to get a QR-sized URI. + 4. Show QR code / copy link. +- Receiving: + 1. Scan QR code / paste link. + 2. Use `strDecode` and unwrap `FileDescriptionURI` to get `ValidFileDescription 'FRecipient`. + 3. Download it as usual with `xftpReceiveFile`, getting `RFDONE` message when the file is fully received. + +It is not necesary to use redirect description if original description can be encoded to fit in 1002 characters. Beyond this size there is a significant jump in QR code complexity. +It is possible to call `encodeFileDescriptionURI` right after upload to test if the URI fits and skip step 2. +When `xftpReceiveFile` receives a decoded description that lacks `redirect` field, the procedure for downloading a file is the same as usual - download chunks and reassemble local file. + +## Agent changes + +### Sending + +Sending and receiving files in agent is a multi-step process mediated by DB entries in `snd_files` and `rcv_files` tables. + +`xftpSendDescription` is tasked with storing original description in a temporary locally-encrypted file, then creating upload task for it. + +It is necessary to preserve redirect metadata so it can be attached to descriptions in the `SFDONE` message sent by a worker: + +```sql +ALTER TABLE snd_files ADD COLUMN redirect_size INTEGER; +ALTER TABLE snd_files ADD COLUMN redirect_digest BLOB; +``` + +### Receiving + +`xftpReceiveFile` gets a file description as an argument and knows if it should follow redirect procedure or run an ordinary download. +For redirects it will prepare a `RcvFile` for redirect and then a placeholder, for the final file. +Agent messages would be sent using the entity ID of the final file, which is stored along with redirect metadata in `RcvFile` for the redirect. + +```sql +ALTER TABLE rcv_files ADD COLUMN redirect_id INTEGER REFERENCES rcv_files ON DELETE CASCADE; -- for later updates +ALTER TABLE rcv_files ADD COLUMN redirect_entity_id BLOB; -- for notifications +ALTER TABLE rcv_files ADD COLUMN redirect_size INTEGER; +ALTER TABLE rcv_files ADD COLUMN redirect_digest BLOB; +``` + +These additional fields will exist on the file that is a short description to receive an actual description of the final file. + +While a description YAML is being downloaded, the application will get `RFPROG` messages tagged for final entity, containing bytes downloaded so far and the total size from the original file. +When the description is fully downloaded, the worker would decode description and check if the stated size and digest match the declared in redirect. +Then it will replace placeholder description in `rcv_files` for destination file with the actual data from downloaded description. +Finally, instead of sending `RFDONE` for redirect, it hands over work to chunk download worker, which will run exactly as if the user requested its download directly. +An application will then receive `RFPROG` and `RFDONE` messages as usual. + +## URI encoding + +File description URIs use the same service schema `simplex:` or its `https://simplex.chat` (or any custom host) equivalent as do contact links and can be extracted from text and processed the same way. +The path section is `/file` (with an optional trailing `/`). +The payload is encoded in the "fragment" part of the link, using `#/?`, followed by a query string. +File description is encoded first in a YAML document, then URL-encoded in under the key `d`. +An application may want to pass extra parameters not necessary to download a file. Those go in the `_` key, encoded as a JSON dictionary. + +An example link: + +`simplex:/file#/?d=chunkSize%3A%2064kb%0Adigest%3A%20OtpnXkECTW4a18Eots2m3O22maeOCMqPUX4ulugIjgMEJfCpTYc_-T257Uw7s9bW_F0G5WBg5BioBWd4Z_OoCw%3D%3D%0Akey%3A%20rNR8_2SJuH7Qve43gV3zszL0R6oY5HSdRZT_paB-wfE%3D%0Anonce%3A%202oKwfK-w75nwyWp8_1Lv6QnQonIRtJmG%0Aparty%3A%20recipient%0Areplicas%3A%0A-%20chunks%3A%0A%20%20-%201%3ATdvaxMnG2Ph1e3QCx3-rpA%3D%3D%3AMC4CAQAwBQYDK2VwBCIEILdErEICvgrBCajDLTX2h3LXyMB7z5vrtLa3XVigJuf-%3ANS46KuYdgOWs6dUeMp7p2oF8rBQ9wQ2Ez6TW6Y6gHg0%3D%0A%20%20-%202%3AH5SRbtKYrXWVXTthrkeWzw%3D%3D%3AMC4CAQAwBQYDK2VwBCIEIGeEPNLt7lUGPfplwsoJLCDFnbIc5Hm31kz5X6rWXmgu%3A7QNRI-gvFx9UM-baXp3YVDli9pcfh3HGFKDhsA9JQHY%3D%0A%20%20-%203%3A_xjukkIl9WZFryUXT0h_TQ%3D%3D%3AMC4CAQAwBQYDK2VwBCIEIIRFBaL1HvUfePvKLuggwUrC_q_ZHd7v08IL9jhM7teC%3Aid2lgLMMjTGsR8SUogJuRdLoEHAc5SDQKFDqlZRSuEY%3D%0A%20%20server%3A%20xftp%3A%2F%2FLcJUMfVhwD8yxjAiSaDzzGF3-kLG4Uh0Fl_ZIjrRwjI%3D%40localhost%3A7002%0Asize%3A%20192kb%0A&_=%7B%22k%22:%22test%22%7D` diff --git a/simplexmq.cabal b/simplexmq.cabal index e7adf4050..1f7027b5f 100644 --- a/simplexmq.cabal +++ b/simplexmq.cabal @@ -5,7 +5,7 @@ cabal-version: 1.12 -- see: https://github.com/sol/hpack name: simplexmq -version: 5.5.2.1 +version: 5.5.3.0 synopsis: SimpleXMQ message broker description: This package includes <./docs/Simplex-Messaging-Server.html server>, <./docs/Simplex-Messaging-Client.html client> and @@ -102,6 +102,7 @@ library Simplex.Messaging.Agent.Store.SQLite.Migrations.M20231222_command_created_at Simplex.Messaging.Agent.Store.SQLite.Migrations.M20231225_failed_work_items Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240121_message_delivery_indexes + Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240124_file_redirect Simplex.Messaging.Agent.TRcvQueues Simplex.Messaging.Client Simplex.Messaging.Client.Agent @@ -142,6 +143,7 @@ library Simplex.Messaging.Server.QueueStore.STM Simplex.Messaging.Server.Stats Simplex.Messaging.Server.StoreLog + Simplex.Messaging.ServiceScheme Simplex.Messaging.TMap Simplex.Messaging.Transport Simplex.Messaging.Transport.Buffer diff --git a/src/Simplex/FileTransfer/Agent.hs b/src/Simplex/FileTransfer/Agent.hs index 0cc6a3bde..3bed1ce21 100644 --- a/src/Simplex/FileTransfer/Agent.hs +++ b/src/Simplex/FileTransfer/Agent.hs @@ -7,6 +7,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TupleSections #-} {-# LANGUAGE TypeApplications #-} {-# OPTIONS_GHC -fno-warn-ambiguous-fields #-} @@ -19,6 +20,7 @@ module Simplex.FileTransfer.Agent xftpDeleteRcvFile', -- Sending files xftpSendFile', + xftpSendDescription', deleteSndFileInternal, deleteSndFileRemote, ) @@ -44,6 +46,7 @@ import Simplex.FileTransfer.Client.Main import Simplex.FileTransfer.Crypto import Simplex.FileTransfer.Description import Simplex.FileTransfer.Protocol (FileParty (..), SFileParty (..)) +import qualified Simplex.FileTransfer.Protocol as XFTP import Simplex.FileTransfer.Transport (XFTPRcvChunkSpec (..)) import Simplex.FileTransfer.Types import Simplex.FileTransfer.Util (removePath, uniqueCombine) @@ -57,6 +60,7 @@ import Simplex.Messaging.Crypto.File (CryptoFile (..), CryptoFileArgs) import qualified Simplex.Messaging.Crypto.File as CF import qualified Simplex.Messaging.Crypto.Lazy as LC import Simplex.Messaging.Encoding +import Simplex.Messaging.Encoding.String (strDecode, strEncode) import Simplex.Messaging.Protocol (EntityId, XFTPServer) import Simplex.Messaging.Util (liftError, tshow, unlessM, whenM) import System.FilePath (takeFileName, ()) @@ -97,7 +101,7 @@ closeXFTPAgent a = do stopWorkers workers = atomically (swapTVar workers M.empty) >>= mapM_ (liftIO . cancelWorker) xftpReceiveFile' :: AgentMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Maybe CryptoFileArgs -> m RcvFileId -xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks}) cfArgs = do +xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks, redirect}) cfArgs = do g <- asks random prefixPath <- getPrefixPath "rcv.xftp" createDirectory prefixPath @@ -107,14 +111,25 @@ xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks}) cfA createDirectory =<< toFSFilePath relTmpPath createEmptyFile =<< toFSFilePath relSavePath let saveFile = CryptoFile relSavePath cfArgs - fId <- withStore c $ \db -> createRcvFile db g userId fd relPrefixPath relTmpPath saveFile - forM_ chunks downloadChunk + fId <- case redirect of + Nothing -> withStore c $ \db -> createRcvFile db g userId fd relPrefixPath relTmpPath saveFile + Just _ -> do + -- prepare description paths + let relTmpPathRedirect = relPrefixPath "xftp.redirect-encrypted" + relSavePathRedirect = relPrefixPath "xftp.redirect-decrypted" + createDirectory =<< toFSFilePath relTmpPathRedirect + createEmptyFile =<< toFSFilePath relSavePathRedirect + cfArgsRedirect <- atomically $ CF.randomArgs g + let saveFileRedirect = CryptoFile relSavePathRedirect $ Just cfArgsRedirect + -- create download tasks + withStore c $ \db -> createRcvFileRedirect db g userId fd relPrefixPath relTmpPathRedirect saveFileRedirect relTmpPath saveFile + forM_ chunks (downloadChunk c) pure fId - where - downloadChunk :: AgentMonad m => FileChunk -> m () - downloadChunk FileChunk {replicas = (FileChunkReplica {server} : _)} = do - void $ getXFTPRcvWorker True c (Just server) - downloadChunk _ = throwError $ INTERNAL "no replicas" + +downloadChunk :: AgentMonad m => AgentClient -> FileChunk -> m () +downloadChunk c FileChunk {replicas = (FileChunkReplica {server} : _)} = do + void $ getXFTPRcvWorker True c (Just server) +downloadChunk _ _ = throwError $ INTERNAL "no replicas" getPrefixPath :: AgentMonad m => String -> m FilePath getPrefixPath suffix = do @@ -172,14 +187,17 @@ runXFTPRcvWorker c srv Worker {doWork} = do relChunkPath = fileTmpPath takeFileName chunkPath agentXFTPDownloadChunk c userId digest replica chunkSpec atomically $ waitUntilForeground c - (complete, progress) <- withStore c $ \db -> runExceptT $ do + (entityId, complete, progress) <- withStore c $ \db -> runExceptT $ do liftIO $ updateRcvFileChunkReceived db (rcvChunkReplicaId replica) rcvChunkId relChunkPath - RcvFile {size = FileSize total, chunks} <- ExceptT $ getRcvFile db rcvFileId + RcvFile {size = FileSize currentSize, chunks, redirect} <- ExceptT $ getRcvFile db rcvFileId let rcvd = receivedSize chunks complete = all chunkReceived chunks + (entityId, total) = case redirect of + Nothing -> (rcvFileEntityId, currentSize) + Just RcvFileRedirect {redirectFileInfo = RedirectFileInfo {size = FileSize finalSize}, redirectEntityId} -> (redirectEntityId, finalSize) liftIO . when complete $ updateRcvFileStatus db rcvFileId RFSReceived - pure (complete, RFPROG rcvd total) - notify c rcvFileEntityId progress + pure (entityId, complete, RFPROG rcvd total) + notify c entityId progress when complete . void $ getXFTPRcvWorker True c Nothing where @@ -223,7 +241,7 @@ runXFTPRcvLocalWorker c Worker {doWork} = do \f@RcvFile {rcvFileId, rcvFileEntityId, tmpPath} -> decryptFile f `catchAgentError` (rcvWorkerInternalError c rcvFileId rcvFileEntityId tmpPath . show) decryptFile :: RcvFile -> m () - decryptFile RcvFile {rcvFileId, rcvFileEntityId, key, nonce, tmpPath, saveFile, status, chunks} = do + decryptFile RcvFile {rcvFileId, rcvFileEntityId, size, digest, key, nonce, tmpPath, saveFile, status, chunks, redirect} = do let CryptoFile savePath cfArgs = saveFile fsSavePath <- toFSFilePath savePath when (status == RFSDecrypting) $ @@ -231,12 +249,33 @@ runXFTPRcvLocalWorker c Worker {doWork} = do withStore' c $ \db -> updateRcvFileStatus db rcvFileId RFSDecrypting chunkPaths <- getChunkPaths chunks encSize <- liftIO $ foldM (\s path -> (s +) . fromIntegral <$> getFileSize path) 0 chunkPaths + when (FileSize encSize /= size) $ throwError $ XFTP XFTP.SIZE + encDigest <- liftIO $ LC.sha512Hash <$> readChunks chunkPaths + when (FileDigest encDigest /= digest) $ throwError $ XFTP XFTP.DIGEST let destFile = CryptoFile fsSavePath cfArgs void $ liftError (INTERNAL . show) $ decryptChunks encSize chunkPaths key nonce $ \_ -> pure destFile - notify c rcvFileEntityId $ RFDONE fsSavePath - forM_ tmpPath (removePath <=< toFSFilePath) - atomically $ waitUntilForeground c - withStore' c (`updateRcvFileComplete` rcvFileId) + case redirect of + Nothing -> do + notify c rcvFileEntityId $ RFDONE fsSavePath + forM_ tmpPath (removePath <=< toFSFilePath) + atomically $ waitUntilForeground c + withStore' c (`updateRcvFileComplete` rcvFileId) + Just RcvFileRedirect {redirectFileInfo, redirectDbId} -> do + let RedirectFileInfo {size = redirectSize, digest = redirectDigest} = redirectFileInfo + forM_ tmpPath (removePath <=< toFSFilePath) + atomically $ waitUntilForeground c + withStore' c (`updateRcvFileComplete` rcvFileId) + -- proceed with redirect + yaml <- liftError (INTERNAL . show) (CF.readFile $ CryptoFile fsSavePath cfArgs) `finally` (toFSFilePath fsSavePath >>= removePath) + next@FileDescription {chunks = nextChunks} <- case strDecode (LB.toStrict yaml) of + Left _ -> throwError . XFTP $ XFTP.REDIRECT "decode error" + Right (ValidFileDescription fd@FileDescription {size = dstSize, digest = dstDigest}) + | dstSize /= redirectSize -> throwError . XFTP $ XFTP.REDIRECT "size mismatch" + | dstDigest /= redirectDigest -> throwError . XFTP $ XFTP.REDIRECT "digest mismatch" + | otherwise -> pure fd + -- register and download chunks from the actual file + withStore c $ \db -> updateRcvFileRedirect db redirectDbId next + forM_ nextChunks (downloadChunk c) where getChunkPaths :: [RcvFileChunk] -> m [FilePath] getChunkPaths [] = pure [] @@ -268,7 +307,23 @@ xftpSendFile' c userId file numRecipients = do key <- atomically $ C.randomSbKey g nonce <- atomically $ C.randomCbNonce g -- saving absolute filePath will not allow to restore file encryption after app update, but it's a short window - fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce + fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce Nothing + void $ getXFTPSndWorker True c Nothing + pure fId + +xftpSendDescription' :: forall m. AgentMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> m SndFileId +xftpSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) numRecipients = do + g <- asks random + prefixPath <- getPrefixPath "snd.xftp" + createDirectory prefixPath + let relPrefixPath = takeFileName prefixPath + let directYaml = prefixPath "direct.yaml" + cfArgs <- atomically $ CF.randomArgs g + let file = CryptoFile directYaml (Just cfArgs) + liftError (INTERNAL . show) $ CF.writeFile file (LB.fromStrict $ strEncode fdDirect) + key <- atomically $ C.randomSbKey g + nonce <- atomically $ C.randomCbNonce g + fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce $ Just RedirectFileInfo {size, digest} void $ getXFTPSndWorker True c Nothing pure fId @@ -423,15 +478,15 @@ runXFTPSndWorker c srv Worker {doWork} = do sndFileToDescrs :: SndFile -> m (ValidFileDescription 'FSender, [ValidFileDescription 'FRecipient]) sndFileToDescrs SndFile {digest = Nothing} = throwError $ INTERNAL "snd file has no digest" sndFileToDescrs SndFile {chunks = []} = throwError $ INTERNAL "snd file has no chunks" - sndFileToDescrs SndFile {digest = Just digest, key, nonce, chunks = chunks@(fstChunk : _)} = do + sndFileToDescrs SndFile {digest = Just digest, key, nonce, chunks = chunks@(fstChunk : _), redirect} = do let chunkSize = FileSize $ sndChunkSize fstChunk size = FileSize $ sum $ map (fromIntegral . sndChunkSize) chunks -- snd description sndDescrChunks <- mapM toSndDescrChunk chunks - let fdSnd = FileDescription {party = SFSender, size, digest, key, nonce, chunkSize, chunks = sndDescrChunks} + let fdSnd = FileDescription {party = SFSender, size, digest, key, nonce, chunkSize, chunks = sndDescrChunks, redirect = Nothing} validFdSnd <- either (throwError . INTERNAL) pure $ validateFileDescription fdSnd -- rcv descriptions - let fdRcv = FileDescription {party = SFRecipient, size, digest, key, nonce, chunkSize, chunks = []} + let fdRcv = FileDescription {party = SFRecipient, size, digest, key, nonce, chunkSize, chunks = [], redirect} fdRcvs = createRcvFileDescriptions fdRcv chunks validFdRcvs <- either (throwError . INTERNAL) pure $ mapM validateFileDescription fdRcvs pure (validFdSnd, validFdRcvs) diff --git a/src/Simplex/FileTransfer/Client/Main.hs b/src/Simplex/FileTransfer/Client/Main.hs index 7c3521fc2..906966fb0 100644 --- a/src/Simplex/FileTransfer/Client/Main.hs +++ b/src/Simplex/FileTransfer/Client/Main.hs @@ -15,6 +15,7 @@ module Simplex.FileTransfer.Client.Main CLIError (..), xftpClientCLI, cliSendFile, + cliSendFileOpts, prepareChunkSizes, prepareChunkSpecs, maxFileSize, @@ -297,8 +298,8 @@ cliSendFileOpts SendOptions {filePath, outputDir, numRecipients, xftpServers, re withExceptT (CLIError . show) $ encryptFile srcFile fileHdr key nonce fileSize' encSize encPath digest <- liftIO $ LC.sha512Hash <$> LB.readFile encPath let chunkSpecs = prepareChunkSpecs encPath chunkSizes - fdRcv = FileDescription {party = SFRecipient, size = FileSize encSize, digest = FileDigest digest, key, nonce, chunkSize = FileSize defChunkSize, chunks = []} - fdSnd = FileDescription {party = SFSender, size = FileSize encSize, digest = FileDigest digest, key, nonce, chunkSize = FileSize defChunkSize, chunks = []} + fdRcv = FileDescription {party = SFRecipient, size = FileSize encSize, digest = FileDigest digest, key, nonce, chunkSize = FileSize defChunkSize, chunks = [], redirect = Nothing} + fdSnd = FileDescription {party = SFSender, size = FileSize encSize, digest = FileDigest digest, key, nonce, chunkSize = FileSize defChunkSize, chunks = [], redirect = Nothing} logInfo $ "encrypted file to " <> tshow encPath pure (encPath, fdRcv, fdSnd, chunkSpecs, encSize) uploadFile :: TVar ChaChaDRG -> [XFTPChunkSpec] -> TVar [Int64] -> Int64 -> ExceptT CLIError IO [SentFileChunk] @@ -526,9 +527,8 @@ prepareChunkSizes size' = prepareSizes size' where (smallSize, bigSize) | size' > size34 chunkSize3 = (chunkSize2, chunkSize3) - | otherwise = (chunkSize1, chunkSize2) - -- | size' > size34 chunkSize2 = (chunkSize1, chunkSize2) - -- | otherwise = (chunkSize0, chunkSize1) + | size' > size34 chunkSize2 = (chunkSize1, chunkSize2) + | otherwise = (chunkSize0, chunkSize1) size34 sz = (fromIntegral sz * 3) `div` 4 prepareSizes 0 = [] prepareSizes size diff --git a/src/Simplex/FileTransfer/Description.hs b/src/Simplex/FileTransfer/Description.hs index cbea3f23b..bf5c91634 100644 --- a/src/Simplex/FileTransfer/Description.hs +++ b/src/Simplex/FileTransfer/Description.hs @@ -12,6 +12,7 @@ module Simplex.FileTransfer.Description ( FileDescription (..), + RedirectFileInfo (..), AFileDescription (..), ValidFileDescription, -- constructor is not exported, use pattern pattern ValidFileDescription, @@ -30,12 +31,17 @@ module Simplex.FileTransfer.Description kb, mb, gb, + FileDescriptionURI (..), + FileClientData, + fileDescriptionURI, + qrSizeLimit, ) where import Control.Applicative (optional) import Control.Monad ((<=<)) import Data.Aeson (FromJSON, ToJSON) +import qualified Data.Aeson as J import qualified Data.Aeson.TH as J import Data.Attoparsec.ByteString.Char8 (Parser) import qualified Data.Attoparsec.ByteString.Char8 as A @@ -50,17 +56,21 @@ import Data.Map (Map) import qualified Data.Map as M import Data.Maybe (fromMaybe) import Data.String +import Data.Text (Text) +import Data.Text.Encoding (encodeUtf8) import Data.Word (Word32) import qualified Data.Yaml as Y import Database.SQLite.Simple.FromField (FromField (..)) import Database.SQLite.Simple.ToField (ToField (..)) import Simplex.FileTransfer.Chunks import Simplex.FileTransfer.Protocol +import Simplex.Messaging.Agent.QueryString import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Encoding.String import Simplex.Messaging.Parsers (defaultJSON, parseAll) import Simplex.Messaging.Protocol (XFTPServer) -import Simplex.Messaging.Util (bshow, (<$?>)) +import Simplex.Messaging.ServiceScheme (ServiceScheme (..)) +import Simplex.Messaging.Util (bshow, safeDecodeUtf8, (<$?>)) data FileDescription (p :: FileParty) = FileDescription { party :: SFileParty p, @@ -69,7 +79,14 @@ data FileDescription (p :: FileParty) = FileDescription key :: C.SbKey, nonce :: C.CbNonce, chunkSize :: FileSize Word32, - chunks :: [FileChunk] + chunks :: [FileChunk], + redirect :: Maybe RedirectFileInfo + } + deriving (Eq, Show) + +data RedirectFileInfo = RedirectFileInfo + { size :: FileSize Int64, + digest :: FileDigest } deriving (Eq, Show) @@ -147,7 +164,8 @@ data YAMLFileDescription = YAMLFileDescription key :: C.SbKey, nonce :: C.CbNonce, chunkSize :: String, - replicas :: [YAMLServerReplicas] + replicas :: [YAMLServerReplicas], + redirect :: Maybe RedirectFileInfo } deriving (Eq, Show) @@ -170,8 +188,16 @@ data FileServerReplica = FileServerReplica newtype FileSize a = FileSize {unFileSize :: a} deriving (Eq, Show) +instance FromJSON a => FromJSON (FileSize a) where + parseJSON v = FileSize <$> Y.parseJSON v + +instance ToJSON a => ToJSON (FileSize a) where + toJSON = Y.toJSON . unFileSize + $(J.deriveJSON defaultJSON ''YAMLServerReplicas) +$(J.deriveJSON defaultJSON ''RedirectFileInfo) + $(J.deriveJSON defaultJSON ''YAMLFileDescription) instance FilePartyI p => StrEncoding (ValidFileDescription p) where @@ -204,7 +230,7 @@ validateFileDescription fd@FileDescription {size, chunks} chunksSize = fromIntegral . foldl' (\s FileChunk {chunkSize} -> s + unFileSize chunkSize) 0 encodeFileDescription :: FileDescription p -> YAMLFileDescription -encodeFileDescription FileDescription {party, size, digest, key, nonce, chunkSize, chunks} = +encodeFileDescription FileDescription {party, size, digest, key, nonce, chunkSize, chunks, redirect} = YAMLFileDescription { party = toFileParty party, size = B.unpack $ strEncode size, @@ -212,9 +238,39 @@ encodeFileDescription FileDescription {party, size, digest, key, nonce, chunkSiz key, nonce, chunkSize = B.unpack $ strEncode chunkSize, - replicas = encodeFileReplicas chunkSize chunks + replicas = encodeFileReplicas chunkSize chunks, + redirect } +data FileDescriptionURI = FileDescriptionURI + { scheme :: ServiceScheme, + description :: ValidFileDescription 'FRecipient, + clientData :: Maybe FileClientData -- JSON-encoded extensions to pass in a link + } + deriving (Eq, Show) + +type FileClientData = Text + +fileDescriptionURI :: ValidFileDescription 'FRecipient -> FileDescriptionURI +fileDescriptionURI vfd = FileDescriptionURI SSSimplex vfd mempty + +instance StrEncoding FileDescriptionURI where + strEncode FileDescriptionURI {scheme, description, clientData} = mconcat [strEncode scheme, "/file", "#/?", queryStr] + where + queryStr = strEncode $ QSP QEscape qs + qs = ("desc", strEncode description) : maybe [] (\cd -> [("data", encodeUtf8 cd)]) clientData + strP = do + scheme <- strP + _ <- "/file" <* optional (A.char '/') <* "#/?" + query <- strP + description <- queryParam "desc" query + let clientData = safeDecodeUtf8 <$> queryParamStr "data" query + pure FileDescriptionURI {scheme, description, clientData} + +-- | URL length in QR code before jumping up to a next size. +qrSizeLimit :: Int +qrSizeLimit = 1002 -- ~2 chunks in URLencoded YAML with some spare size for server hosts + instance (Integral a, Show a) => StrEncoding (FileSize a) where strEncode (FileSize b) | b' /= 0 = bshow b @@ -285,13 +341,13 @@ unfoldChunksToReplicas defChunkSize = concatMap chunkReplicas in FileServerReplica {chunkNo, server, replicaId, replicaKey, digest = digest', chunkSize = chunkSize'} decodeFileDescription :: YAMLFileDescription -> Either String AFileDescription -decodeFileDescription YAMLFileDescription {party, size, digest, key, nonce, chunkSize, replicas} = do +decodeFileDescription YAMLFileDescription {party, size, digest, key, nonce, chunkSize, replicas, redirect} = do size' <- strDecode $ B.pack size chunkSize' <- strDecode $ B.pack chunkSize replicas' <- decodeFileParts replicas chunks <- foldReplicasToChunks chunkSize' replicas' pure $ case aFileParty party of - AFP party' -> AFD FileDescription {party = party', size = size', digest, key, nonce, chunkSize = chunkSize', chunks} + AFP party' -> AFD FileDescription {party = party', size = size', digest, key, nonce, chunkSize = chunkSize', chunks, redirect} where decodeFileParts = fmap concat . mapM decodeYAMLServerReplicas diff --git a/src/Simplex/FileTransfer/Protocol.hs b/src/Simplex/FileTransfer/Protocol.hs index 578aefd15..5602c4268 100644 --- a/src/Simplex/FileTransfer/Protocol.hs +++ b/src/Simplex/FileTransfer/Protocol.hs @@ -338,6 +338,8 @@ data XFTPErrorType HAS_FILE | -- | file IO error FILE_IO + | -- | bad redirect data + REDIRECT {redirectError :: String} | -- | internal server error INTERNAL | -- | used internally, never returned by the server (to be removed) @@ -347,8 +349,12 @@ data XFTPErrorType instance StrEncoding XFTPErrorType where strEncode = \case CMD e -> "CMD " <> bshow e + REDIRECT e -> "REDIRECT " <> bshow e e -> bshow e - strP = "CMD " *> (CMD <$> parseRead1) <|> parseRead1 + strP = + "CMD " *> (CMD <$> parseRead1) + <|> "REDIRECT " *> (REDIRECT <$> parseRead A.takeByteString) + <|> parseRead1 instance Encoding XFTPErrorType where smpEncode = \case @@ -363,6 +369,7 @@ instance Encoding XFTPErrorType where NO_FILE -> "NO_FILE" HAS_FILE -> "HAS_FILE" FILE_IO -> "FILE_IO" + REDIRECT err -> "REDIRECT " <> smpEncode err INTERNAL -> "INTERNAL" DUPLICATE_ -> "DUPLICATE_" @@ -379,6 +386,7 @@ instance Encoding XFTPErrorType where "NO_FILE" -> pure NO_FILE "HAS_FILE" -> pure HAS_FILE "FILE_IO" -> pure FILE_IO + "REDIRECT" -> REDIRECT <$> _smpP "INTERNAL" -> pure INTERNAL "DUPLICATE_" -> pure DUPLICATE_ _ -> fail "bad error type" diff --git a/src/Simplex/FileTransfer/Server/Main.hs b/src/Simplex/FileTransfer/Server/Main.hs index 4f5d0558c..33975298e 100644 --- a/src/Simplex/FileTransfer/Server/Main.hs +++ b/src/Simplex/FileTransfer/Server/Main.hs @@ -33,7 +33,7 @@ import System.IO (BufferMode (..), hSetBuffering, stderr, stdout) import Text.Read (readMaybe) xftpServerVersion :: String -xftpServerVersion = "1.2.2.0" +xftpServerVersion = "1.2.3.0" xftpServerCLI :: FilePath -> FilePath -> IO () xftpServerCLI cfgPath logPath = do @@ -42,6 +42,10 @@ xftpServerCLI cfgPath logPath = do doesFileExist iniFile >>= \case True -> exitError $ "Error: server is already initialized (" <> iniFile <> " exists).\nRun `" <> executableName <> " start`." _ -> initializeServer opts + OnlineCert certOpts -> + doesFileExist iniFile >>= \case + True -> genOnline cfgPath certOpts + _ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`." Start -> doesFileExist iniFile >>= \case True -> readIniFile iniFile >>= either exitError runServer @@ -179,6 +183,7 @@ xftpServerCLI cfgPath logPath = do data CliCommand = Init InitOptions + | OnlineCert CertOptions | Start | Delete @@ -196,6 +201,7 @@ cliCommandP :: FilePath -> FilePath -> FilePath -> Parser CliCommand cliCommandP cfgPath logPath iniFile = hsubparser ( command "init" (info (Init <$> initP) (progDesc $ "Initialize server - creates " <> cfgPath <> " and " <> logPath <> " directories and configuration files")) + <> command "cert" (info (OnlineCert <$> certOptionsP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")")) <> command "start" (info (pure Start) (progDesc $ "Start server (configuration: " <> iniFile <> ")")) <> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files")) ) diff --git a/src/Simplex/FileTransfer/Types.hs b/src/Simplex/FileTransfer/Types.hs index e51cb14e3..513822a82 100644 --- a/src/Simplex/FileTransfer/Types.hs +++ b/src/Simplex/FileTransfer/Types.hs @@ -47,6 +47,7 @@ data RcvFile = RcvFile key :: C.SbKey, nonce :: C.CbNonce, chunkSize :: FileSize Word32, + redirect :: Maybe RcvFileRedirect, chunks :: [RcvFileChunk], prefixPath :: FilePath, tmpPath :: Maybe FilePath, @@ -108,6 +109,13 @@ data RcvFileChunkReplica = RcvFileChunkReplica } deriving (Eq, Show) +data RcvFileRedirect = RcvFileRedirect + { redirectDbId :: DBRcvFileId, + redirectEntityId :: RcvFileId, + redirectFileInfo :: RedirectFileInfo + } + deriving (Eq, Show) + -- Sending files type DBSndFileId = Int64 @@ -124,7 +132,8 @@ data SndFile = SndFile srcFile :: CryptoFile, prefixPath :: Maybe FilePath, status :: SndFileStatus, - deleted :: Bool + deleted :: Bool, + redirect :: Maybe RedirectFileInfo } deriving (Eq, Show) diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs index 3a39686bf..950443585 100644 --- a/src/Simplex/Messaging/Agent.hs +++ b/src/Simplex/Messaging/Agent.hs @@ -93,6 +93,7 @@ module Simplex.Messaging.Agent xftpReceiveFile, xftpDeleteRcvFile, xftpSendFile, + xftpSendDescription, xftpDeleteSndFileInternal, xftpDeleteSndFileRemote, rcNewHostPairing, @@ -137,7 +138,7 @@ import qualified Data.Text as T import Data.Time.Clock import Data.Time.Clock.System (systemToUTCTime) import Data.Word (Word16) -import Simplex.FileTransfer.Agent (closeXFTPAgent, deleteSndFileInternal, deleteSndFileRemote, startXFTPWorkers, toFSFilePath, xftpDeleteRcvFile', xftpReceiveFile', xftpSendFile') +import Simplex.FileTransfer.Agent (closeXFTPAgent, deleteSndFileInternal, deleteSndFileRemote, startXFTPWorkers, toFSFilePath, xftpDeleteRcvFile', xftpReceiveFile', xftpSendDescription', xftpSendFile') import Simplex.FileTransfer.Description (ValidFileDescription) import Simplex.FileTransfer.Protocol (FileParty (..)) import Simplex.FileTransfer.Util (removePath) @@ -163,6 +164,7 @@ import Simplex.Messaging.Notifications.Types import Simplex.Messaging.Parsers (parse) import Simplex.Messaging.Protocol (BrokerMsg, EntityId, ErrorType (AUTH), MsgBody, MsgFlags (..), NtfServer, ProtoServerWithAuth, ProtocolTypeI (..), SMPMsgMeta, SProtocolType (..), SndPublicVerifyKey, SubscriptionMode (..), UserProtocol, XFTPServerWithAuth) import qualified Simplex.Messaging.Protocol as SMP +import Simplex.Messaging.ServiceScheme (ServiceScheme (..)) import qualified Simplex.Messaging.TMap as TM import Simplex.Messaging.Util import Simplex.Messaging.Version @@ -375,7 +377,7 @@ checkNtfToken c = withAgentEnv c . checkNtfToken' c deleteNtfToken :: AgentErrorMonad m => AgentClient -> DeviceToken -> m () deleteNtfToken c = withAgentEnv c . deleteNtfToken' c -getNtfToken :: AgentErrorMonad m => AgentClient -> m (DeviceToken, NtfTknStatus, NotificationsMode) +getNtfToken :: AgentErrorMonad m => AgentClient -> m (DeviceToken, NtfTknStatus, NotificationsMode, NtfServer) getNtfToken c = withAgentEnv c $ getNtfToken' c getNtfTokenData :: AgentErrorMonad m => AgentClient -> m NtfToken @@ -400,6 +402,10 @@ xftpDeleteRcvFile c = withAgentEnv c . xftpDeleteRcvFile' c xftpSendFile :: AgentErrorMonad m => AgentClient -> UserId -> CryptoFile -> Int -> m SndFileId xftpSendFile c = withAgentEnv c .:. xftpSendFile' c +-- | Send XFTP file +xftpSendDescription :: AgentErrorMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> m SndFileId +xftpSendDescription c = withAgentEnv c .:. xftpSendDescription' c + -- | Delete XFTP snd file internally (deletes work files from file system and db records) xftpDeleteSndFileInternal :: AgentErrorMonad m => AgentClient -> SndFileId -> m () xftpDeleteSndFileInternal c = withAgentEnv c . deleteSndFileInternal c @@ -643,7 +649,7 @@ newRcvConnSrv c userId connId enableNtfs cMode clientData subMode srv = do when enableNtfs $ do ns <- asks ntfSupervisor atomically $ sendNtfSubCommand ns (connId, NSCCreate) - let crData = ConnReqUriData CRSSimplex smpAgentVRange [qUri] clientData + let crData = ConnReqUriData SSSimplex smpAgentVRange [qUri] clientData case cMode of SCMContact -> pure (connId, CRContactUri crData) SCMInvitation -> do @@ -1685,10 +1691,10 @@ deleteNtfToken' c deviceToken = deleteNtfSubs c NSCSmpDelete _ -> throwError $ CMD PROHIBITED -getNtfToken' :: AgentMonad m => AgentClient -> m (DeviceToken, NtfTknStatus, NotificationsMode) +getNtfToken' :: AgentMonad m => AgentClient -> m (DeviceToken, NtfTknStatus, NotificationsMode, NtfServer) getNtfToken' c = withStore' c getSavedNtfToken >>= \case - Just NtfToken {deviceToken, ntfTknStatus, ntfMode} -> pure (deviceToken, ntfTknStatus, ntfMode) + Just NtfToken {deviceToken, ntfTknStatus, ntfMode, ntfServer} -> pure (deviceToken, ntfTknStatus, ntfMode, ntfServer) _ -> throwError $ CMD PROHIBITED getNtfTokenData' :: AgentMonad m => AgentClient -> m NtfToken diff --git a/src/Simplex/Messaging/Agent/Protocol.hs b/src/Simplex/Messaging/Agent/Protocol.hs index 54777bb64..0771601ab 100644 --- a/src/Simplex/Messaging/Agent/Protocol.hs +++ b/src/Simplex/Messaging/Agent/Protocol.hs @@ -97,7 +97,7 @@ module Simplex.Messaging.Agent.Protocol AConnectionRequestUri (..), ConnReqUriData (..), CRClientData, - ConnReqScheme (..), + ServiceScheme, simplexChat, AgentErrorType (..), CommandErrorType (..), @@ -197,7 +197,6 @@ import Simplex.Messaging.Protocol SMPServer, SMPServerWithAuth, SndPublicVerifyKey, - SrvLoc (..), SubscriptionMode, legacyEncodeServer, legacyServerP, @@ -208,6 +207,7 @@ import Simplex.Messaging.Protocol pattern SMPServer, ) import qualified Simplex.Messaging.Protocol as SMP +import Simplex.Messaging.ServiceScheme import Simplex.Messaging.Transport (Transport (..), TransportError, serializeTransportError, transportErrorP) import Simplex.Messaging.Transport.Client (TransportHost, TransportHosts_ (..)) import Simplex.Messaging.Util @@ -1123,13 +1123,13 @@ instance forall m. ConnectionModeI m => StrEncoding (ConnectionRequestUri m) whe instance StrEncoding AConnectionRequestUri where strEncode (ACR _ cr) = strEncode cr strP = do - _crScheme :: ConnReqScheme <- strP + _crScheme :: ServiceScheme <- strP crMode <- A.char '/' *> crModeP <* optional (A.char '/') <* "#/?" query <- strP crAgentVRange <- queryParam "v" query crSmpQueues <- queryParam "smp" query let crClientData = safeDecodeUtf8 <$> queryParamStr "data" query - let crData = ConnReqUriData {crScheme = CRSSimplex, crAgentVRange, crSmpQueues, crClientData} + let crData = ConnReqUriData {crScheme = SSSimplex, crAgentVRange, crSmpQueues, crClientData} case crMode of CMInvitation -> do crE2eParams <- queryParam "e2e" query @@ -1328,7 +1328,7 @@ instance Eq AConnectionRequestUri where deriving instance Show AConnectionRequestUri data ConnReqUriData = ConnReqUriData - { crScheme :: ConnReqScheme, + { crScheme :: ServiceScheme, crAgentVRange :: VersionRange, crSmpQueues :: NonEmpty SMPQueueUri, crClientData :: Maybe CRClientData @@ -1337,20 +1337,6 @@ data ConnReqUriData = ConnReqUriData type CRClientData = Text -data ConnReqScheme = CRSSimplex | CRSAppServer SrvLoc - deriving (Eq, Show) - -instance StrEncoding ConnReqScheme where - strEncode = \case - CRSSimplex -> "simplex:" - CRSAppServer srv -> "https://" <> strEncode srv - strP = - "simplex:" $> CRSSimplex - <|> "https://" *> (CRSAppServer <$> strP) - -simplexChat :: ConnReqScheme -simplexChat = CRSAppServer $ SrvLoc "simplex.chat" "" - -- | SMP queue status. data QueueStatus = -- | queue is created diff --git a/src/Simplex/Messaging/Agent/Store/SQLite.hs b/src/Simplex/Messaging/Agent/Store/SQLite.hs index 06f41be9e..9865fe424 100644 --- a/src/Simplex/Messaging/Agent/Store/SQLite.hs +++ b/src/Simplex/Messaging/Agent/Store/SQLite.hs @@ -164,6 +164,7 @@ module Simplex.Messaging.Agent.Store.SQLite -- Rcv files createRcvFile, + createRcvFileRedirect, getRcvFile, getRcvFileByEntityId, updateRcvChunkReplicaDelay, @@ -171,6 +172,7 @@ module Simplex.Messaging.Agent.Store.SQLite updateRcvFileStatus, updateRcvFileError, updateRcvFileComplete, + updateRcvFileRedirect, updateRcvFileNoTmpPath, updateRcvFileDeleted, deleteRcvFile', @@ -231,6 +233,7 @@ import Data.ByteArray (ScrubbedBytes) import qualified Data.ByteArray as BA import Data.ByteString (ByteString) import qualified Data.ByteString.Base64.URL as U +import qualified Data.ByteString.Char8 as B import Data.Char (toLower) import Data.Functor (($>)) import Data.IORef @@ -255,7 +258,7 @@ import qualified Database.SQLite3 as SQLite3 import Network.Socket (ServiceName) import Simplex.FileTransfer.Client (XFTPChunkSpec (..)) import Simplex.FileTransfer.Description -import Simplex.FileTransfer.Protocol (FileParty (..)) +import Simplex.FileTransfer.Protocol (FileParty (..), SFileParty (..)) import Simplex.FileTransfer.Types import Simplex.Messaging.Agent.Protocol import Simplex.Messaging.Agent.RetryInterval (RI2State (..)) @@ -2263,38 +2266,66 @@ getXFTPServerId_ db ProtocolServer {host, port, keyHash} = do DB.query db "SELECT xftp_server_id FROM xftp_servers WHERE xftp_host = ? AND xftp_port = ? AND xftp_key_hash = ?" (host, port, keyHash) createRcvFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription 'FRecipient -> FilePath -> FilePath -> CryptoFile -> IO (Either StoreError RcvFileId) -createRcvFile db gVar userId fd@FileDescription {chunks} prefixPath tmpPath (CryptoFile savePath cfArgs) = runExceptT $ do - (rcvFileEntityId, rcvFileId) <- ExceptT $ insertRcvFile fd +createRcvFile db gVar userId fd@FileDescription {chunks} prefixPath tmpPath file = runExceptT $ do + (rcvFileEntityId, rcvFileId) <- ExceptT $ insertRcvFile db gVar userId fd prefixPath tmpPath file Nothing Nothing liftIO $ forM_ chunks $ \fc@FileChunk {replicas} -> do - chunkId <- insertChunk fc rcvFileId - forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertReplica rno replica chunkId + chunkId <- insertRcvFileChunk db fc rcvFileId + forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertRcvFileChunkReplica db rno replica chunkId pure rcvFileEntityId + +createRcvFileRedirect :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription FRecipient -> FilePath -> FilePath -> CryptoFile -> FilePath -> CryptoFile -> IO (Either StoreError RcvFileId) +createRcvFileRedirect _ _ _ FileDescription {redirect = Nothing} _ _ _ _ _ = pure $ Left $ SEInternal "createRcvFileRedirect called without redirect" +createRcvFileRedirect db gVar userId redirectFd@FileDescription {chunks = redirectChunks, redirect = Just RedirectFileInfo {size, digest}} prefixPath redirectPath redirectFile dstPath dstFile = runExceptT $ do + (dstEntityId, dstId) <- ExceptT $ insertRcvFile db gVar userId dummyDst prefixPath dstPath dstFile Nothing Nothing + (_, redirectId) <- ExceptT $ insertRcvFile db gVar userId redirectFd prefixPath redirectPath redirectFile (Just dstId) (Just dstEntityId) + liftIO $ + forM_ redirectChunks $ \fc@FileChunk {replicas} -> do + chunkId <- insertRcvFileChunk db fc redirectId + forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertRcvFileChunkReplica db rno replica chunkId + pure dstEntityId where - insertRcvFile :: FileDescription 'FRecipient -> IO (Either StoreError (RcvFileId, DBRcvFileId)) - insertRcvFile FileDescription {size, digest, key, nonce, chunkSize} = runExceptT $ do - rcvFileEntityId <- ExceptT $ - createWithRandomId gVar $ \rcvFileEntityId -> - DB.execute - db - "INSERT INTO rcv_files (rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)" - ((rcvFileEntityId, userId, size, digest, key, nonce, chunkSize) :. (prefixPath, tmpPath, savePath, fileKey <$> cfArgs, fileNonce <$> cfArgs, RFSReceiving)) - rcvFileId <- liftIO $ insertedRowId db - pure (rcvFileEntityId, rcvFileId) - insertChunk :: FileChunk -> DBRcvFileId -> IO Int64 - insertChunk FileChunk {chunkNo, chunkSize, digest} rcvFileId = do + dummyDst = FileDescription + { party = SFRecipient, + size, + digest, + redirect = Nothing, + -- updated later with updateRcvFileRedirect + key = C.unsafeSbKey $ B.replicate 32 '#', + nonce = C.cbNonce "", + chunkSize = FileSize 0, + chunks = [] + } + +insertRcvFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription 'FRecipient -> FilePath -> FilePath -> CryptoFile -> Maybe DBRcvFileId -> Maybe RcvFileId -> IO (Either StoreError (RcvFileId, DBRcvFileId)) +insertRcvFile db gVar userId FileDescription {size, digest, key, nonce, chunkSize, redirect} prefixPath tmpPath (CryptoFile savePath cfArgs) redirectId_ redirectEntityId_ = runExceptT $ do + let (redirectDigest_, redirectSize_) = case redirect of + Just RedirectFileInfo {digest = d, size = s} -> (Just d, Just s) + Nothing -> (Nothing, Nothing) + rcvFileEntityId <- ExceptT $ + createWithRandomId gVar $ \rcvFileEntityId -> DB.execute db - "INSERT INTO rcv_file_chunks (rcv_file_id, chunk_no, chunk_size, digest) VALUES (?,?,?,?)" - (rcvFileId, chunkNo, chunkSize, digest) - insertedRowId db - insertReplica :: Int -> FileChunkReplica -> Int64 -> IO () - insertReplica replicaNo FileChunkReplica {server, replicaId, replicaKey} chunkId = do - srvId <- createXFTPServer_ db server - DB.execute - db - "INSERT INTO rcv_file_chunk_replicas (replica_number, rcv_file_chunk_id, xftp_server_id, replica_id, replica_key) VALUES (?,?,?,?,?)" - (replicaNo, chunkId, srvId, replicaId, replicaKey) + "INSERT INTO rcv_files (rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, redirect_id, redirect_entity_id, redirect_digest, redirect_size) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + ((rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, fileKey <$> cfArgs, fileNonce <$> cfArgs, RFSReceiving, redirectId_, redirectEntityId_, redirectDigest_, redirectSize_)) + rcvFileId <- liftIO $ insertedRowId db + pure (rcvFileEntityId, rcvFileId) + +insertRcvFileChunk :: DB.Connection -> FileChunk -> DBRcvFileId -> IO Int64 +insertRcvFileChunk db FileChunk {chunkNo, chunkSize, digest} rcvFileId = do + DB.execute + db + "INSERT INTO rcv_file_chunks (rcv_file_id, chunk_no, chunk_size, digest) VALUES (?,?,?,?)" + (rcvFileId, chunkNo, chunkSize, digest) + insertedRowId db + +insertRcvFileChunkReplica :: DB.Connection -> Int -> FileChunkReplica -> Int64 -> IO () +insertRcvFileChunkReplica db replicaNo FileChunkReplica {server, replicaId, replicaKey} chunkId = do + srvId <- createXFTPServer_ db server + DB.execute + db + "INSERT INTO rcv_file_chunk_replicas (replica_number, rcv_file_chunk_id, xftp_server_id, replica_id, replica_key) VALUES (?,?,?,?,?)" + (replicaNo, chunkId, srvId, replicaId, replicaKey) getRcvFileByEntityId :: DB.Connection -> RcvFileId -> IO (Either StoreError RcvFile) getRcvFileByEntityId db rcvFileEntityId = runExceptT $ do @@ -2318,17 +2349,21 @@ getRcvFile db rcvFileId = runExceptT $ do DB.query db [sql| - SELECT rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, deleted + SELECT rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, deleted, redirect_id, redirect_entity_id, redirect_size, redirect_digest FROM rcv_files WHERE rcv_file_id = ? |] (Only rcvFileId) where - toFile :: (RcvFileId, UserId, FileSize Int64, FileDigest, C.SbKey, C.CbNonce, FileSize Word32, FilePath, Maybe FilePath) :. (FilePath, Maybe C.SbKey, Maybe C.CbNonce, RcvFileStatus, Bool) -> RcvFile - toFile ((rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, saveKey_, saveNonce_, status, deleted)) = + toFile :: (RcvFileId, UserId, FileSize Int64, FileDigest, C.SbKey, C.CbNonce, FileSize Word32, FilePath, Maybe FilePath) :. (FilePath, Maybe C.SbKey, Maybe C.CbNonce, RcvFileStatus, Bool, Maybe DBRcvFileId, Maybe RcvFileId, Maybe (FileSize Int64), Maybe FileDigest) -> RcvFile + toFile ((rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, saveKey_, saveNonce_, status, deleted, redirectDbId, redirectEntityId, redirectSize_, redirectDigest_)) = let cfArgs = CFArgs <$> saveKey_ <*> saveNonce_ saveFile = CryptoFile savePath cfArgs - in RcvFile {rcvFileId, rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath, saveFile, status, deleted, chunks = []} + redirect = RcvFileRedirect + <$> redirectDbId + <*> redirectEntityId + <*> (RedirectFileInfo <$> redirectSize_ <*> redirectDigest_) + in RcvFile {rcvFileId, rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, redirect, prefixPath, tmpPath, saveFile, status, deleted, chunks = []} getChunks :: RcvFileId -> UserId -> FilePath -> IO [RcvFileChunk] getChunks rcvFileEntityId userId fileTmpPath = do chunks <- @@ -2394,6 +2429,14 @@ updateRcvFileComplete db rcvFileId = do updatedAt <- getCurrentTime DB.execute db "UPDATE rcv_files SET tmp_path = NULL, status = ?, updated_at = ? WHERE rcv_file_id = ?" (RFSComplete, updatedAt, rcvFileId) +updateRcvFileRedirect :: DB.Connection -> DBRcvFileId -> FileDescription 'FRecipient -> IO (Either StoreError ()) +updateRcvFileRedirect db rcvFileId FileDescription {key, nonce, chunkSize, chunks} = runExceptT $ do + updatedAt <- liftIO getCurrentTime + liftIO $ DB.execute db "UPDATE rcv_files SET key = ?, nonce = ?, chunk_size = ?, updated_at = ? WHERE rcv_file_id = ?" (key, nonce, chunkSize, updatedAt, rcvFileId) + liftIO $ forM_ chunks $ \fc@FileChunk {replicas} -> do + chunkId <- insertRcvFileChunk db fc rcvFileId + forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertRcvFileChunkReplica db rno replica chunkId + updateRcvFileNoTmpPath :: DB.Connection -> DBRcvFileId -> IO () updateRcvFileNoTmpPath db rcvFileId = do updatedAt <- getCurrentTime @@ -2541,13 +2584,18 @@ getRcvFilesExpired db ttl = do |] (Only cutoffTs) -createSndFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> CryptoFile -> Int -> FilePath -> C.SbKey -> C.CbNonce -> IO (Either StoreError SndFileId) -createSndFile db gVar userId (CryptoFile path cfArgs) numRecipients prefixPath key nonce = +createSndFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> CryptoFile -> Int -> FilePath -> C.SbKey -> C.CbNonce -> Maybe RedirectFileInfo -> IO (Either StoreError SndFileId) +createSndFile db gVar userId (CryptoFile path cfArgs) numRecipients prefixPath key nonce redirect_ = createWithRandomId gVar $ \sndFileEntityId -> DB.execute db - "INSERT INTO snd_files (snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, prefix_path, key, nonce, status) VALUES (?,?,?,?,?,?,?,?,?,?)" - (sndFileEntityId, userId, path, fileKey <$> cfArgs, fileNonce <$> cfArgs, numRecipients, prefixPath, key, nonce, SFSNew) + "INSERT INTO snd_files (snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, prefix_path, key, nonce, status, redirect_size, redirect_digest) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)" + ((sndFileEntityId, userId, path, fileKey <$> cfArgs, fileNonce <$> cfArgs, numRecipients) :. (prefixPath, key, nonce, SFSNew, redirectSize_, redirectDigest_)) + where + (redirectSize_, redirectDigest_) = + case redirect_ of + Nothing -> (Nothing, Nothing) + Just RedirectFileInfo {size, digest} -> (Just size, Just digest) getSndFileByEntityId :: DB.Connection -> SndFileId -> IO (Either StoreError SndFile) getSndFileByEntityId db sndFileEntityId = runExceptT $ do @@ -2571,17 +2619,18 @@ getSndFile db sndFileId = runExceptT $ do DB.query db [sql| - SELECT snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, digest, prefix_path, key, nonce, status, deleted + SELECT snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, digest, prefix_path, key, nonce, status, deleted, redirect_size, redirect_digest FROM snd_files WHERE snd_file_id = ? |] (Only sndFileId) where - toFile :: (SndFileId, UserId, FilePath, Maybe C.SbKey, Maybe C.CbNonce, Int, Maybe FileDigest, Maybe FilePath, C.SbKey, C.CbNonce, SndFileStatus, Bool) -> SndFile - toFile (sndFileEntityId, userId, srcPath, srcKey_, srcNonce_, numRecipients, digest, prefixPath, key, nonce, status, deleted) = + toFile :: (SndFileId, UserId, FilePath, Maybe C.SbKey, Maybe C.CbNonce, Int, Maybe FileDigest, Maybe FilePath, C.SbKey, C.CbNonce) :. (SndFileStatus, Bool, Maybe (FileSize Int64), Maybe FileDigest) -> SndFile + toFile ((sndFileEntityId, userId, srcPath, srcKey_, srcNonce_, numRecipients, digest, prefixPath, key, nonce) :. (status, deleted, redirectSize_, redirectDigest_)) = let cfArgs = CFArgs <$> srcKey_ <*> srcNonce_ srcFile = CryptoFile srcPath cfArgs - in SndFile {sndFileId, sndFileEntityId, userId, srcFile, numRecipients, digest, prefixPath, key, nonce, status, deleted, chunks = []} + redirect = RedirectFileInfo <$> redirectSize_ <*> redirectDigest_ + in SndFile {sndFileId, sndFileEntityId, userId, srcFile, numRecipients, digest, prefixPath, key, nonce, status, deleted, redirect, chunks = []} getChunks :: SndFileId -> UserId -> Int -> FilePath -> IO [SndFileChunk] getChunks sndFileEntityId userId numRecipients filePrefixPath = do chunks <- diff --git a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations.hs b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations.hs index 2d8ad3a8c..83c900f72 100644 --- a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations.hs +++ b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations.hs @@ -68,6 +68,7 @@ import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20230829_crypto_files import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20231222_command_created_at import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20231225_failed_work_items import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240121_message_delivery_indexes +import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240124_file_redirect import Simplex.Messaging.Encoding.String import Simplex.Messaging.Parsers (dropPrefix, sumTypeJSON) import Simplex.Messaging.Transport.Client (TransportHost) @@ -104,7 +105,8 @@ schemaMigrations = ("m20230829_crypto_files", m20230829_crypto_files, Just down_m20230829_crypto_files), ("m20231222_command_created_at", m20231222_command_created_at, Just down_m20231222_command_created_at), ("m20231225_failed_work_items", m20231225_failed_work_items, Just down_m20231225_failed_work_items), - ("m20240121_message_delivery_indexes", m20240121_message_delivery_indexes, Just down_m20240121_message_delivery_indexes) + ("m20240121_message_delivery_indexes", m20240121_message_delivery_indexes, Just down_m20240121_message_delivery_indexes), + ("m20240124_file_redirect", m20240124_file_redirect, Just down_m20240124_file_redirect) ] -- | The list of migrations in ascending order by date diff --git a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20240124_file_redirect.hs b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20240124_file_redirect.hs new file mode 100644 index 000000000..e02932d63 --- /dev/null +++ b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20240124_file_redirect.hs @@ -0,0 +1,34 @@ +{-# LANGUAGE QuasiQuotes #-} + +module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240124_file_redirect where + +import Database.SQLite.Simple (Query) +import Database.SQLite.Simple.QQ (sql) + +m20240124_file_redirect :: Query +m20240124_file_redirect = + [sql| +ALTER TABLE snd_files ADD COLUMN redirect_size INTEGER; +ALTER TABLE snd_files ADD COLUMN redirect_digest BLOB; + +ALTER TABLE rcv_files ADD COLUMN redirect_id INTEGER REFERENCES rcv_files ON DELETE CASCADE; +ALTER TABLE rcv_files ADD COLUMN redirect_entity_id BLOB; +ALTER TABLE rcv_files ADD COLUMN redirect_size INTEGER; +ALTER TABLE rcv_files ADD COLUMN redirect_digest BLOB; + +CREATE INDEX idx_rcv_files_redirect_id on rcv_files(redirect_id); +|] + +down_m20240124_file_redirect :: Query +down_m20240124_file_redirect = + [sql| +DROP INDEX idx_rcv_files_redirect_id; + +ALTER TABLE snd_files DROP COLUMN redirect_size; +ALTER TABLE snd_files DROP COLUMN redirect_digest; + +ALTER TABLE rcv_files DROP COLUMN redirect_id; +ALTER TABLE rcv_files DROP COLUMN redirect_entity_id; +ALTER TABLE rcv_files DROP COLUMN redirect_size; +ALTER TABLE rcv_files DROP COLUMN redirect_digest; +|] diff --git a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql index 4ff419401..32df1520d 100644 --- a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql +++ b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql @@ -279,6 +279,10 @@ CREATE TABLE rcv_files( save_file_key BLOB, save_file_nonce BLOB, failed INTEGER DEFAULT 0, + redirect_id INTEGER REFERENCES rcv_files ON DELETE CASCADE, + redirect_entity_id BLOB, + redirect_size INTEGER, + redirect_digest BLOB, UNIQUE(rcv_file_entity_id) ); CREATE TABLE rcv_file_chunks( @@ -322,7 +326,9 @@ CREATE TABLE snd_files( , src_file_key BLOB, src_file_nonce BLOB, - failed INTEGER DEFAULT 0 + failed INTEGER DEFAULT 0, + redirect_size INTEGER, + redirect_digest BLOB ); CREATE TABLE snd_file_chunks( snd_file_chunk_id INTEGER PRIMARY KEY, @@ -508,3 +514,4 @@ CREATE INDEX idx_snd_message_deliveries_expired ON snd_message_deliveries( failed, internal_id ); +CREATE INDEX idx_rcv_files_redirect_id on rcv_files(redirect_id); diff --git a/src/Simplex/Messaging/Notifications/Server/Main.hs b/src/Simplex/Messaging/Notifications/Server/Main.hs index 53d134abd..0462cf154 100644 --- a/src/Simplex/Messaging/Notifications/Server/Main.hs +++ b/src/Simplex/Messaging/Notifications/Server/Main.hs @@ -30,7 +30,7 @@ import System.IO (BufferMode (..), hSetBuffering, stderr, stdout) import Text.Read (readMaybe) ntfServerVersion :: String -ntfServerVersion = "1.7.2.0" +ntfServerVersion = "1.7.3.0" defaultSMPBatchDelay :: Int defaultSMPBatchDelay = 10000 @@ -42,6 +42,10 @@ ntfServerCLI cfgPath logPath = doesFileExist iniFile >>= \case True -> exitError $ "Error: server is already initialized (" <> iniFile <> " exists).\nRun `" <> executableName <> " start`." _ -> initializeServer opts + OnlineCert certOpts -> + doesFileExist iniFile >>= \case + True -> genOnline cfgPath certOpts + _ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`." Start -> doesFileExist iniFile >>= \case True -> readIniFile iniFile >>= either exitError runServer @@ -143,6 +147,7 @@ ntfServerCLI cfgPath logPath = data CliCommand = Init InitOptions + | OnlineCert CertOptions | Start | Delete @@ -158,6 +163,7 @@ cliCommandP :: FilePath -> FilePath -> FilePath -> Parser CliCommand cliCommandP cfgPath logPath iniFile = hsubparser ( command "init" (info (Init <$> initP) (progDesc $ "Initialize server - creates " <> cfgPath <> " and " <> logPath <> " directories and configuration files")) + <> command "cert" (info (OnlineCert <$> certOptionsP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")")) <> command "start" (info (pure Start) (progDesc $ "Start server (configuration: " <> iniFile <> ")")) <> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files")) ) diff --git a/src/Simplex/Messaging/Protocol.hs b/src/Simplex/Messaging/Protocol.hs index f2571a9d4..e8be3b126 100644 --- a/src/Simplex/Messaging/Protocol.hs +++ b/src/Simplex/Messaging/Protocol.hs @@ -173,11 +173,12 @@ import Data.String import Data.Time.Clock.System (SystemTime (..)) import Data.Type.Equality import GHC.TypeLits (ErrorMessage (..), TypeError, type (+)) -import Network.Socket (HostName, ServiceName) +import Network.Socket (ServiceName) import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Encoding import Simplex.Messaging.Encoding.String import Simplex.Messaging.Parsers +import Simplex.Messaging.ServiceScheme import Simplex.Messaging.Transport import Simplex.Messaging.Transport.Client (TransportHost, TransportHosts (..)) import Simplex.Messaging.Util (bshow, eitherToMaybe, (<$?>)) @@ -915,16 +916,6 @@ serverStrP = do where portP = show <$> (A.char ':' *> (A.decimal :: Parser Int)) -data SrvLoc = SrvLoc HostName ServiceName - deriving (Eq, Ord, Show) - -instance StrEncoding SrvLoc where - strEncode (SrvLoc host port) = B.pack $ host <> if null port then "" else ':' : port - strP = SrvLoc <$> host <*> (port <|> pure "") - where - host = B.unpack <$> A.takeWhile1 (A.notInClass ":#,;/ ") - port = show <$> (A.char ':' *> (A.decimal :: Parser Int)) - -- | Transmission correlation ID. newtype CorrId = CorrId {bs :: ByteString} deriving (Eq, Ord, Show) diff --git a/src/Simplex/Messaging/Server/CLI.hs b/src/Simplex/Messaging/Server/CLI.hs index 249e9dcc0..9531a2ca5 100644 --- a/src/Simplex/Messaging/Server/CLI.hs +++ b/src/Simplex/Messaging/Server/CLI.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE ApplicativeDo #-} {-# LANGUAGE DataKinds #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE FlexibleContexts #-} @@ -10,6 +11,7 @@ module Simplex.Messaging.Server.CLI where import Control.Monad +import Data.ASN1.Types (asn1CharacterToString) import Data.ByteString.Char8 (ByteString) import qualified Data.ByteString.Char8 as B import Data.Either (fromRight) @@ -17,6 +19,8 @@ import Data.Ini (Ini, lookupValue) import Data.Text (Text) import qualified Data.Text as T import Data.Text.Encoding (encodeUtf8) +import qualified Data.X509 as X +import qualified Data.X509.File as XF import Data.X509.Validation (Fingerprint (..)) import Network.Socket (HostName, ServiceName) import Options.Applicative @@ -27,12 +31,14 @@ import Simplex.Messaging.Transport.Server (loadFingerprint) import Simplex.Messaging.Transport.WebSockets (WS) import Simplex.Messaging.Util (eitherToMaybe, whenM) import System.Directory (doesDirectoryExist, listDirectory, removeDirectoryRecursive, removePathForcibly) +import System.Environment (lookupEnv) import System.Exit (exitFailure) import System.FilePath (combine) import System.IO (IOMode (..), hFlush, hGetLine, stdout, withFile) import System.Process (readCreateProcess, shell) +import Text.Read (readMaybe) -exitError :: String -> IO () +exitError :: String -> IO a exitError msg = putStrLn msg >> exitFailure confirmOrExit :: String -> IO () @@ -84,14 +90,18 @@ getCliCommand' cmdP version = versionOption = infoOption version (long "version" <> short 'v' <> help "Show version") createServerX509 :: FilePath -> X509Config -> IO ByteString -createServerX509 cfgPath x509cfg = do - createOpensslCaConf - createOpensslServerConf +createServerX509 = createServerX509_ True + +createServerX509_ :: Bool -> FilePath -> X509Config -> IO ByteString +createServerX509_ createCA cfgPath x509cfg = do let alg = show $ signAlgorithm (x509cfg :: X509Config) -- CA certificate (identity/offline) - run $ "openssl genpkey -algorithm " <> alg <> " -out " <> c caKeyFile - run $ "openssl req -new -x509 -days 999999 -config " <> c opensslCaConfFile <> " -extensions v3 -key " <> c caKeyFile <> " -out " <> c caCrtFile + when createCA $ do + createOpensslCaConf + run $ "openssl genpkey -algorithm " <> alg <> " -out " <> c caKeyFile + run $ "openssl req -new -x509 -days 999999 -config " <> c opensslCaConfFile <> " -extensions v3 -key " <> c caKeyFile <> " -out " <> c caCrtFile -- server certificate (online) + createOpensslServerConf run $ "openssl genpkey -algorithm " <> alg <> " -out " <> c serverKeyFile run $ "openssl req -new -config " <> c opensslServerConfFile <> " -reqexts v3 -key " <> c serverKeyFile <> " -out " <> c serverCsrFile run $ "openssl x509 -req -days 999999 -extfile " <> c opensslServerConfFile <> " -extensions v3 -in " <> c serverCsrFile <> " -CA " <> c caCrtFile <> " -CAkey " <> c caKeyFile <> " -CAcreateserial -out " <> c serverCrtFile @@ -131,6 +141,59 @@ createServerX509 cfgPath x509cfg = do withFile (c fingerprintFile) WriteMode (`B.hPutStrLn` strEncode fp) pure fp +data CertOptions = CertOptions + { signAlgorithm_ :: Maybe SignAlgorithm, + commonName_ :: Maybe HostName + } + deriving (Show) + +certOptionsP :: Parser CertOptions +certOptionsP = do + signAlgorithm_ <- + optional $ + option + (maybeReader readMaybe) + ( long "sign-algorithm" + <> short 'a' + <> help "Set new signature algorithm used for TLS certificates: ED25519, ED448" + <> metavar "ALG" + ) + commonName_ <- + optional $ + strOption + ( long "cn" + <> help + "Set new Common Name for TLS online certificate" + <> metavar "FQDN" + ) + pure CertOptions {signAlgorithm_, commonName_} + +genOnline :: FilePath -> CertOptions -> IO () +genOnline cfgPath CertOptions {signAlgorithm_, commonName_} = do + (signAlgorithm, commonName) <- + case (signAlgorithm_, commonName_) of + (Just alg, Just cn) -> pure (alg, cn) + _ -> + XF.readSignedObject certPath >>= \case + [old] -> either exitError pure . fromX509 . X.signedObject $ X.getSigned old + [] -> exitError $ "No certificate found at " <> certPath + _ -> exitError $ "Too many certificates at " <> certPath + let x509cfg = defaultX509Config {signAlgorithm, commonName} + void $ createServerX509_ False cfgPath x509cfg + putStrLn "Generated new server credentials" + warnCAPrivateKeyFile cfgPath x509cfg + where + certPath = combine cfgPath $ serverCrtFile defaultX509Config + fromX509 X.Certificate {certSignatureAlg, certSubjectDN} = (,) <$> maybe oldAlg Right signAlgorithm_ <*> maybe oldCN Right commonName_ + where + oldAlg = case certSignatureAlg of + X.SignatureALG_IntrinsicHash X.PubKeyALG_Ed448 -> Right ED448 + X.SignatureALG_IntrinsicHash X.PubKeyALG_Ed25519 -> Right ED25519 + alg -> Left $ "Unexpected signature algorithm " <> show alg + oldCN = case X.getDnElement X.DnCommonName certSubjectDN of + Nothing -> Left "Certificate subject has no CN element" + Just cn -> maybe (Left "Certificate subject CN decoding failed") Right $ asn1CharacterToString cn + warnCAPrivateKeyFile :: FilePath -> X509Config -> IO () warnCAPrivateKeyFile cfgPath X509Config {caKeyFile} = putStrLn $ @@ -235,3 +298,6 @@ printServiceInfo serverVersion srv@(ProtoServerWithAuth ProtocolServer {keyHash} clearDirIfExists :: FilePath -> IO () clearDirIfExists path = whenM (doesDirectoryExist path) $ listDirectory path >>= mapM_ (removePathForcibly . combine path) + +getEnvPath :: String -> FilePath -> IO FilePath +getEnvPath name def = maybe def (\case "" -> def; f -> f) <$> lookupEnv name diff --git a/src/Simplex/Messaging/Server/Main.hs b/src/Simplex/Messaging/Server/Main.hs index e02256aab..13b2af4a3 100644 --- a/src/Simplex/Messaging/Server/Main.hs +++ b/src/Simplex/Messaging/Server/Main.hs @@ -41,6 +41,10 @@ smpServerCLI cfgPath logPath = doesFileExist iniFile >>= \case True -> exitError $ "Error: server is already initialized (" <> iniFile <> " exists).\nRun `" <> executableName <> " start`." _ -> initializeServer opts + OnlineCert certOpts -> + doesFileExist iniFile >>= \case + True -> genOnline cfgPath certOpts + _ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`." Start -> doesFileExist iniFile >>= \case True -> readIniFile iniFile >>= either exitError runServer @@ -56,8 +60,8 @@ smpServerCLI cfgPath logPath = defaultServerPort = "5223" executableName = "smp-server" storeLogFilePath = combine logPath "smp-server-store.log" - initializeServer opts - | scripted opts = initialize opts + initializeServer opts@InitOptions {ip, fqdn, scripted} + | scripted = initialize opts | otherwise = do putStrLn "Use `smp-server init -h` for available options." void $ withPrompt "SMP server will be initialized (press Enter)" getLine @@ -65,9 +69,9 @@ smpServerCLI cfgPath logPath = logStats <- onOffPrompt "Enable logging daily statistics" False putStrLn "Require a password to create new messaging queues?" password <- withPrompt "'r' for random (default), 'n' - no password, or enter password: " serverPassword - let host = fromMaybe (ip opts) (fqdn opts) + let host = fromMaybe ip fqdn host' <- withPrompt ("Enter server FQDN or IP address for certificate (" <> host <> "): ") getLine - initialize opts {enableStoreLog, logStats, fqdn = if null host' then fqdn opts else Just host', password} + initialize opts {enableStoreLog, logStats, fqdn = if null host' then fqdn else Just host', password} where serverPassword = getLine >>= \case @@ -78,7 +82,7 @@ smpServerCLI cfgPath logPath = case strDecode $ encodeUtf8 $ T.pack s of Right auth -> pure . Just $ ServerPassword auth _ -> putStrLn "Invalid password. Only latin letters, digits and symbols other than '@' and ':' are allowed" >> serverPassword - initialize InitOptions {enableStoreLog, logStats, signAlgorithm, ip, fqdn, password} = do + initialize InitOptions {enableStoreLog, logStats, signAlgorithm, password} = do clearDirIfExists cfgPath clearDirIfExists logPath createDirectoryIfMissing True cfgPath @@ -210,6 +214,7 @@ smpServerCLI cfgPath logPath = data CliCommand = Init InitOptions + | OnlineCert CertOptions | Start | Delete @@ -231,6 +236,7 @@ cliCommandP :: FilePath -> FilePath -> FilePath -> Parser CliCommand cliCommandP cfgPath logPath iniFile = hsubparser ( command "init" (info (Init <$> initP) (progDesc $ "Initialize server - creates " <> cfgPath <> " and " <> logPath <> " directories and configuration files")) + <> command "cert" (info (OnlineCert <$> certOptionsP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")")) <> command "start" (info (pure Start) (progDesc $ "Start server (configuration: " <> iniFile <> ")")) <> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files")) ) @@ -255,7 +261,7 @@ cliCommandP cfgPath logPath iniFile = ( long "sign-algorithm" <> short 'a' <> help "Signature algorithm used for TLS certificates: ED25519, ED448" - <> value ED448 + <> value ED25519 <> showDefault <> metavar "ALG" ) @@ -295,3 +301,4 @@ cliCommandP cfgPath logPath iniFile = pure InitOptions {enableStoreLog, logStats, signAlgorithm, ip, fqdn, password, scripted} parseBasicAuth :: ReadM ServerPassword parseBasicAuth = eitherReader $ fmap ServerPassword . strDecode . B.pack + diff --git a/src/Simplex/Messaging/ServiceScheme.hs b/src/Simplex/Messaging/ServiceScheme.hs new file mode 100644 index 000000000..3cd828aa7 --- /dev/null +++ b/src/Simplex/Messaging/ServiceScheme.hs @@ -0,0 +1,35 @@ +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE OverloadedStrings #-} + +module Simplex.Messaging.ServiceScheme where + +import Control.Applicative ((<|>)) +import qualified Data.Attoparsec.ByteString.Char8 as A +import qualified Data.ByteString.Char8 as B +import Data.Functor (($>)) +import Network.Socket (HostName, ServiceName) +import Simplex.Messaging.Encoding.String (StrEncoding (..)) + +data ServiceScheme = SSSimplex | SSAppServer SrvLoc + deriving (Eq, Show) + +instance StrEncoding ServiceScheme where + strEncode = \case + SSSimplex -> "simplex:" + SSAppServer srv -> "https://" <> strEncode srv + strP = + "simplex:" $> SSSimplex + <|> "https://" *> (SSAppServer <$> strP) + +data SrvLoc = SrvLoc HostName ServiceName + deriving (Eq, Ord, Show) + +instance StrEncoding SrvLoc where + strEncode (SrvLoc host port) = B.pack $ host <> if null port then "" else ':' : port + strP = SrvLoc <$> host <*> (port <|> pure "") + where + host = B.unpack <$> A.takeWhile1 (A.notInClass ":#,;/ ") + port = show <$> (A.char ':' *> (A.decimal :: A.Parser Int)) + +simplexChat :: ServiceScheme +simplexChat = SSAppServer $ SrvLoc "simplex.chat" "" diff --git a/tests/AgentTests/ConnectionRequestTests.hs b/tests/AgentTests/ConnectionRequestTests.hs index 9548443a7..e6668a1dd 100644 --- a/tests/AgentTests/ConnectionRequestTests.hs +++ b/tests/AgentTests/ConnectionRequestTests.hs @@ -13,6 +13,7 @@ import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Crypto.Ratchet import Simplex.Messaging.Encoding.String import Simplex.Messaging.Protocol (ProtocolServer (..), supportedSMPClientVRange) +import Simplex.Messaging.ServiceScheme (ServiceScheme (..)) import Simplex.Messaging.Version import Test.Hspec @@ -51,7 +52,7 @@ testDhKeyStrUri = urlEncode True testDhKeyStr connReqData :: ConnReqUriData connReqData = ConnReqUriData - { crScheme = CRSSimplex, + { crScheme = SSSimplex, crAgentVRange = mkVersionRange 1 1, crSmpQueues = [queueV1], crClientData = Nothing diff --git a/tests/AgentTests/SQLiteTests.hs b/tests/AgentTests/SQLiteTests.hs index 98fb47b80..9d258bdf7 100644 --- a/tests/AgentTests/SQLiteTests.hs +++ b/tests/AgentTests/SQLiteTests.hs @@ -658,7 +658,8 @@ rcvFileDescr1 = chunkSize = defaultChunkSize, replicas = [FileChunkReplica {server = xftpServer1, replicaId, replicaKey = testFileReplicaKey}] } - ] + ], + redirect = Nothing } where defaultChunkSize = FileSize $ mb 8 @@ -716,9 +717,9 @@ testGetNextSndFileToPrepare st = do withTransaction st $ \db -> do Right Nothing <- getNextSndFileToPrepare db 86400 - Right _ <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce + Right _ <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce Nothing DB.execute_ db "UPDATE snd_files SET status = 'new', num_recipients = 'bad' WHERE snd_file_id = 1" - Right fId2 <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce + Right fId2 <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce Nothing DB.execute_ db "UPDATE snd_files SET status = 'new' WHERE snd_file_id = 2" Left e <- getNextSndFileToPrepare db 86400 @@ -744,12 +745,12 @@ testGetNextSndChunkToUpload st = do Right Nothing <- getNextSndChunkToUpload db xftpServer1 86400 -- create file 1 - Right _ <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce + Right _ <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce Nothing updateSndFileEncrypted db 1 (FileDigest "abc") [(XFTPChunkSpec "filepath" 1 1, FileDigest "ghi")] createSndFileReplica_ db 1 newSndChunkReplica1 DB.execute_ db "UPDATE snd_files SET num_recipients = 'bad' WHERE snd_file_id = 1" -- create file 2 - Right fId2 <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce + Right fId2 <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce Nothing updateSndFileEncrypted db 2 (FileDigest "abc") [(XFTPChunkSpec "filepath" 1 1, FileDigest "ghi")] createSndFileReplica_ db 2 newSndChunkReplica1 diff --git a/tests/CLITests.hs b/tests/CLITests.hs index 40915515b..032c1f748 100644 --- a/tests/CLITests.hs +++ b/tests/CLITests.hs @@ -1,9 +1,12 @@ +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} module CLITests where import Data.Ini (lookupValue, readIniFile) import Data.List (isPrefixOf) +import qualified Data.X509 as X +import qualified Data.X509.File as XF import Simplex.FileTransfer.Server.Main (xftpServerCLI, xftpServerVersion) import Simplex.Messaging.Notifications.Server.Main import Simplex.Messaging.Server.Main @@ -11,6 +14,7 @@ import Simplex.Messaging.Transport (simplexMQVersion) import Simplex.Messaging.Util (catchAll_) import System.Directory (doesFileExist) import System.Environment (withArgs) +import System.FilePath (()) import System.IO.Silently (capture_) import System.Timeout (timeout) import Test.Hspec @@ -51,6 +55,7 @@ cliTests = do smpServerTest :: Bool -> Bool -> IO () smpServerTest storeLog basicAuth = do + -- init capture_ (withArgs (["init", "-y"] <> ["-l" | storeLog] <> ["--no-password" | not basicAuth]) $ smpServerCLI cfgPath logPath) >>= (`shouldSatisfy` (("Server initialized, you can modify configuration in " <> cfgPath <> "/smp-server.ini") `isPrefixOf`)) Right ini <- readIniFile $ cfgPath <> "/smp-server.ini" @@ -61,12 +66,30 @@ smpServerTest storeLog basicAuth = do lookupValue "AUTH" "new_queues" ini `shouldBe` Right "on" lookupValue "INACTIVE_CLIENTS" "disconnect" ini `shouldBe` Right "off" doesFileExist (cfgPath <> "/ca.key") `shouldReturn` True + -- start r <- lines <$> capture_ (withArgs ["start"] $ (100000 `timeout` smpServerCLI cfgPath logPath) `catchAll_` pure (Just ())) r `shouldContain` ["SMP server v" <> simplexMQVersion] r `shouldContain` (if storeLog then ["Store log: " <> logPath <> "/smp-server-store.log"] else ["Store log disabled."]) r `shouldContain` ["Listening on port 5223 (TLS)..."] r `shouldContain` ["not expiring inactive clients"] r `shouldContain` (if basicAuth then ["creating new queues requires password"] else ["creating new queues allowed"]) + -- cert + let certPath = cfgPath "server.crt" + oldCrt@X.Certificate {} <- + XF.readSignedObject certPath >>= \case + [cert] -> pure . X.signedObject $ X.getSigned cert + _ -> error "bad crt format" + r' <- lines <$> capture_ (withArgs ["cert"] $ (100000 `timeout` smpServerCLI cfgPath logPath) `catchAll_` pure (Just ())) + r' `shouldContain` ["Generated new server credentials"] + newCrt <- + XF.readSignedObject certPath >>= \case + [cert] -> pure . X.signedObject $ X.getSigned cert + _ -> error "bad crt format after cert" + X.certSignatureAlg oldCrt `shouldBe` X.certSignatureAlg newCrt + X.certSubjectDN oldCrt `shouldBe` X.certSubjectDN newCrt + X.certSerial oldCrt `shouldNotBe` X.certSerial newCrt + X.certPubKey oldCrt `shouldNotBe` X.certPubKey newCrt + -- delete capture_ (withStdin "Y" . withArgs ["delete"] $ smpServerCLI cfgPath logPath) >>= (`shouldSatisfy` ("WARNING: deleting the server will make all queues inaccessible" `isPrefixOf`)) doesFileExist (cfgPath <> "/ca.key") `shouldReturn` False diff --git a/tests/FileDescriptionTests.hs b/tests/FileDescriptionTests.hs index 676882bcd..61dd638fe 100644 --- a/tests/FileDescriptionTests.hs +++ b/tests/FileDescriptionTests.hs @@ -13,16 +13,20 @@ import Simplex.FileTransfer.Description import Simplex.FileTransfer.Protocol import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Encoding.String (StrEncoding (..)) +import Simplex.Messaging.ServiceScheme (ServiceScheme (..)) import System.Directory (removeFile) import Test.Hspec fileDescriptionTests :: Spec -fileDescriptionTests = +fileDescriptionTests = do describe "file description parsing / serializing" $ do it "parse YAML file description" testParseYAMLFileDescription it "serialize YAML file description" testSerializeYAMLFileDescription it "parse file description" testParseFileDescription it "serialize file description" testSerializeFileDescription + describe "file description URIs" $ do + it "round trip file description URI" testFileDescriptionURI + it "round trip file description URI with extra JSON" testFileDescriptionURIExtras fileDescPath :: FilePath fileDescPath = "tests/fixtures/file_description.yaml" @@ -82,7 +86,8 @@ fileDesc = FileChunkReplica {server = "xftp://abc=@example3.com", replicaId, replicaKey} ] } - ] + ], + redirect = Nothing } where defaultChunkSize = FileSize $ mb 8 @@ -128,7 +133,8 @@ yamlFileDesc = "3:YWJj:MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe" ] } - ] + ], + redirect = Nothing } testParseYAMLFileDescription :: IO () @@ -157,6 +163,18 @@ testSerializeFileDescription = withRemoveTmpFile $ do fdExp <- B.readFile fileDescPath fdSer `shouldBe` fdExp +testFileDescriptionURI :: IO () +testFileDescriptionURI = do + vfd <- either fail pure $ validateFileDescription fileDesc + let descr = FileDescriptionURI SSSimplex vfd mempty + strDecode (strEncode descr) `shouldBe` Right descr + +testFileDescriptionURIExtras :: IO () +testFileDescriptionURIExtras = do + vfd <- either fail pure $ validateFileDescription fileDesc + let descr = FileDescriptionURI SSSimplex vfd $ Just "{\"something\":\"extra\",\"more\":true}" + strDecode (strEncode descr) `shouldBe` Right descr + withRemoveTmpFile :: IO () -> IO () withRemoveTmpFile = bracket_ diff --git a/tests/XFTPAgent.hs b/tests/XFTPAgent.hs index 07ac62da6..6ac64a4c1 100644 --- a/tests/XFTPAgent.hs +++ b/tests/XFTPAgent.hs @@ -1,14 +1,15 @@ {-# LANGUAGE DataKinds #-} {-# LANGUAGE GADTs #-} +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PatternSynonyms #-} {-# LANGUAGE ScopedTypeVariables #-} module XFTPAgent where import AgentTests.FunctionalAPITests (get, getSMPAgentClient', rfGet, runRight, runRight_, sfGet) -import Control.Concurrent (threadDelay) -import Control.Concurrent.STM + import Control.Logger.Simple import Control.Monad import Control.Monad.Except @@ -19,10 +20,10 @@ import Data.Int (Int64) import Data.List (find, isSuffixOf) import Data.Maybe (fromJust) import SMPAgentClient (agentCfg, initAgentServers, testDB, testDB2, testDB3) -import Simplex.FileTransfer.Description +import Simplex.FileTransfer.Description (FileDescription (..), FileDescriptionURI (..), ValidFileDescription, fileDescriptionURI, mb, qrSizeLimit, pattern ValidFileDescription) import Simplex.FileTransfer.Protocol (FileParty (..), XFTPErrorType (AUTH)) import Simplex.FileTransfer.Server.Env (XFTPServerConfig (..)) -import Simplex.Messaging.Agent (AgentClient, disconnectAgentClient, testProtocolServer, xftpDeleteRcvFile, xftpDeleteSndFileInternal, xftpDeleteSndFileRemote, xftpReceiveFile, xftpSendFile, xftpStartWorkers) +import Simplex.Messaging.Agent (AgentClient, disconnectAgentClient, testProtocolServer, xftpDeleteRcvFile, xftpDeleteSndFileInternal, xftpDeleteSndFileRemote, xftpReceiveFile, xftpSendDescription, xftpSendFile, xftpStartWorkers) import Simplex.Messaging.Agent.Client (ProtocolTestFailure (..), ProtocolTestStep (..)) import Simplex.Messaging.Agent.Protocol (ACommand (..), AgentErrorType (..), BrokerErrorType (..), RcvFileId, SndFileId, noAuthSrv) import qualified Simplex.Messaging.Crypto as C @@ -31,10 +32,12 @@ import qualified Simplex.Messaging.Crypto.File as CF import Simplex.Messaging.Encoding.String (StrEncoding (..)) import Simplex.Messaging.Protocol (BasicAuth, ProtoServerWithAuth (..), ProtocolServer (..), XFTPServerWithAuth) import Simplex.Messaging.Server.Expiration (ExpirationConfig (..)) +import Simplex.Messaging.Util (tshow) import System.Directory (doesDirectoryExist, doesFileExist, getFileSize, listDirectory, removeFile) import System.FilePath (()) -import System.Timeout (timeout) import Test.Hspec +import UnliftIO +import UnliftIO.Concurrent import XFTPCLI import XFTPClient @@ -42,6 +45,8 @@ xftpAgentTests :: Spec xftpAgentTests = around_ testBracket . describe "agent XFTP API" $ do it "should send and receive file" testXFTPAgentSendReceive it "should send and receive with encrypted local files" testXFTPAgentSendReceiveEncrypted + it "should send and receive large file with a redirect" testXFTPAgentSendReceiveRedirect + it "should send and receive small file without a redirect" testXFTPAgentSendReceiveNoRedirect it "should resume receiving file after restart" testXFTPAgentReceiveRestore it "should cleanup rcv tmp path after permanent error" testXFTPAgentReceiveCleanup it "should resume sending file after restart" testXFTPAgentSendRestore @@ -135,14 +140,112 @@ testXFTPAgentSendReceiveEncrypted = withXFTPServer $ do xftpDeleteRcvFile rcp rfId disconnectAgentClient rcp +testXFTPAgentSendReceiveRedirect :: HasCallStack => IO () +testXFTPAgentSendReceiveRedirect = withXFTPServer $ do + --- sender + filePathIn <- createRandomFile + let fileSize = mb 17 + totalSize = fileSize + mb 1 + sndr <- getSMPAgentClient' 1 agentCfg initAgentServers testDB + directFileId <- runRight $ xftpSendFile sndr 1 (CryptoFile filePathIn Nothing) 1 + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 4194304 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 8388608 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 12582912 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 16777216 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 17825792 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG totalSize totalSize) + vfdDirect <- + sfGet sndr >>= \case + (_, _, SFDONE _snd (vfd : _)) -> pure vfd + r -> error $ "Expected SFDONE, got " <> show r + redirectFileId <- runRight $ xftpSendDescription sndr 1 vfdDirect 1 + logInfo $ "File sent, sending redirect: " <> tshow redirectFileId + sfGet sndr `shouldReturn` ("", redirectFileId, SFPROG 65536 65536) + vfdRedirect@(ValidFileDescription fdRedirect) <- + sfGet sndr >>= \case + (_, _, SFDONE _snd (vfd : _)) -> pure vfd + r -> error $ "Expected SFDONE, got " <> show r + case fdRedirect of + FileDescription {redirect = Just _} -> pure () + _ -> error "missing RedirectFileInfo" + let uri = strEncode $ fileDescriptionURI vfdRedirect + case strDecode uri of + Left err -> fail err + Right ok -> ok `shouldBe` fileDescriptionURI vfdRedirect + disconnectAgentClient sndr + --- recipient + rcp <- getSMPAgentClient' 2 agentCfg initAgentServers testDB2 + FileDescriptionURI {description} <- either fail pure $ strDecode uri + + rcvFileId <- runRight $ xftpReceiveFile rcp 1 description Nothing + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 65536 totalSize) -- extra RFPROG before switching to real file + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 4194304 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 8388608 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 12582912 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 16777216 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 17825792 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG totalSize totalSize) + out <- + rfGet rcp >>= \case + (_, _, RFDONE out) -> pure out + r -> error $ "Expected RFDONE, got " <> show r + disconnectAgentClient rcp + + inBytes <- B.readFile filePathIn + B.readFile out `shouldReturn` inBytes + +testXFTPAgentSendReceiveNoRedirect :: HasCallStack => IO () +testXFTPAgentSendReceiveNoRedirect = withXFTPServer $ do + --- sender + let fileSize = mb 5 + filePathIn <- createRandomFile_ fileSize "testfile" + sndr <- getSMPAgentClient' 1 agentCfg initAgentServers testDB + directFileId <- runRight $ xftpSendFile sndr 1 (CryptoFile filePathIn Nothing) 1 + let totalSize = fileSize + mb 1 + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 4194304 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 5242880 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG totalSize totalSize) + vfdDirect <- + sfGet sndr >>= \case + (_, _, SFDONE _snd (vfd : _)) -> pure vfd + r -> error $ "Expected SFDONE, got " <> show r + B.putStrLn $ strEncode vfdDirect + let uri = strEncode $ fileDescriptionURI vfdDirect + B.length uri `shouldSatisfy` (< qrSizeLimit) + case strDecode uri of + Left err -> fail err + Right ok -> ok `shouldBe` fileDescriptionURI vfdDirect + disconnectAgentClient sndr + --- recipient + rcp <- getSMPAgentClient' 2 agentCfg initAgentServers testDB2 + FileDescriptionURI {description} <- either fail pure $ strDecode uri + let ValidFileDescription FileDescription {redirect} = description + redirect `shouldBe` Nothing + rcvFileId <- runRight $ xftpReceiveFile rcp 1 description Nothing + -- NO extra "RFPROG 65k 65k" before switching to real file + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 4194304 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 5242880 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG totalSize totalSize) + out <- + rfGet rcp >>= \case + (_, _, RFDONE out) -> pure out + r -> error $ "Expected RFDONE, got " <> show r + disconnectAgentClient rcp + + inBytes <- B.readFile filePathIn + B.readFile out `shouldReturn` inBytes + createRandomFile :: HasCallStack => IO FilePath createRandomFile = createRandomFile' "testfile" createRandomFile' :: HasCallStack => FilePath -> IO FilePath -createRandomFile' fileName = do +createRandomFile' = createRandomFile_ (mb 17 :: Integer) + +createRandomFile_ :: (HasCallStack, Integral s, Show s) => s -> FilePath -> IO FilePath +createRandomFile_ size fileName = do let filePath = senderFiles fileName - xftpCLI ["rand", filePath, "17mb"] `shouldReturn` ["File created: " <> filePath] - getFileSize filePath `shouldReturn` mb 17 + xftpCLI ["rand", filePath, show size] `shouldReturn` ["File created: " <> filePath] + getFileSize filePath `shouldReturn` toInteger size pure filePath testSend :: HasCallStack => AgentClient -> FilePath -> ExceptT AgentErrorType IO (SndFileId, ValidFileDescription 'FSender, ValidFileDescription 'FRecipient, ValidFileDescription 'FRecipient) diff --git a/tests/XFTPCLI.hs b/tests/XFTPCLI.hs index 14c62aea5..567db6f9b 100644 --- a/tests/XFTPCLI.hs +++ b/tests/XFTPCLI.hs @@ -152,7 +152,7 @@ testPrepareChunkSizes = do prepareChunkSizes (mb 2 + 1) `shouldBe` [mb 1, mb 1, kb 256] prepareChunkSizes (3 * kb 256 + 1) `shouldBe` [mb 1] prepareChunkSizes (3 * kb 256) `shouldBe` r3 (kb 256) - prepareChunkSizes 1 `shouldBe` [kb 256] + prepareChunkSizes 1 `shouldBe` [kb 64] where r3 = replicate 3 diff --git a/tests/XFTPClient.hs b/tests/XFTPClient.hs index 312135072..af4e6dd36 100644 --- a/tests/XFTPClient.hs +++ b/tests/XFTPClient.hs @@ -102,7 +102,7 @@ testXFTPServerConfig = storeLogFile = Nothing, filesPath = xftpServerFiles, fileSizeQuota = Nothing, - allowedChunkSizes = [kb 128, kb 256, mb 1, mb 4], + allowedChunkSizes = [kb 64, kb 128, kb 256, mb 1, mb 4], allowNewFiles = True, newFileBasicAuth = Nothing, fileExpiration = Just defaultFileExpiration,