From 8de23c15ad089507cd180746a6ca9fd7a3296dc7 Mon Sep 17 00:00:00 2001 From: spaced4ndy <8711996+spaced4ndy@users.noreply.github.com> Date: Mon, 12 Feb 2024 16:51:37 +0400 Subject: [PATCH 1/8] agent: return ntf server in getNtfToken (#986) --- src/Simplex/Messaging/Agent.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs index 36dcee49d..2ffe6f0f7 100644 --- a/src/Simplex/Messaging/Agent.hs +++ b/src/Simplex/Messaging/Agent.hs @@ -375,7 +375,7 @@ checkNtfToken c = withAgentEnv c . checkNtfToken' c deleteNtfToken :: AgentErrorMonad m => AgentClient -> DeviceToken -> m () deleteNtfToken c = withAgentEnv c . deleteNtfToken' c -getNtfToken :: AgentErrorMonad m => AgentClient -> m (DeviceToken, NtfTknStatus, NotificationsMode) +getNtfToken :: AgentErrorMonad m => AgentClient -> m (DeviceToken, NtfTknStatus, NotificationsMode, NtfServer) getNtfToken c = withAgentEnv c $ getNtfToken' c getNtfTokenData :: AgentErrorMonad m => AgentClient -> m NtfToken @@ -1685,10 +1685,10 @@ deleteNtfToken' c deviceToken = deleteNtfSubs c NSCSmpDelete _ -> throwError $ CMD PROHIBITED -getNtfToken' :: AgentMonad m => AgentClient -> m (DeviceToken, NtfTknStatus, NotificationsMode) +getNtfToken' :: AgentMonad m => AgentClient -> m (DeviceToken, NtfTknStatus, NotificationsMode, NtfServer) getNtfToken' c = withStore' c getSavedNtfToken >>= \case - Just NtfToken {deviceToken, ntfTknStatus, ntfMode} -> pure (deviceToken, ntfTknStatus, ntfMode) + Just NtfToken {deviceToken, ntfTknStatus, ntfMode, ntfServer} -> pure (deviceToken, ntfTknStatus, ntfMode, ntfServer) _ -> throwError $ CMD PROHIBITED getNtfTokenData' :: AgentMonad m => AgentClient -> m NtfToken From 57e7c8ef6b73c007384ee3b07edc1fb5615c6300 Mon Sep 17 00:00:00 2001 From: Alexander Bondarenko <486682+dpwiz@users.noreply.github.com> Date: Mon, 12 Feb 2024 12:17:08 -0800 Subject: [PATCH 2/8] smp-server: add cert CLI command to rotate online certificate (#984) * smp-server: add gen-online CLI command * use CN and algo from old certificate * add cert checks to test * rename command * fix test * cert --------- Co-authored-by: Evgeny Poberezkin --- src/Simplex/Messaging/Server/CLI.hs | 16 +++--- src/Simplex/Messaging/Server/Main.hs | 73 +++++++++++++++++++++++++--- tests/CLITests.hs | 23 +++++++++ 3 files changed, 100 insertions(+), 12 deletions(-) diff --git a/src/Simplex/Messaging/Server/CLI.hs b/src/Simplex/Messaging/Server/CLI.hs index 249e9dcc0..26e3dfa42 100644 --- a/src/Simplex/Messaging/Server/CLI.hs +++ b/src/Simplex/Messaging/Server/CLI.hs @@ -32,7 +32,7 @@ import System.FilePath (combine) import System.IO (IOMode (..), hFlush, hGetLine, stdout, withFile) import System.Process (readCreateProcess, shell) -exitError :: String -> IO () +exitError :: String -> IO a exitError msg = putStrLn msg >> exitFailure confirmOrExit :: String -> IO () @@ -84,14 +84,18 @@ getCliCommand' cmdP version = versionOption = infoOption version (long "version" <> short 'v' <> help "Show version") createServerX509 :: FilePath -> X509Config -> IO ByteString -createServerX509 cfgPath x509cfg = do - createOpensslCaConf - createOpensslServerConf +createServerX509 = createServerX509_ True + +createServerX509_ :: Bool -> FilePath -> X509Config -> IO ByteString +createServerX509_ createCA cfgPath x509cfg = do let alg = show $ signAlgorithm (x509cfg :: X509Config) -- CA certificate (identity/offline) - run $ "openssl genpkey -algorithm " <> alg <> " -out " <> c caKeyFile - run $ "openssl req -new -x509 -days 999999 -config " <> c opensslCaConfFile <> " -extensions v3 -key " <> c caKeyFile <> " -out " <> c caCrtFile + when createCA $ do + createOpensslCaConf + run $ "openssl genpkey -algorithm " <> alg <> " -out " <> c caKeyFile + run $ "openssl req -new -x509 -days 999999 -config " <> c opensslCaConfFile <> " -extensions v3 -key " <> c caKeyFile <> " -out " <> c caCrtFile -- server certificate (online) + createOpensslServerConf run $ "openssl genpkey -algorithm " <> alg <> " -out " <> c serverKeyFile run $ "openssl req -new -config " <> c opensslServerConfFile <> " -reqexts v3 -key " <> c serverKeyFile <> " -out " <> c serverCsrFile run $ "openssl x509 -req -days 999999 -extfile " <> c opensslServerConfFile <> " -extensions v3 -in " <> c serverCsrFile <> " -CA " <> c caCrtFile <> " -CAkey " <> c caKeyFile <> " -CAcreateserial -out " <> c serverCrtFile diff --git a/src/Simplex/Messaging/Server/Main.hs b/src/Simplex/Messaging/Server/Main.hs index e02256aab..f52f0311b 100644 --- a/src/Simplex/Messaging/Server/Main.hs +++ b/src/Simplex/Messaging/Server/Main.hs @@ -10,12 +10,15 @@ module Simplex.Messaging.Server.Main where import Control.Concurrent.STM import Control.Monad (void) +import Data.ASN1.Types.String (asn1CharacterToString) import qualified Data.ByteString.Char8 as B import Data.Functor (($>)) import Data.Ini (lookupValue, readIniFile) import Data.Maybe (fromMaybe) import qualified Data.Text as T import Data.Text.Encoding (encodeUtf8) +import qualified Data.X509 as X +import qualified Data.X509.File as XF import Network.Socket (HostName) import Options.Applicative import qualified Simplex.Messaging.Crypto as C @@ -41,6 +44,10 @@ smpServerCLI cfgPath logPath = doesFileExist iniFile >>= \case True -> exitError $ "Error: server is already initialized (" <> iniFile <> " exists).\nRun `" <> executableName <> " start`." _ -> initializeServer opts + OnlineCert certOpts -> + doesFileExist iniFile >>= \case + True -> genOnline certOpts + _ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`." Start -> doesFileExist iniFile >>= \case True -> readIniFile iniFile >>= either exitError runServer @@ -56,8 +63,8 @@ smpServerCLI cfgPath logPath = defaultServerPort = "5223" executableName = "smp-server" storeLogFilePath = combine logPath "smp-server-store.log" - initializeServer opts - | scripted opts = initialize opts + initializeServer opts@InitOptions {ip, fqdn, scripted} + | scripted = initialize opts | otherwise = do putStrLn "Use `smp-server init -h` for available options." void $ withPrompt "SMP server will be initialized (press Enter)" getLine @@ -65,9 +72,9 @@ smpServerCLI cfgPath logPath = logStats <- onOffPrompt "Enable logging daily statistics" False putStrLn "Require a password to create new messaging queues?" password <- withPrompt "'r' for random (default), 'n' - no password, or enter password: " serverPassword - let host = fromMaybe (ip opts) (fqdn opts) + let host = fromMaybe ip fqdn host' <- withPrompt ("Enter server FQDN or IP address for certificate (" <> host <> "): ") getLine - initialize opts {enableStoreLog, logStats, fqdn = if null host' then fqdn opts else Just host', password} + initialize opts {enableStoreLog, logStats, fqdn = if null host' then fqdn else Just host', password} where serverPassword = getLine >>= \case @@ -78,7 +85,7 @@ smpServerCLI cfgPath logPath = case strDecode $ encodeUtf8 $ T.pack s of Right auth -> pure . Just $ ServerPassword auth _ -> putStrLn "Invalid password. Only latin letters, digits and symbols other than '@' and ':' are allowed" >> serverPassword - initialize InitOptions {enableStoreLog, logStats, signAlgorithm, ip, fqdn, password} = do + initialize InitOptions {enableStoreLog, logStats, signAlgorithm, password} = do clearDirIfExists cfgPath clearDirIfExists logPath createDirectoryIfMissing True cfgPath @@ -136,6 +143,30 @@ smpServerCLI cfgPath logPath = \disconnect: off\n" <> ("# ttl: " <> show (ttl defaultInactiveClientExpiration) <> "\n") <> ("# check_interval: " <> show (checkInterval defaultInactiveClientExpiration) <> "\n") + genOnline CertOptions {signAlgorithm_, commonName_} = do + (signAlgorithm, commonName) <- + case (signAlgorithm_, commonName_) of + (Just alg, Just cn) -> pure (alg, cn) + _ -> + XF.readSignedObject certPath >>= \case + [old] -> either exitError pure . fromX509 . X.signedObject $ X.getSigned old + [] -> exitError $ "No certificate found at " <> certPath + _ -> exitError $ "Too many certificates at " <> certPath + let x509cfg = defaultX509Config {signAlgorithm, commonName} + createServerX509_ False cfgPath x509cfg + putStrLn "Generated new server credentials" + warnCAPrivateKeyFile cfgPath x509cfg + where + certPath = combine cfgPath $ serverCrtFile defaultX509Config + fromX509 X.Certificate {certSignatureAlg, certSubjectDN} = (,) <$> maybe oldAlg Right signAlgorithm_ <*> maybe oldCN Right commonName_ + where + oldAlg = case certSignatureAlg of + X.SignatureALG_IntrinsicHash X.PubKeyALG_Ed448 -> Right ED448 + X.SignatureALG_IntrinsicHash X.PubKeyALG_Ed25519 -> Right ED25519 + alg -> Left $ "Unexpected signature algorithm " <> show alg + oldCN = case X.getDnElement X.DnCommonName certSubjectDN of + Nothing -> Left "Certificate subject has no CN element" + Just cn -> maybe (Left "Certificate subject CN decoding failed") Right $ asn1CharacterToString cn runServer ini = do hSetBuffering stdout LineBuffering hSetBuffering stderr LineBuffering @@ -210,6 +241,7 @@ smpServerCLI cfgPath logPath = data CliCommand = Init InitOptions + | OnlineCert CertOptions | Start | Delete @@ -227,10 +259,17 @@ data InitOptions = InitOptions data ServerPassword = ServerPassword BasicAuth | SPRandom deriving (Show) +data CertOptions = CertOptions + { signAlgorithm_ :: Maybe SignAlgorithm, + commonName_ :: Maybe HostName + } + deriving (Show) + cliCommandP :: FilePath -> FilePath -> FilePath -> Parser CliCommand cliCommandP cfgPath logPath iniFile = hsubparser ( command "init" (info (Init <$> initP) (progDesc $ "Initialize server - creates " <> cfgPath <> " and " <> logPath <> " directories and configuration files")) + <> command "cert" (info (OnlineCert <$> certP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")")) <> command "start" (info (pure Start) (progDesc $ "Start server (configuration: " <> iniFile <> ")")) <> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files")) ) @@ -255,7 +294,7 @@ cliCommandP cfgPath logPath iniFile = ( long "sign-algorithm" <> short 'a' <> help "Signature algorithm used for TLS certificates: ED25519, ED448" - <> value ED448 + <> value ED25519 <> showDefault <> metavar "ALG" ) @@ -293,5 +332,27 @@ cliCommandP cfgPath logPath iniFile = <> help "Non-interactive initialization using command-line options" ) pure InitOptions {enableStoreLog, logStats, signAlgorithm, ip, fqdn, password, scripted} + certP :: Parser CertOptions + certP = do + signAlgorithm_ <- + optional $ + option + (maybeReader readMaybe) + ( long "sign-algorithm" + <> short 'a' + <> help "Set new signature algorithm used for TLS certificates: ED25519, ED448" + <> showDefault + <> metavar "ALG" + ) + commonName_ <- + optional $ + strOption + ( long "cn" + <> help + "Set new Common Name for TLS online certificate" + <> showDefault + <> metavar "FQDN" + ) + pure CertOptions {signAlgorithm_, commonName_} parseBasicAuth :: ReadM ServerPassword parseBasicAuth = eitherReader $ fmap ServerPassword . strDecode . B.pack diff --git a/tests/CLITests.hs b/tests/CLITests.hs index 40915515b..032c1f748 100644 --- a/tests/CLITests.hs +++ b/tests/CLITests.hs @@ -1,9 +1,12 @@ +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} module CLITests where import Data.Ini (lookupValue, readIniFile) import Data.List (isPrefixOf) +import qualified Data.X509 as X +import qualified Data.X509.File as XF import Simplex.FileTransfer.Server.Main (xftpServerCLI, xftpServerVersion) import Simplex.Messaging.Notifications.Server.Main import Simplex.Messaging.Server.Main @@ -11,6 +14,7 @@ import Simplex.Messaging.Transport (simplexMQVersion) import Simplex.Messaging.Util (catchAll_) import System.Directory (doesFileExist) import System.Environment (withArgs) +import System.FilePath (()) import System.IO.Silently (capture_) import System.Timeout (timeout) import Test.Hspec @@ -51,6 +55,7 @@ cliTests = do smpServerTest :: Bool -> Bool -> IO () smpServerTest storeLog basicAuth = do + -- init capture_ (withArgs (["init", "-y"] <> ["-l" | storeLog] <> ["--no-password" | not basicAuth]) $ smpServerCLI cfgPath logPath) >>= (`shouldSatisfy` (("Server initialized, you can modify configuration in " <> cfgPath <> "/smp-server.ini") `isPrefixOf`)) Right ini <- readIniFile $ cfgPath <> "/smp-server.ini" @@ -61,12 +66,30 @@ smpServerTest storeLog basicAuth = do lookupValue "AUTH" "new_queues" ini `shouldBe` Right "on" lookupValue "INACTIVE_CLIENTS" "disconnect" ini `shouldBe` Right "off" doesFileExist (cfgPath <> "/ca.key") `shouldReturn` True + -- start r <- lines <$> capture_ (withArgs ["start"] $ (100000 `timeout` smpServerCLI cfgPath logPath) `catchAll_` pure (Just ())) r `shouldContain` ["SMP server v" <> simplexMQVersion] r `shouldContain` (if storeLog then ["Store log: " <> logPath <> "/smp-server-store.log"] else ["Store log disabled."]) r `shouldContain` ["Listening on port 5223 (TLS)..."] r `shouldContain` ["not expiring inactive clients"] r `shouldContain` (if basicAuth then ["creating new queues requires password"] else ["creating new queues allowed"]) + -- cert + let certPath = cfgPath "server.crt" + oldCrt@X.Certificate {} <- + XF.readSignedObject certPath >>= \case + [cert] -> pure . X.signedObject $ X.getSigned cert + _ -> error "bad crt format" + r' <- lines <$> capture_ (withArgs ["cert"] $ (100000 `timeout` smpServerCLI cfgPath logPath) `catchAll_` pure (Just ())) + r' `shouldContain` ["Generated new server credentials"] + newCrt <- + XF.readSignedObject certPath >>= \case + [cert] -> pure . X.signedObject $ X.getSigned cert + _ -> error "bad crt format after cert" + X.certSignatureAlg oldCrt `shouldBe` X.certSignatureAlg newCrt + X.certSubjectDN oldCrt `shouldBe` X.certSubjectDN newCrt + X.certSerial oldCrt `shouldNotBe` X.certSerial newCrt + X.certPubKey oldCrt `shouldNotBe` X.certPubKey newCrt + -- delete capture_ (withStdin "Y" . withArgs ["delete"] $ smpServerCLI cfgPath logPath) >>= (`shouldSatisfy` ("WARNING: deleting the server will make all queues inaccessible" `isPrefixOf`)) doesFileExist (cfgPath <> "/ca.key") `shouldReturn` False From 2f7a288280f4ae74ef1a70f3cb173345c95a4da1 Mon Sep 17 00:00:00 2001 From: Alexander Bondarenko <486682+dpwiz@users.noreply.github.com> Date: Tue, 13 Feb 2024 06:08:49 -0800 Subject: [PATCH 3/8] xftp: add sending and receiving via URI-encoded redirects (#968) * xftp: add URI encoding for FileDescription * tweak URI * allow smaller blocks * draft xftpReceiveFileFollow' and xftpSendFilePublic' * add sending with redirect * allow 64k chunks * add migrations with redirect fields * add test case * fix deadlock * revert CLI code * WIP: working send/receive via URI * fix field ambiguity * cleanup * update agent db schema * update minimal chunk size * add rfc * apply suggestions from code review Co-authored-by: Evgeny Poberezkin * add createRcvFileRedirect * extract Simplex.Messaging.ServiceScheme and reuse for files * update db schema * check size/digest on receive complete * cleanup * use SIZE/DIGEST errors for redirects too * split digest/size errors from redirect checks * fix redirect error encoding * rename RedirectMeta to RedirectFileInfo * use query encoding for file URI * group maybe fields under RcvFileRedirect * add extras field * update rfc * add extras encoding and no-redirect tests * fix toStrict for old ghc * extra client data in file descr URI * remove decoded yaml file --------- Co-authored-by: Evgeny Poberezkin --- rfcs/2024-01-26-file-links.md | 73 ++++++++++ simplexmq.cabal | 2 + src/Simplex/FileTransfer/Agent.hs | 97 ++++++++++--- src/Simplex/FileTransfer/Client/Main.hs | 10 +- src/Simplex/FileTransfer/Description.hs | 70 +++++++++- src/Simplex/FileTransfer/Protocol.hs | 10 +- src/Simplex/FileTransfer/Types.hs | 11 +- src/Simplex/Messaging/Agent.hs | 10 +- src/Simplex/Messaging/Agent/Protocol.hs | 24 +--- src/Simplex/Messaging/Agent/Store/SQLite.hs | 127 ++++++++++++------ .../Agent/Store/SQLite/Migrations.hs | 4 +- .../Migrations/M20240124_file_redirect.hs | 30 +++++ .../Store/SQLite/Migrations/agent_schema.sql | 8 +- src/Simplex/Messaging/Protocol.hs | 13 +- src/Simplex/Messaging/ServiceScheme.hs | 35 +++++ tests/AgentTests/ConnectionRequestTests.hs | 3 +- tests/AgentTests/SQLiteTests.hs | 11 +- tests/FileDescriptionTests.hs | 24 +++- tests/XFTPAgent.hs | 119 ++++++++++++++-- tests/XFTPCLI.hs | 2 +- tests/XFTPClient.hs | 2 +- 21 files changed, 558 insertions(+), 127 deletions(-) create mode 100644 rfcs/2024-01-26-file-links.md create mode 100644 src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20240124_file_redirect.hs create mode 100644 src/Simplex/Messaging/ServiceScheme.hs diff --git a/rfcs/2024-01-26-file-links.md b/rfcs/2024-01-26-file-links.md new file mode 100644 index 000000000..3ff2f430e --- /dev/null +++ b/rfcs/2024-01-26-file-links.md @@ -0,0 +1,73 @@ +# Sending large file descriptions + +It is desirable to provide a QR code/URI from which a file can be downloaded. This way files may be addressed outside a chat client. +Currently the `xftp` CLI tool can generate YAML file descriptions that can be used to receive a file. +It is possible to pass such a description as an URI, but descriptions for files larger than ~8 MBs (two 4 MB chunks) would give QR codes that are difficult to process. +A user can manually upload description and get a shorter one. Typically descriptions for files that are up to ~20 GBs would still be small enough to not require another pass, and that is way beyond any current (or, reasonable, fwiw) limitations. + +It is possible to streamline this process, so any application using simplexmq agent can easily send file descriptions and follow redirects. +A file description with a redirect contains an extra field with final file size and digest so it can be followed automatically. + +The flow would be like this: + +- Sending: + 1. Upload file as usual with `xftpSendFile`, get recipient file descriptions in `SFDONE` message. + 2. Upload one of the file descriptions with `xftpSendDescription`, get its redirect-description in its `SFDONE` message. + 3. Wrap in `FileDescriptionURI` and use `strEncode` to get a QR-sized URI. + 4. Show QR code / copy link. +- Receiving: + 1. Scan QR code / paste link. + 2. Use `strDecode` and unwrap `FileDescriptionURI` to get `ValidFileDescription 'FRecipient`. + 3. Download it as usual with `xftpReceiveFile`, getting `RFDONE` message when the file is fully received. + +It is not necesary to use redirect description if original description can be encoded to fit in 1002 characters. Beyond this size there is a significant jump in QR code complexity. +It is possible to call `encodeFileDescriptionURI` right after upload to test if the URI fits and skip step 2. +When `xftpReceiveFile` receives a decoded description that lacks `redirect` field, the procedure for downloading a file is the same as usual - download chunks and reassemble local file. + +## Agent changes + +### Sending + +Sending and receiving files in agent is a multi-step process mediated by DB entries in `snd_files` and `rcv_files` tables. + +`xftpSendDescription` is tasked with storing original description in a temporary locally-encrypted file, then creating upload task for it. + +It is necessary to preserve redirect metadata so it can be attached to descriptions in the `SFDONE` message sent by a worker: + +```sql +ALTER TABLE snd_files ADD COLUMN redirect_size INTEGER; +ALTER TABLE snd_files ADD COLUMN redirect_digest BLOB; +``` + +### Receiving + +`xftpReceiveFile` gets a file description as an argument and knows if it should follow redirect procedure or run an ordinary download. +For redirects it will prepare a `RcvFile` for redirect and then a placeholder, for the final file. +Agent messages would be sent using the entity ID of the final file, which is stored along with redirect metadata in `RcvFile` for the redirect. + +```sql +ALTER TABLE rcv_files ADD COLUMN redirect_id INTEGER REFERENCES rcv_files ON DELETE CASCADE; -- for later updates +ALTER TABLE rcv_files ADD COLUMN redirect_entity_id BLOB; -- for notifications +ALTER TABLE rcv_files ADD COLUMN redirect_size INTEGER; +ALTER TABLE rcv_files ADD COLUMN redirect_digest BLOB; +``` + +These additional fields will exist on the file that is a short description to receive an actual description of the final file. + +While a description YAML is being downloaded, the application will get `RFPROG` messages tagged for final entity, containing bytes downloaded so far and the total size from the original file. +When the description is fully downloaded, the worker would decode description and check if the stated size and digest match the declared in redirect. +Then it will replace placeholder description in `rcv_files` for destination file with the actual data from downloaded description. +Finally, instead of sending `RFDONE` for redirect, it hands over work to chunk download worker, which will run exactly as if the user requested its download directly. +An application will then receive `RFPROG` and `RFDONE` messages as usual. + +## URI encoding + +File description URIs use the same service schema `simplex:` or its `https://simplex.chat` (or any custom host) equivalent as do contact links and can be extracted from text and processed the same way. +The path section is `/file` (with an optional trailing `/`). +The payload is encoded in the "fragment" part of the link, using `#/?`, followed by a query string. +File description is encoded first in a YAML document, then URL-encoded in under the key `d`. +An application may want to pass extra parameters not necessary to download a file. Those go in the `_` key, encoded as a JSON dictionary. + +An example link: + +`simplex:/file#/?d=chunkSize%3A%2064kb%0Adigest%3A%20OtpnXkECTW4a18Eots2m3O22maeOCMqPUX4ulugIjgMEJfCpTYc_-T257Uw7s9bW_F0G5WBg5BioBWd4Z_OoCw%3D%3D%0Akey%3A%20rNR8_2SJuH7Qve43gV3zszL0R6oY5HSdRZT_paB-wfE%3D%0Anonce%3A%202oKwfK-w75nwyWp8_1Lv6QnQonIRtJmG%0Aparty%3A%20recipient%0Areplicas%3A%0A-%20chunks%3A%0A%20%20-%201%3ATdvaxMnG2Ph1e3QCx3-rpA%3D%3D%3AMC4CAQAwBQYDK2VwBCIEILdErEICvgrBCajDLTX2h3LXyMB7z5vrtLa3XVigJuf-%3ANS46KuYdgOWs6dUeMp7p2oF8rBQ9wQ2Ez6TW6Y6gHg0%3D%0A%20%20-%202%3AH5SRbtKYrXWVXTthrkeWzw%3D%3D%3AMC4CAQAwBQYDK2VwBCIEIGeEPNLt7lUGPfplwsoJLCDFnbIc5Hm31kz5X6rWXmgu%3A7QNRI-gvFx9UM-baXp3YVDli9pcfh3HGFKDhsA9JQHY%3D%0A%20%20-%203%3A_xjukkIl9WZFryUXT0h_TQ%3D%3D%3AMC4CAQAwBQYDK2VwBCIEIIRFBaL1HvUfePvKLuggwUrC_q_ZHd7v08IL9jhM7teC%3Aid2lgLMMjTGsR8SUogJuRdLoEHAc5SDQKFDqlZRSuEY%3D%0A%20%20server%3A%20xftp%3A%2F%2FLcJUMfVhwD8yxjAiSaDzzGF3-kLG4Uh0Fl_ZIjrRwjI%3D%40localhost%3A7002%0Asize%3A%20192kb%0A&_=%7B%22k%22:%22test%22%7D` diff --git a/simplexmq.cabal b/simplexmq.cabal index 256e9996d..dd01a2849 100644 --- a/simplexmq.cabal +++ b/simplexmq.cabal @@ -102,6 +102,7 @@ library Simplex.Messaging.Agent.Store.SQLite.Migrations.M20231222_command_created_at Simplex.Messaging.Agent.Store.SQLite.Migrations.M20231225_failed_work_items Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240121_message_delivery_indexes + Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240124_file_redirect Simplex.Messaging.Agent.TRcvQueues Simplex.Messaging.Client Simplex.Messaging.Client.Agent @@ -142,6 +143,7 @@ library Simplex.Messaging.Server.QueueStore.STM Simplex.Messaging.Server.Stats Simplex.Messaging.Server.StoreLog + Simplex.Messaging.ServiceScheme Simplex.Messaging.TMap Simplex.Messaging.Transport Simplex.Messaging.Transport.Buffer diff --git a/src/Simplex/FileTransfer/Agent.hs b/src/Simplex/FileTransfer/Agent.hs index 0cc6a3bde..18f39de42 100644 --- a/src/Simplex/FileTransfer/Agent.hs +++ b/src/Simplex/FileTransfer/Agent.hs @@ -7,6 +7,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TupleSections #-} {-# LANGUAGE TypeApplications #-} {-# OPTIONS_GHC -fno-warn-ambiguous-fields #-} @@ -19,6 +20,7 @@ module Simplex.FileTransfer.Agent xftpDeleteRcvFile', -- Sending files xftpSendFile', + xftpSendDescription', deleteSndFileInternal, deleteSndFileRemote, ) @@ -44,6 +46,7 @@ import Simplex.FileTransfer.Client.Main import Simplex.FileTransfer.Crypto import Simplex.FileTransfer.Description import Simplex.FileTransfer.Protocol (FileParty (..), SFileParty (..)) +import qualified Simplex.FileTransfer.Protocol as XFTP import Simplex.FileTransfer.Transport (XFTPRcvChunkSpec (..)) import Simplex.FileTransfer.Types import Simplex.FileTransfer.Util (removePath, uniqueCombine) @@ -57,6 +60,7 @@ import Simplex.Messaging.Crypto.File (CryptoFile (..), CryptoFileArgs) import qualified Simplex.Messaging.Crypto.File as CF import qualified Simplex.Messaging.Crypto.Lazy as LC import Simplex.Messaging.Encoding +import Simplex.Messaging.Encoding.String (strDecode, strEncode) import Simplex.Messaging.Protocol (EntityId, XFTPServer) import Simplex.Messaging.Util (liftError, tshow, unlessM, whenM) import System.FilePath (takeFileName, ()) @@ -97,7 +101,7 @@ closeXFTPAgent a = do stopWorkers workers = atomically (swapTVar workers M.empty) >>= mapM_ (liftIO . cancelWorker) xftpReceiveFile' :: AgentMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Maybe CryptoFileArgs -> m RcvFileId -xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks}) cfArgs = do +xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks, redirect}) cfArgs = do g <- asks random prefixPath <- getPrefixPath "rcv.xftp" createDirectory prefixPath @@ -107,14 +111,25 @@ xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks}) cfA createDirectory =<< toFSFilePath relTmpPath createEmptyFile =<< toFSFilePath relSavePath let saveFile = CryptoFile relSavePath cfArgs - fId <- withStore c $ \db -> createRcvFile db g userId fd relPrefixPath relTmpPath saveFile - forM_ chunks downloadChunk + fId <- case redirect of + Nothing -> withStore c $ \db -> createRcvFile db g userId fd relPrefixPath relTmpPath saveFile + Just _ -> do + -- prepare description paths + let relTmpPathRedirect = relPrefixPath "xftp.redirect-encrypted" + relSavePathRedirect = relPrefixPath "xftp.redirect-decrypted" + createDirectory =<< toFSFilePath relTmpPathRedirect + createEmptyFile =<< toFSFilePath relSavePathRedirect + cfArgsRedirect <- atomically $ CF.randomArgs g + let saveFileRedirect = CryptoFile relSavePathRedirect $ Just cfArgsRedirect + -- create download tasks + withStore c $ \db -> createRcvFileRedirect db g userId fd relPrefixPath relTmpPathRedirect saveFileRedirect relTmpPath saveFile + forM_ chunks (downloadChunk c) pure fId - where - downloadChunk :: AgentMonad m => FileChunk -> m () - downloadChunk FileChunk {replicas = (FileChunkReplica {server} : _)} = do - void $ getXFTPRcvWorker True c (Just server) - downloadChunk _ = throwError $ INTERNAL "no replicas" + +downloadChunk :: AgentMonad m => AgentClient -> FileChunk -> m () +downloadChunk c FileChunk {replicas = (FileChunkReplica {server} : _)} = do + void $ getXFTPRcvWorker True c (Just server) +downloadChunk _ _ = throwError $ INTERNAL "no replicas" getPrefixPath :: AgentMonad m => String -> m FilePath getPrefixPath suffix = do @@ -172,14 +187,17 @@ runXFTPRcvWorker c srv Worker {doWork} = do relChunkPath = fileTmpPath takeFileName chunkPath agentXFTPDownloadChunk c userId digest replica chunkSpec atomically $ waitUntilForeground c - (complete, progress) <- withStore c $ \db -> runExceptT $ do + (entityId, complete, progress) <- withStore c $ \db -> runExceptT $ do liftIO $ updateRcvFileChunkReceived db (rcvChunkReplicaId replica) rcvChunkId relChunkPath - RcvFile {size = FileSize total, chunks} <- ExceptT $ getRcvFile db rcvFileId + RcvFile {size = FileSize currentSize, chunks, redirect} <- ExceptT $ getRcvFile db rcvFileId let rcvd = receivedSize chunks complete = all chunkReceived chunks + (entityId, total) = case redirect of + Nothing -> (rcvFileEntityId, currentSize) + Just RcvFileRedirect {redirectFileInfo = RedirectFileInfo {size = FileSize finalSize}, redirectEntityId} -> (redirectEntityId, finalSize) liftIO . when complete $ updateRcvFileStatus db rcvFileId RFSReceived - pure (complete, RFPROG rcvd total) - notify c rcvFileEntityId progress + pure (entityId, complete, RFPROG rcvd total) + notify c entityId progress when complete . void $ getXFTPRcvWorker True c Nothing where @@ -223,7 +241,7 @@ runXFTPRcvLocalWorker c Worker {doWork} = do \f@RcvFile {rcvFileId, rcvFileEntityId, tmpPath} -> decryptFile f `catchAgentError` (rcvWorkerInternalError c rcvFileId rcvFileEntityId tmpPath . show) decryptFile :: RcvFile -> m () - decryptFile RcvFile {rcvFileId, rcvFileEntityId, key, nonce, tmpPath, saveFile, status, chunks} = do + decryptFile RcvFile {rcvFileId, rcvFileEntityId, size, digest, key, nonce, tmpPath, saveFile, status, chunks, redirect} = do let CryptoFile savePath cfArgs = saveFile fsSavePath <- toFSFilePath savePath when (status == RFSDecrypting) $ @@ -231,12 +249,33 @@ runXFTPRcvLocalWorker c Worker {doWork} = do withStore' c $ \db -> updateRcvFileStatus db rcvFileId RFSDecrypting chunkPaths <- getChunkPaths chunks encSize <- liftIO $ foldM (\s path -> (s +) . fromIntegral <$> getFileSize path) 0 chunkPaths + when (FileSize encSize /= size) $ throwError $ XFTP XFTP.SIZE + encDigest <- liftIO $ LC.sha512Hash <$> readChunks chunkPaths + when (FileDigest encDigest /= digest) $ throwError $ XFTP XFTP.DIGEST let destFile = CryptoFile fsSavePath cfArgs void $ liftError (INTERNAL . show) $ decryptChunks encSize chunkPaths key nonce $ \_ -> pure destFile - notify c rcvFileEntityId $ RFDONE fsSavePath - forM_ tmpPath (removePath <=< toFSFilePath) - atomically $ waitUntilForeground c - withStore' c (`updateRcvFileComplete` rcvFileId) + case redirect of + Nothing -> do + notify c rcvFileEntityId $ RFDONE fsSavePath + forM_ tmpPath (removePath <=< toFSFilePath) + atomically $ waitUntilForeground c + withStore' c (`updateRcvFileComplete` rcvFileId) + Just RcvFileRedirect {redirectFileInfo, redirectDbId} -> do + let RedirectFileInfo {size = redirectSize, digest = redirectDigest} = redirectFileInfo + forM_ tmpPath (removePath <=< toFSFilePath) + atomically $ waitUntilForeground c + withStore' c (`updateRcvFileComplete` rcvFileId) + -- proceed with redirect + yaml <- liftError (INTERNAL . show) (CF.readFile $ CryptoFile fsSavePath cfArgs) `finally` (toFSFilePath fsSavePath >>= removePath) + next@FileDescription {chunks = nextChunks} <- case strDecode (LB.toStrict yaml) of + Left _ -> throwError . XFTP $ XFTP.REDIRECT "decode error" + Right (ValidFileDescription fd@FileDescription {size = dstSize, digest = dstDigest}) + | dstSize /= redirectSize -> throwError . XFTP $ XFTP.REDIRECT "size mismatch" + | dstDigest /= redirectDigest -> throwError . XFTP $ XFTP.REDIRECT "digest mismatch" + | otherwise -> pure fd + -- register and download chunks from the actual file + withStore c $ \db -> updateRcvFileRedirect db redirectDbId next + forM_ nextChunks (downloadChunk c) where getChunkPaths :: [RcvFileChunk] -> m [FilePath] getChunkPaths [] = pure [] @@ -268,7 +307,23 @@ xftpSendFile' c userId file numRecipients = do key <- atomically $ C.randomSbKey g nonce <- atomically $ C.randomCbNonce g -- saving absolute filePath will not allow to restore file encryption after app update, but it's a short window - fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce + fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce Nothing + void $ getXFTPSndWorker True c Nothing + pure fId + +xftpSendDescription' :: forall m. AgentMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> m SndFileId +xftpSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) = do + g <- asks random + prefixPath <- getPrefixPath "snd.xftp" + createDirectory prefixPath + let relPrefixPath = takeFileName prefixPath + let directYaml = prefixPath "direct.yaml" + cfArgs <- atomically $ CF.randomArgs g + let file = CryptoFile directYaml (Just cfArgs) + liftError (INTERNAL . show) $ CF.writeFile file (LB.fromStrict $ strEncode fdDirect) + key <- atomically $ C.randomSbKey g + nonce <- atomically $ C.randomCbNonce g + fId <- withStore c $ \db -> createSndFile db g userId file 1 relPrefixPath key nonce $ Just RedirectFileInfo {size, digest} void $ getXFTPSndWorker True c Nothing pure fId @@ -423,15 +478,15 @@ runXFTPSndWorker c srv Worker {doWork} = do sndFileToDescrs :: SndFile -> m (ValidFileDescription 'FSender, [ValidFileDescription 'FRecipient]) sndFileToDescrs SndFile {digest = Nothing} = throwError $ INTERNAL "snd file has no digest" sndFileToDescrs SndFile {chunks = []} = throwError $ INTERNAL "snd file has no chunks" - sndFileToDescrs SndFile {digest = Just digest, key, nonce, chunks = chunks@(fstChunk : _)} = do + sndFileToDescrs SndFile {digest = Just digest, key, nonce, chunks = chunks@(fstChunk : _), redirect} = do let chunkSize = FileSize $ sndChunkSize fstChunk size = FileSize $ sum $ map (fromIntegral . sndChunkSize) chunks -- snd description sndDescrChunks <- mapM toSndDescrChunk chunks - let fdSnd = FileDescription {party = SFSender, size, digest, key, nonce, chunkSize, chunks = sndDescrChunks} + let fdSnd = FileDescription {party = SFSender, size, digest, key, nonce, chunkSize, chunks = sndDescrChunks, redirect = Nothing} validFdSnd <- either (throwError . INTERNAL) pure $ validateFileDescription fdSnd -- rcv descriptions - let fdRcv = FileDescription {party = SFRecipient, size, digest, key, nonce, chunkSize, chunks = []} + let fdRcv = FileDescription {party = SFRecipient, size, digest, key, nonce, chunkSize, chunks = [], redirect} fdRcvs = createRcvFileDescriptions fdRcv chunks validFdRcvs <- either (throwError . INTERNAL) pure $ mapM validateFileDescription fdRcvs pure (validFdSnd, validFdRcvs) diff --git a/src/Simplex/FileTransfer/Client/Main.hs b/src/Simplex/FileTransfer/Client/Main.hs index 7c3521fc2..906966fb0 100644 --- a/src/Simplex/FileTransfer/Client/Main.hs +++ b/src/Simplex/FileTransfer/Client/Main.hs @@ -15,6 +15,7 @@ module Simplex.FileTransfer.Client.Main CLIError (..), xftpClientCLI, cliSendFile, + cliSendFileOpts, prepareChunkSizes, prepareChunkSpecs, maxFileSize, @@ -297,8 +298,8 @@ cliSendFileOpts SendOptions {filePath, outputDir, numRecipients, xftpServers, re withExceptT (CLIError . show) $ encryptFile srcFile fileHdr key nonce fileSize' encSize encPath digest <- liftIO $ LC.sha512Hash <$> LB.readFile encPath let chunkSpecs = prepareChunkSpecs encPath chunkSizes - fdRcv = FileDescription {party = SFRecipient, size = FileSize encSize, digest = FileDigest digest, key, nonce, chunkSize = FileSize defChunkSize, chunks = []} - fdSnd = FileDescription {party = SFSender, size = FileSize encSize, digest = FileDigest digest, key, nonce, chunkSize = FileSize defChunkSize, chunks = []} + fdRcv = FileDescription {party = SFRecipient, size = FileSize encSize, digest = FileDigest digest, key, nonce, chunkSize = FileSize defChunkSize, chunks = [], redirect = Nothing} + fdSnd = FileDescription {party = SFSender, size = FileSize encSize, digest = FileDigest digest, key, nonce, chunkSize = FileSize defChunkSize, chunks = [], redirect = Nothing} logInfo $ "encrypted file to " <> tshow encPath pure (encPath, fdRcv, fdSnd, chunkSpecs, encSize) uploadFile :: TVar ChaChaDRG -> [XFTPChunkSpec] -> TVar [Int64] -> Int64 -> ExceptT CLIError IO [SentFileChunk] @@ -526,9 +527,8 @@ prepareChunkSizes size' = prepareSizes size' where (smallSize, bigSize) | size' > size34 chunkSize3 = (chunkSize2, chunkSize3) - | otherwise = (chunkSize1, chunkSize2) - -- | size' > size34 chunkSize2 = (chunkSize1, chunkSize2) - -- | otherwise = (chunkSize0, chunkSize1) + | size' > size34 chunkSize2 = (chunkSize1, chunkSize2) + | otherwise = (chunkSize0, chunkSize1) size34 sz = (fromIntegral sz * 3) `div` 4 prepareSizes 0 = [] prepareSizes size diff --git a/src/Simplex/FileTransfer/Description.hs b/src/Simplex/FileTransfer/Description.hs index cbea3f23b..bf5c91634 100644 --- a/src/Simplex/FileTransfer/Description.hs +++ b/src/Simplex/FileTransfer/Description.hs @@ -12,6 +12,7 @@ module Simplex.FileTransfer.Description ( FileDescription (..), + RedirectFileInfo (..), AFileDescription (..), ValidFileDescription, -- constructor is not exported, use pattern pattern ValidFileDescription, @@ -30,12 +31,17 @@ module Simplex.FileTransfer.Description kb, mb, gb, + FileDescriptionURI (..), + FileClientData, + fileDescriptionURI, + qrSizeLimit, ) where import Control.Applicative (optional) import Control.Monad ((<=<)) import Data.Aeson (FromJSON, ToJSON) +import qualified Data.Aeson as J import qualified Data.Aeson.TH as J import Data.Attoparsec.ByteString.Char8 (Parser) import qualified Data.Attoparsec.ByteString.Char8 as A @@ -50,17 +56,21 @@ import Data.Map (Map) import qualified Data.Map as M import Data.Maybe (fromMaybe) import Data.String +import Data.Text (Text) +import Data.Text.Encoding (encodeUtf8) import Data.Word (Word32) import qualified Data.Yaml as Y import Database.SQLite.Simple.FromField (FromField (..)) import Database.SQLite.Simple.ToField (ToField (..)) import Simplex.FileTransfer.Chunks import Simplex.FileTransfer.Protocol +import Simplex.Messaging.Agent.QueryString import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Encoding.String import Simplex.Messaging.Parsers (defaultJSON, parseAll) import Simplex.Messaging.Protocol (XFTPServer) -import Simplex.Messaging.Util (bshow, (<$?>)) +import Simplex.Messaging.ServiceScheme (ServiceScheme (..)) +import Simplex.Messaging.Util (bshow, safeDecodeUtf8, (<$?>)) data FileDescription (p :: FileParty) = FileDescription { party :: SFileParty p, @@ -69,7 +79,14 @@ data FileDescription (p :: FileParty) = FileDescription key :: C.SbKey, nonce :: C.CbNonce, chunkSize :: FileSize Word32, - chunks :: [FileChunk] + chunks :: [FileChunk], + redirect :: Maybe RedirectFileInfo + } + deriving (Eq, Show) + +data RedirectFileInfo = RedirectFileInfo + { size :: FileSize Int64, + digest :: FileDigest } deriving (Eq, Show) @@ -147,7 +164,8 @@ data YAMLFileDescription = YAMLFileDescription key :: C.SbKey, nonce :: C.CbNonce, chunkSize :: String, - replicas :: [YAMLServerReplicas] + replicas :: [YAMLServerReplicas], + redirect :: Maybe RedirectFileInfo } deriving (Eq, Show) @@ -170,8 +188,16 @@ data FileServerReplica = FileServerReplica newtype FileSize a = FileSize {unFileSize :: a} deriving (Eq, Show) +instance FromJSON a => FromJSON (FileSize a) where + parseJSON v = FileSize <$> Y.parseJSON v + +instance ToJSON a => ToJSON (FileSize a) where + toJSON = Y.toJSON . unFileSize + $(J.deriveJSON defaultJSON ''YAMLServerReplicas) +$(J.deriveJSON defaultJSON ''RedirectFileInfo) + $(J.deriveJSON defaultJSON ''YAMLFileDescription) instance FilePartyI p => StrEncoding (ValidFileDescription p) where @@ -204,7 +230,7 @@ validateFileDescription fd@FileDescription {size, chunks} chunksSize = fromIntegral . foldl' (\s FileChunk {chunkSize} -> s + unFileSize chunkSize) 0 encodeFileDescription :: FileDescription p -> YAMLFileDescription -encodeFileDescription FileDescription {party, size, digest, key, nonce, chunkSize, chunks} = +encodeFileDescription FileDescription {party, size, digest, key, nonce, chunkSize, chunks, redirect} = YAMLFileDescription { party = toFileParty party, size = B.unpack $ strEncode size, @@ -212,9 +238,39 @@ encodeFileDescription FileDescription {party, size, digest, key, nonce, chunkSiz key, nonce, chunkSize = B.unpack $ strEncode chunkSize, - replicas = encodeFileReplicas chunkSize chunks + replicas = encodeFileReplicas chunkSize chunks, + redirect } +data FileDescriptionURI = FileDescriptionURI + { scheme :: ServiceScheme, + description :: ValidFileDescription 'FRecipient, + clientData :: Maybe FileClientData -- JSON-encoded extensions to pass in a link + } + deriving (Eq, Show) + +type FileClientData = Text + +fileDescriptionURI :: ValidFileDescription 'FRecipient -> FileDescriptionURI +fileDescriptionURI vfd = FileDescriptionURI SSSimplex vfd mempty + +instance StrEncoding FileDescriptionURI where + strEncode FileDescriptionURI {scheme, description, clientData} = mconcat [strEncode scheme, "/file", "#/?", queryStr] + where + queryStr = strEncode $ QSP QEscape qs + qs = ("desc", strEncode description) : maybe [] (\cd -> [("data", encodeUtf8 cd)]) clientData + strP = do + scheme <- strP + _ <- "/file" <* optional (A.char '/') <* "#/?" + query <- strP + description <- queryParam "desc" query + let clientData = safeDecodeUtf8 <$> queryParamStr "data" query + pure FileDescriptionURI {scheme, description, clientData} + +-- | URL length in QR code before jumping up to a next size. +qrSizeLimit :: Int +qrSizeLimit = 1002 -- ~2 chunks in URLencoded YAML with some spare size for server hosts + instance (Integral a, Show a) => StrEncoding (FileSize a) where strEncode (FileSize b) | b' /= 0 = bshow b @@ -285,13 +341,13 @@ unfoldChunksToReplicas defChunkSize = concatMap chunkReplicas in FileServerReplica {chunkNo, server, replicaId, replicaKey, digest = digest', chunkSize = chunkSize'} decodeFileDescription :: YAMLFileDescription -> Either String AFileDescription -decodeFileDescription YAMLFileDescription {party, size, digest, key, nonce, chunkSize, replicas} = do +decodeFileDescription YAMLFileDescription {party, size, digest, key, nonce, chunkSize, replicas, redirect} = do size' <- strDecode $ B.pack size chunkSize' <- strDecode $ B.pack chunkSize replicas' <- decodeFileParts replicas chunks <- foldReplicasToChunks chunkSize' replicas' pure $ case aFileParty party of - AFP party' -> AFD FileDescription {party = party', size = size', digest, key, nonce, chunkSize = chunkSize', chunks} + AFP party' -> AFD FileDescription {party = party', size = size', digest, key, nonce, chunkSize = chunkSize', chunks, redirect} where decodeFileParts = fmap concat . mapM decodeYAMLServerReplicas diff --git a/src/Simplex/FileTransfer/Protocol.hs b/src/Simplex/FileTransfer/Protocol.hs index 578aefd15..5602c4268 100644 --- a/src/Simplex/FileTransfer/Protocol.hs +++ b/src/Simplex/FileTransfer/Protocol.hs @@ -338,6 +338,8 @@ data XFTPErrorType HAS_FILE | -- | file IO error FILE_IO + | -- | bad redirect data + REDIRECT {redirectError :: String} | -- | internal server error INTERNAL | -- | used internally, never returned by the server (to be removed) @@ -347,8 +349,12 @@ data XFTPErrorType instance StrEncoding XFTPErrorType where strEncode = \case CMD e -> "CMD " <> bshow e + REDIRECT e -> "REDIRECT " <> bshow e e -> bshow e - strP = "CMD " *> (CMD <$> parseRead1) <|> parseRead1 + strP = + "CMD " *> (CMD <$> parseRead1) + <|> "REDIRECT " *> (REDIRECT <$> parseRead A.takeByteString) + <|> parseRead1 instance Encoding XFTPErrorType where smpEncode = \case @@ -363,6 +369,7 @@ instance Encoding XFTPErrorType where NO_FILE -> "NO_FILE" HAS_FILE -> "HAS_FILE" FILE_IO -> "FILE_IO" + REDIRECT err -> "REDIRECT " <> smpEncode err INTERNAL -> "INTERNAL" DUPLICATE_ -> "DUPLICATE_" @@ -379,6 +386,7 @@ instance Encoding XFTPErrorType where "NO_FILE" -> pure NO_FILE "HAS_FILE" -> pure HAS_FILE "FILE_IO" -> pure FILE_IO + "REDIRECT" -> REDIRECT <$> _smpP "INTERNAL" -> pure INTERNAL "DUPLICATE_" -> pure DUPLICATE_ _ -> fail "bad error type" diff --git a/src/Simplex/FileTransfer/Types.hs b/src/Simplex/FileTransfer/Types.hs index e51cb14e3..513822a82 100644 --- a/src/Simplex/FileTransfer/Types.hs +++ b/src/Simplex/FileTransfer/Types.hs @@ -47,6 +47,7 @@ data RcvFile = RcvFile key :: C.SbKey, nonce :: C.CbNonce, chunkSize :: FileSize Word32, + redirect :: Maybe RcvFileRedirect, chunks :: [RcvFileChunk], prefixPath :: FilePath, tmpPath :: Maybe FilePath, @@ -108,6 +109,13 @@ data RcvFileChunkReplica = RcvFileChunkReplica } deriving (Eq, Show) +data RcvFileRedirect = RcvFileRedirect + { redirectDbId :: DBRcvFileId, + redirectEntityId :: RcvFileId, + redirectFileInfo :: RedirectFileInfo + } + deriving (Eq, Show) + -- Sending files type DBSndFileId = Int64 @@ -124,7 +132,8 @@ data SndFile = SndFile srcFile :: CryptoFile, prefixPath :: Maybe FilePath, status :: SndFileStatus, - deleted :: Bool + deleted :: Bool, + redirect :: Maybe RedirectFileInfo } deriving (Eq, Show) diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs index 2ffe6f0f7..911358992 100644 --- a/src/Simplex/Messaging/Agent.hs +++ b/src/Simplex/Messaging/Agent.hs @@ -93,6 +93,7 @@ module Simplex.Messaging.Agent xftpReceiveFile, xftpDeleteRcvFile, xftpSendFile, + xftpSendDescription, xftpDeleteSndFileInternal, xftpDeleteSndFileRemote, rcNewHostPairing, @@ -137,7 +138,7 @@ import qualified Data.Text as T import Data.Time.Clock import Data.Time.Clock.System (systemToUTCTime) import Data.Word (Word16) -import Simplex.FileTransfer.Agent (closeXFTPAgent, deleteSndFileInternal, deleteSndFileRemote, startXFTPWorkers, toFSFilePath, xftpDeleteRcvFile', xftpReceiveFile', xftpSendFile') +import Simplex.FileTransfer.Agent (closeXFTPAgent, deleteSndFileInternal, deleteSndFileRemote, startXFTPWorkers, toFSFilePath, xftpDeleteRcvFile', xftpReceiveFile', xftpSendDescription', xftpSendFile') import Simplex.FileTransfer.Description (ValidFileDescription) import Simplex.FileTransfer.Protocol (FileParty (..)) import Simplex.FileTransfer.Util (removePath) @@ -163,6 +164,7 @@ import Simplex.Messaging.Notifications.Types import Simplex.Messaging.Parsers (parse) import Simplex.Messaging.Protocol (BrokerMsg, EntityId, ErrorType (AUTH), MsgBody, MsgFlags (..), NtfServer, ProtoServerWithAuth, ProtocolTypeI (..), SMPMsgMeta, SProtocolType (..), SndPublicVerifyKey, SubscriptionMode (..), UserProtocol, XFTPServerWithAuth) import qualified Simplex.Messaging.Protocol as SMP +import Simplex.Messaging.ServiceScheme (ServiceScheme (..)) import qualified Simplex.Messaging.TMap as TM import Simplex.Messaging.Util import Simplex.Messaging.Version @@ -400,6 +402,10 @@ xftpDeleteRcvFile c = withAgentEnv c . xftpDeleteRcvFile' c xftpSendFile :: AgentErrorMonad m => AgentClient -> UserId -> CryptoFile -> Int -> m SndFileId xftpSendFile c = withAgentEnv c .:. xftpSendFile' c +-- | Send XFTP file +xftpSendDescription :: AgentErrorMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> m SndFileId +xftpSendDescription c = withAgentEnv c .: xftpSendDescription' c + -- | Delete XFTP snd file internally (deletes work files from file system and db records) xftpDeleteSndFileInternal :: AgentErrorMonad m => AgentClient -> SndFileId -> m () xftpDeleteSndFileInternal c = withAgentEnv c . deleteSndFileInternal c @@ -643,7 +649,7 @@ newRcvConnSrv c userId connId enableNtfs cMode clientData subMode srv = do when enableNtfs $ do ns <- asks ntfSupervisor atomically $ sendNtfSubCommand ns (connId, NSCCreate) - let crData = ConnReqUriData CRSSimplex smpAgentVRange [qUri] clientData + let crData = ConnReqUriData SSSimplex smpAgentVRange [qUri] clientData case cMode of SCMContact -> pure (connId, CRContactUri crData) SCMInvitation -> do diff --git a/src/Simplex/Messaging/Agent/Protocol.hs b/src/Simplex/Messaging/Agent/Protocol.hs index 54777bb64..0771601ab 100644 --- a/src/Simplex/Messaging/Agent/Protocol.hs +++ b/src/Simplex/Messaging/Agent/Protocol.hs @@ -97,7 +97,7 @@ module Simplex.Messaging.Agent.Protocol AConnectionRequestUri (..), ConnReqUriData (..), CRClientData, - ConnReqScheme (..), + ServiceScheme, simplexChat, AgentErrorType (..), CommandErrorType (..), @@ -197,7 +197,6 @@ import Simplex.Messaging.Protocol SMPServer, SMPServerWithAuth, SndPublicVerifyKey, - SrvLoc (..), SubscriptionMode, legacyEncodeServer, legacyServerP, @@ -208,6 +207,7 @@ import Simplex.Messaging.Protocol pattern SMPServer, ) import qualified Simplex.Messaging.Protocol as SMP +import Simplex.Messaging.ServiceScheme import Simplex.Messaging.Transport (Transport (..), TransportError, serializeTransportError, transportErrorP) import Simplex.Messaging.Transport.Client (TransportHost, TransportHosts_ (..)) import Simplex.Messaging.Util @@ -1123,13 +1123,13 @@ instance forall m. ConnectionModeI m => StrEncoding (ConnectionRequestUri m) whe instance StrEncoding AConnectionRequestUri where strEncode (ACR _ cr) = strEncode cr strP = do - _crScheme :: ConnReqScheme <- strP + _crScheme :: ServiceScheme <- strP crMode <- A.char '/' *> crModeP <* optional (A.char '/') <* "#/?" query <- strP crAgentVRange <- queryParam "v" query crSmpQueues <- queryParam "smp" query let crClientData = safeDecodeUtf8 <$> queryParamStr "data" query - let crData = ConnReqUriData {crScheme = CRSSimplex, crAgentVRange, crSmpQueues, crClientData} + let crData = ConnReqUriData {crScheme = SSSimplex, crAgentVRange, crSmpQueues, crClientData} case crMode of CMInvitation -> do crE2eParams <- queryParam "e2e" query @@ -1328,7 +1328,7 @@ instance Eq AConnectionRequestUri where deriving instance Show AConnectionRequestUri data ConnReqUriData = ConnReqUriData - { crScheme :: ConnReqScheme, + { crScheme :: ServiceScheme, crAgentVRange :: VersionRange, crSmpQueues :: NonEmpty SMPQueueUri, crClientData :: Maybe CRClientData @@ -1337,20 +1337,6 @@ data ConnReqUriData = ConnReqUriData type CRClientData = Text -data ConnReqScheme = CRSSimplex | CRSAppServer SrvLoc - deriving (Eq, Show) - -instance StrEncoding ConnReqScheme where - strEncode = \case - CRSSimplex -> "simplex:" - CRSAppServer srv -> "https://" <> strEncode srv - strP = - "simplex:" $> CRSSimplex - <|> "https://" *> (CRSAppServer <$> strP) - -simplexChat :: ConnReqScheme -simplexChat = CRSAppServer $ SrvLoc "simplex.chat" "" - -- | SMP queue status. data QueueStatus = -- | queue is created diff --git a/src/Simplex/Messaging/Agent/Store/SQLite.hs b/src/Simplex/Messaging/Agent/Store/SQLite.hs index 06f41be9e..9865fe424 100644 --- a/src/Simplex/Messaging/Agent/Store/SQLite.hs +++ b/src/Simplex/Messaging/Agent/Store/SQLite.hs @@ -164,6 +164,7 @@ module Simplex.Messaging.Agent.Store.SQLite -- Rcv files createRcvFile, + createRcvFileRedirect, getRcvFile, getRcvFileByEntityId, updateRcvChunkReplicaDelay, @@ -171,6 +172,7 @@ module Simplex.Messaging.Agent.Store.SQLite updateRcvFileStatus, updateRcvFileError, updateRcvFileComplete, + updateRcvFileRedirect, updateRcvFileNoTmpPath, updateRcvFileDeleted, deleteRcvFile', @@ -231,6 +233,7 @@ import Data.ByteArray (ScrubbedBytes) import qualified Data.ByteArray as BA import Data.ByteString (ByteString) import qualified Data.ByteString.Base64.URL as U +import qualified Data.ByteString.Char8 as B import Data.Char (toLower) import Data.Functor (($>)) import Data.IORef @@ -255,7 +258,7 @@ import qualified Database.SQLite3 as SQLite3 import Network.Socket (ServiceName) import Simplex.FileTransfer.Client (XFTPChunkSpec (..)) import Simplex.FileTransfer.Description -import Simplex.FileTransfer.Protocol (FileParty (..)) +import Simplex.FileTransfer.Protocol (FileParty (..), SFileParty (..)) import Simplex.FileTransfer.Types import Simplex.Messaging.Agent.Protocol import Simplex.Messaging.Agent.RetryInterval (RI2State (..)) @@ -2263,38 +2266,66 @@ getXFTPServerId_ db ProtocolServer {host, port, keyHash} = do DB.query db "SELECT xftp_server_id FROM xftp_servers WHERE xftp_host = ? AND xftp_port = ? AND xftp_key_hash = ?" (host, port, keyHash) createRcvFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription 'FRecipient -> FilePath -> FilePath -> CryptoFile -> IO (Either StoreError RcvFileId) -createRcvFile db gVar userId fd@FileDescription {chunks} prefixPath tmpPath (CryptoFile savePath cfArgs) = runExceptT $ do - (rcvFileEntityId, rcvFileId) <- ExceptT $ insertRcvFile fd +createRcvFile db gVar userId fd@FileDescription {chunks} prefixPath tmpPath file = runExceptT $ do + (rcvFileEntityId, rcvFileId) <- ExceptT $ insertRcvFile db gVar userId fd prefixPath tmpPath file Nothing Nothing liftIO $ forM_ chunks $ \fc@FileChunk {replicas} -> do - chunkId <- insertChunk fc rcvFileId - forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertReplica rno replica chunkId + chunkId <- insertRcvFileChunk db fc rcvFileId + forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertRcvFileChunkReplica db rno replica chunkId pure rcvFileEntityId + +createRcvFileRedirect :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription FRecipient -> FilePath -> FilePath -> CryptoFile -> FilePath -> CryptoFile -> IO (Either StoreError RcvFileId) +createRcvFileRedirect _ _ _ FileDescription {redirect = Nothing} _ _ _ _ _ = pure $ Left $ SEInternal "createRcvFileRedirect called without redirect" +createRcvFileRedirect db gVar userId redirectFd@FileDescription {chunks = redirectChunks, redirect = Just RedirectFileInfo {size, digest}} prefixPath redirectPath redirectFile dstPath dstFile = runExceptT $ do + (dstEntityId, dstId) <- ExceptT $ insertRcvFile db gVar userId dummyDst prefixPath dstPath dstFile Nothing Nothing + (_, redirectId) <- ExceptT $ insertRcvFile db gVar userId redirectFd prefixPath redirectPath redirectFile (Just dstId) (Just dstEntityId) + liftIO $ + forM_ redirectChunks $ \fc@FileChunk {replicas} -> do + chunkId <- insertRcvFileChunk db fc redirectId + forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertRcvFileChunkReplica db rno replica chunkId + pure dstEntityId where - insertRcvFile :: FileDescription 'FRecipient -> IO (Either StoreError (RcvFileId, DBRcvFileId)) - insertRcvFile FileDescription {size, digest, key, nonce, chunkSize} = runExceptT $ do - rcvFileEntityId <- ExceptT $ - createWithRandomId gVar $ \rcvFileEntityId -> - DB.execute - db - "INSERT INTO rcv_files (rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)" - ((rcvFileEntityId, userId, size, digest, key, nonce, chunkSize) :. (prefixPath, tmpPath, savePath, fileKey <$> cfArgs, fileNonce <$> cfArgs, RFSReceiving)) - rcvFileId <- liftIO $ insertedRowId db - pure (rcvFileEntityId, rcvFileId) - insertChunk :: FileChunk -> DBRcvFileId -> IO Int64 - insertChunk FileChunk {chunkNo, chunkSize, digest} rcvFileId = do + dummyDst = FileDescription + { party = SFRecipient, + size, + digest, + redirect = Nothing, + -- updated later with updateRcvFileRedirect + key = C.unsafeSbKey $ B.replicate 32 '#', + nonce = C.cbNonce "", + chunkSize = FileSize 0, + chunks = [] + } + +insertRcvFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription 'FRecipient -> FilePath -> FilePath -> CryptoFile -> Maybe DBRcvFileId -> Maybe RcvFileId -> IO (Either StoreError (RcvFileId, DBRcvFileId)) +insertRcvFile db gVar userId FileDescription {size, digest, key, nonce, chunkSize, redirect} prefixPath tmpPath (CryptoFile savePath cfArgs) redirectId_ redirectEntityId_ = runExceptT $ do + let (redirectDigest_, redirectSize_) = case redirect of + Just RedirectFileInfo {digest = d, size = s} -> (Just d, Just s) + Nothing -> (Nothing, Nothing) + rcvFileEntityId <- ExceptT $ + createWithRandomId gVar $ \rcvFileEntityId -> DB.execute db - "INSERT INTO rcv_file_chunks (rcv_file_id, chunk_no, chunk_size, digest) VALUES (?,?,?,?)" - (rcvFileId, chunkNo, chunkSize, digest) - insertedRowId db - insertReplica :: Int -> FileChunkReplica -> Int64 -> IO () - insertReplica replicaNo FileChunkReplica {server, replicaId, replicaKey} chunkId = do - srvId <- createXFTPServer_ db server - DB.execute - db - "INSERT INTO rcv_file_chunk_replicas (replica_number, rcv_file_chunk_id, xftp_server_id, replica_id, replica_key) VALUES (?,?,?,?,?)" - (replicaNo, chunkId, srvId, replicaId, replicaKey) + "INSERT INTO rcv_files (rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, redirect_id, redirect_entity_id, redirect_digest, redirect_size) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + ((rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, fileKey <$> cfArgs, fileNonce <$> cfArgs, RFSReceiving, redirectId_, redirectEntityId_, redirectDigest_, redirectSize_)) + rcvFileId <- liftIO $ insertedRowId db + pure (rcvFileEntityId, rcvFileId) + +insertRcvFileChunk :: DB.Connection -> FileChunk -> DBRcvFileId -> IO Int64 +insertRcvFileChunk db FileChunk {chunkNo, chunkSize, digest} rcvFileId = do + DB.execute + db + "INSERT INTO rcv_file_chunks (rcv_file_id, chunk_no, chunk_size, digest) VALUES (?,?,?,?)" + (rcvFileId, chunkNo, chunkSize, digest) + insertedRowId db + +insertRcvFileChunkReplica :: DB.Connection -> Int -> FileChunkReplica -> Int64 -> IO () +insertRcvFileChunkReplica db replicaNo FileChunkReplica {server, replicaId, replicaKey} chunkId = do + srvId <- createXFTPServer_ db server + DB.execute + db + "INSERT INTO rcv_file_chunk_replicas (replica_number, rcv_file_chunk_id, xftp_server_id, replica_id, replica_key) VALUES (?,?,?,?,?)" + (replicaNo, chunkId, srvId, replicaId, replicaKey) getRcvFileByEntityId :: DB.Connection -> RcvFileId -> IO (Either StoreError RcvFile) getRcvFileByEntityId db rcvFileEntityId = runExceptT $ do @@ -2318,17 +2349,21 @@ getRcvFile db rcvFileId = runExceptT $ do DB.query db [sql| - SELECT rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, deleted + SELECT rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, deleted, redirect_id, redirect_entity_id, redirect_size, redirect_digest FROM rcv_files WHERE rcv_file_id = ? |] (Only rcvFileId) where - toFile :: (RcvFileId, UserId, FileSize Int64, FileDigest, C.SbKey, C.CbNonce, FileSize Word32, FilePath, Maybe FilePath) :. (FilePath, Maybe C.SbKey, Maybe C.CbNonce, RcvFileStatus, Bool) -> RcvFile - toFile ((rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, saveKey_, saveNonce_, status, deleted)) = + toFile :: (RcvFileId, UserId, FileSize Int64, FileDigest, C.SbKey, C.CbNonce, FileSize Word32, FilePath, Maybe FilePath) :. (FilePath, Maybe C.SbKey, Maybe C.CbNonce, RcvFileStatus, Bool, Maybe DBRcvFileId, Maybe RcvFileId, Maybe (FileSize Int64), Maybe FileDigest) -> RcvFile + toFile ((rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, saveKey_, saveNonce_, status, deleted, redirectDbId, redirectEntityId, redirectSize_, redirectDigest_)) = let cfArgs = CFArgs <$> saveKey_ <*> saveNonce_ saveFile = CryptoFile savePath cfArgs - in RcvFile {rcvFileId, rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath, saveFile, status, deleted, chunks = []} + redirect = RcvFileRedirect + <$> redirectDbId + <*> redirectEntityId + <*> (RedirectFileInfo <$> redirectSize_ <*> redirectDigest_) + in RcvFile {rcvFileId, rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, redirect, prefixPath, tmpPath, saveFile, status, deleted, chunks = []} getChunks :: RcvFileId -> UserId -> FilePath -> IO [RcvFileChunk] getChunks rcvFileEntityId userId fileTmpPath = do chunks <- @@ -2394,6 +2429,14 @@ updateRcvFileComplete db rcvFileId = do updatedAt <- getCurrentTime DB.execute db "UPDATE rcv_files SET tmp_path = NULL, status = ?, updated_at = ? WHERE rcv_file_id = ?" (RFSComplete, updatedAt, rcvFileId) +updateRcvFileRedirect :: DB.Connection -> DBRcvFileId -> FileDescription 'FRecipient -> IO (Either StoreError ()) +updateRcvFileRedirect db rcvFileId FileDescription {key, nonce, chunkSize, chunks} = runExceptT $ do + updatedAt <- liftIO getCurrentTime + liftIO $ DB.execute db "UPDATE rcv_files SET key = ?, nonce = ?, chunk_size = ?, updated_at = ? WHERE rcv_file_id = ?" (key, nonce, chunkSize, updatedAt, rcvFileId) + liftIO $ forM_ chunks $ \fc@FileChunk {replicas} -> do + chunkId <- insertRcvFileChunk db fc rcvFileId + forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertRcvFileChunkReplica db rno replica chunkId + updateRcvFileNoTmpPath :: DB.Connection -> DBRcvFileId -> IO () updateRcvFileNoTmpPath db rcvFileId = do updatedAt <- getCurrentTime @@ -2541,13 +2584,18 @@ getRcvFilesExpired db ttl = do |] (Only cutoffTs) -createSndFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> CryptoFile -> Int -> FilePath -> C.SbKey -> C.CbNonce -> IO (Either StoreError SndFileId) -createSndFile db gVar userId (CryptoFile path cfArgs) numRecipients prefixPath key nonce = +createSndFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> CryptoFile -> Int -> FilePath -> C.SbKey -> C.CbNonce -> Maybe RedirectFileInfo -> IO (Either StoreError SndFileId) +createSndFile db gVar userId (CryptoFile path cfArgs) numRecipients prefixPath key nonce redirect_ = createWithRandomId gVar $ \sndFileEntityId -> DB.execute db - "INSERT INTO snd_files (snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, prefix_path, key, nonce, status) VALUES (?,?,?,?,?,?,?,?,?,?)" - (sndFileEntityId, userId, path, fileKey <$> cfArgs, fileNonce <$> cfArgs, numRecipients, prefixPath, key, nonce, SFSNew) + "INSERT INTO snd_files (snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, prefix_path, key, nonce, status, redirect_size, redirect_digest) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)" + ((sndFileEntityId, userId, path, fileKey <$> cfArgs, fileNonce <$> cfArgs, numRecipients) :. (prefixPath, key, nonce, SFSNew, redirectSize_, redirectDigest_)) + where + (redirectSize_, redirectDigest_) = + case redirect_ of + Nothing -> (Nothing, Nothing) + Just RedirectFileInfo {size, digest} -> (Just size, Just digest) getSndFileByEntityId :: DB.Connection -> SndFileId -> IO (Either StoreError SndFile) getSndFileByEntityId db sndFileEntityId = runExceptT $ do @@ -2571,17 +2619,18 @@ getSndFile db sndFileId = runExceptT $ do DB.query db [sql| - SELECT snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, digest, prefix_path, key, nonce, status, deleted + SELECT snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, digest, prefix_path, key, nonce, status, deleted, redirect_size, redirect_digest FROM snd_files WHERE snd_file_id = ? |] (Only sndFileId) where - toFile :: (SndFileId, UserId, FilePath, Maybe C.SbKey, Maybe C.CbNonce, Int, Maybe FileDigest, Maybe FilePath, C.SbKey, C.CbNonce, SndFileStatus, Bool) -> SndFile - toFile (sndFileEntityId, userId, srcPath, srcKey_, srcNonce_, numRecipients, digest, prefixPath, key, nonce, status, deleted) = + toFile :: (SndFileId, UserId, FilePath, Maybe C.SbKey, Maybe C.CbNonce, Int, Maybe FileDigest, Maybe FilePath, C.SbKey, C.CbNonce) :. (SndFileStatus, Bool, Maybe (FileSize Int64), Maybe FileDigest) -> SndFile + toFile ((sndFileEntityId, userId, srcPath, srcKey_, srcNonce_, numRecipients, digest, prefixPath, key, nonce) :. (status, deleted, redirectSize_, redirectDigest_)) = let cfArgs = CFArgs <$> srcKey_ <*> srcNonce_ srcFile = CryptoFile srcPath cfArgs - in SndFile {sndFileId, sndFileEntityId, userId, srcFile, numRecipients, digest, prefixPath, key, nonce, status, deleted, chunks = []} + redirect = RedirectFileInfo <$> redirectSize_ <*> redirectDigest_ + in SndFile {sndFileId, sndFileEntityId, userId, srcFile, numRecipients, digest, prefixPath, key, nonce, status, deleted, redirect, chunks = []} getChunks :: SndFileId -> UserId -> Int -> FilePath -> IO [SndFileChunk] getChunks sndFileEntityId userId numRecipients filePrefixPath = do chunks <- diff --git a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations.hs b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations.hs index 2d8ad3a8c..83c900f72 100644 --- a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations.hs +++ b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations.hs @@ -68,6 +68,7 @@ import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20230829_crypto_files import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20231222_command_created_at import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20231225_failed_work_items import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240121_message_delivery_indexes +import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240124_file_redirect import Simplex.Messaging.Encoding.String import Simplex.Messaging.Parsers (dropPrefix, sumTypeJSON) import Simplex.Messaging.Transport.Client (TransportHost) @@ -104,7 +105,8 @@ schemaMigrations = ("m20230829_crypto_files", m20230829_crypto_files, Just down_m20230829_crypto_files), ("m20231222_command_created_at", m20231222_command_created_at, Just down_m20231222_command_created_at), ("m20231225_failed_work_items", m20231225_failed_work_items, Just down_m20231225_failed_work_items), - ("m20240121_message_delivery_indexes", m20240121_message_delivery_indexes, Just down_m20240121_message_delivery_indexes) + ("m20240121_message_delivery_indexes", m20240121_message_delivery_indexes, Just down_m20240121_message_delivery_indexes), + ("m20240124_file_redirect", m20240124_file_redirect, Just down_m20240124_file_redirect) ] -- | The list of migrations in ascending order by date diff --git a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20240124_file_redirect.hs b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20240124_file_redirect.hs new file mode 100644 index 000000000..2062d5d3d --- /dev/null +++ b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20240124_file_redirect.hs @@ -0,0 +1,30 @@ +{-# LANGUAGE QuasiQuotes #-} + +module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240124_file_redirect where + +import Database.SQLite.Simple (Query) +import Database.SQLite.Simple.QQ (sql) + +m20240124_file_redirect :: Query +m20240124_file_redirect = + [sql| +ALTER TABLE snd_files ADD COLUMN redirect_size INTEGER; +ALTER TABLE snd_files ADD COLUMN redirect_digest BLOB; + +ALTER TABLE rcv_files ADD COLUMN redirect_id INTEGER REFERENCES rcv_files ON DELETE CASCADE; +ALTER TABLE rcv_files ADD COLUMN redirect_entity_id BLOB; +ALTER TABLE rcv_files ADD COLUMN redirect_size INTEGER; +ALTER TABLE rcv_files ADD COLUMN redirect_digest BLOB; +|] + +down_m20240124_file_redirect :: Query +down_m20240124_file_redirect = + [sql| +ALTER TABLE snd_files DROP COLUMN redirect_size; +ALTER TABLE snd_files DROP COLUMN redirect_digest; + +ALTER TABLE rcv_files DROP COLUMN redirect_id; +ALTER TABLE rcv_files DROP COLUMN redirect_entity_id; +ALTER TABLE rcv_files DROP COLUMN redirect_size; +ALTER TABLE rcv_files DROP COLUMN redirect_digest; +|] diff --git a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql index 4ff419401..66acc5288 100644 --- a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql +++ b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql @@ -279,6 +279,10 @@ CREATE TABLE rcv_files( save_file_key BLOB, save_file_nonce BLOB, failed INTEGER DEFAULT 0, + redirect_id INTEGER REFERENCES rcv_files ON DELETE CASCADE, + redirect_entity_id BLOB, + redirect_size INTEGER, + redirect_digest BLOB, UNIQUE(rcv_file_entity_id) ); CREATE TABLE rcv_file_chunks( @@ -322,7 +326,9 @@ CREATE TABLE snd_files( , src_file_key BLOB, src_file_nonce BLOB, - failed INTEGER DEFAULT 0 + failed INTEGER DEFAULT 0, + redirect_size INTEGER, + redirect_digest BLOB ); CREATE TABLE snd_file_chunks( snd_file_chunk_id INTEGER PRIMARY KEY, diff --git a/src/Simplex/Messaging/Protocol.hs b/src/Simplex/Messaging/Protocol.hs index f2571a9d4..e8be3b126 100644 --- a/src/Simplex/Messaging/Protocol.hs +++ b/src/Simplex/Messaging/Protocol.hs @@ -173,11 +173,12 @@ import Data.String import Data.Time.Clock.System (SystemTime (..)) import Data.Type.Equality import GHC.TypeLits (ErrorMessage (..), TypeError, type (+)) -import Network.Socket (HostName, ServiceName) +import Network.Socket (ServiceName) import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Encoding import Simplex.Messaging.Encoding.String import Simplex.Messaging.Parsers +import Simplex.Messaging.ServiceScheme import Simplex.Messaging.Transport import Simplex.Messaging.Transport.Client (TransportHost, TransportHosts (..)) import Simplex.Messaging.Util (bshow, eitherToMaybe, (<$?>)) @@ -915,16 +916,6 @@ serverStrP = do where portP = show <$> (A.char ':' *> (A.decimal :: Parser Int)) -data SrvLoc = SrvLoc HostName ServiceName - deriving (Eq, Ord, Show) - -instance StrEncoding SrvLoc where - strEncode (SrvLoc host port) = B.pack $ host <> if null port then "" else ':' : port - strP = SrvLoc <$> host <*> (port <|> pure "") - where - host = B.unpack <$> A.takeWhile1 (A.notInClass ":#,;/ ") - port = show <$> (A.char ':' *> (A.decimal :: Parser Int)) - -- | Transmission correlation ID. newtype CorrId = CorrId {bs :: ByteString} deriving (Eq, Ord, Show) diff --git a/src/Simplex/Messaging/ServiceScheme.hs b/src/Simplex/Messaging/ServiceScheme.hs new file mode 100644 index 000000000..3cd828aa7 --- /dev/null +++ b/src/Simplex/Messaging/ServiceScheme.hs @@ -0,0 +1,35 @@ +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE OverloadedStrings #-} + +module Simplex.Messaging.ServiceScheme where + +import Control.Applicative ((<|>)) +import qualified Data.Attoparsec.ByteString.Char8 as A +import qualified Data.ByteString.Char8 as B +import Data.Functor (($>)) +import Network.Socket (HostName, ServiceName) +import Simplex.Messaging.Encoding.String (StrEncoding (..)) + +data ServiceScheme = SSSimplex | SSAppServer SrvLoc + deriving (Eq, Show) + +instance StrEncoding ServiceScheme where + strEncode = \case + SSSimplex -> "simplex:" + SSAppServer srv -> "https://" <> strEncode srv + strP = + "simplex:" $> SSSimplex + <|> "https://" *> (SSAppServer <$> strP) + +data SrvLoc = SrvLoc HostName ServiceName + deriving (Eq, Ord, Show) + +instance StrEncoding SrvLoc where + strEncode (SrvLoc host port) = B.pack $ host <> if null port then "" else ':' : port + strP = SrvLoc <$> host <*> (port <|> pure "") + where + host = B.unpack <$> A.takeWhile1 (A.notInClass ":#,;/ ") + port = show <$> (A.char ':' *> (A.decimal :: A.Parser Int)) + +simplexChat :: ServiceScheme +simplexChat = SSAppServer $ SrvLoc "simplex.chat" "" diff --git a/tests/AgentTests/ConnectionRequestTests.hs b/tests/AgentTests/ConnectionRequestTests.hs index 9548443a7..e6668a1dd 100644 --- a/tests/AgentTests/ConnectionRequestTests.hs +++ b/tests/AgentTests/ConnectionRequestTests.hs @@ -13,6 +13,7 @@ import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Crypto.Ratchet import Simplex.Messaging.Encoding.String import Simplex.Messaging.Protocol (ProtocolServer (..), supportedSMPClientVRange) +import Simplex.Messaging.ServiceScheme (ServiceScheme (..)) import Simplex.Messaging.Version import Test.Hspec @@ -51,7 +52,7 @@ testDhKeyStrUri = urlEncode True testDhKeyStr connReqData :: ConnReqUriData connReqData = ConnReqUriData - { crScheme = CRSSimplex, + { crScheme = SSSimplex, crAgentVRange = mkVersionRange 1 1, crSmpQueues = [queueV1], crClientData = Nothing diff --git a/tests/AgentTests/SQLiteTests.hs b/tests/AgentTests/SQLiteTests.hs index 98fb47b80..9d258bdf7 100644 --- a/tests/AgentTests/SQLiteTests.hs +++ b/tests/AgentTests/SQLiteTests.hs @@ -658,7 +658,8 @@ rcvFileDescr1 = chunkSize = defaultChunkSize, replicas = [FileChunkReplica {server = xftpServer1, replicaId, replicaKey = testFileReplicaKey}] } - ] + ], + redirect = Nothing } where defaultChunkSize = FileSize $ mb 8 @@ -716,9 +717,9 @@ testGetNextSndFileToPrepare st = do withTransaction st $ \db -> do Right Nothing <- getNextSndFileToPrepare db 86400 - Right _ <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce + Right _ <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce Nothing DB.execute_ db "UPDATE snd_files SET status = 'new', num_recipients = 'bad' WHERE snd_file_id = 1" - Right fId2 <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce + Right fId2 <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce Nothing DB.execute_ db "UPDATE snd_files SET status = 'new' WHERE snd_file_id = 2" Left e <- getNextSndFileToPrepare db 86400 @@ -744,12 +745,12 @@ testGetNextSndChunkToUpload st = do Right Nothing <- getNextSndChunkToUpload db xftpServer1 86400 -- create file 1 - Right _ <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce + Right _ <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce Nothing updateSndFileEncrypted db 1 (FileDigest "abc") [(XFTPChunkSpec "filepath" 1 1, FileDigest "ghi")] createSndFileReplica_ db 1 newSndChunkReplica1 DB.execute_ db "UPDATE snd_files SET num_recipients = 'bad' WHERE snd_file_id = 1" -- create file 2 - Right fId2 <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce + Right fId2 <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce Nothing updateSndFileEncrypted db 2 (FileDigest "abc") [(XFTPChunkSpec "filepath" 1 1, FileDigest "ghi")] createSndFileReplica_ db 2 newSndChunkReplica1 diff --git a/tests/FileDescriptionTests.hs b/tests/FileDescriptionTests.hs index 676882bcd..61dd638fe 100644 --- a/tests/FileDescriptionTests.hs +++ b/tests/FileDescriptionTests.hs @@ -13,16 +13,20 @@ import Simplex.FileTransfer.Description import Simplex.FileTransfer.Protocol import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Encoding.String (StrEncoding (..)) +import Simplex.Messaging.ServiceScheme (ServiceScheme (..)) import System.Directory (removeFile) import Test.Hspec fileDescriptionTests :: Spec -fileDescriptionTests = +fileDescriptionTests = do describe "file description parsing / serializing" $ do it "parse YAML file description" testParseYAMLFileDescription it "serialize YAML file description" testSerializeYAMLFileDescription it "parse file description" testParseFileDescription it "serialize file description" testSerializeFileDescription + describe "file description URIs" $ do + it "round trip file description URI" testFileDescriptionURI + it "round trip file description URI with extra JSON" testFileDescriptionURIExtras fileDescPath :: FilePath fileDescPath = "tests/fixtures/file_description.yaml" @@ -82,7 +86,8 @@ fileDesc = FileChunkReplica {server = "xftp://abc=@example3.com", replicaId, replicaKey} ] } - ] + ], + redirect = Nothing } where defaultChunkSize = FileSize $ mb 8 @@ -128,7 +133,8 @@ yamlFileDesc = "3:YWJj:MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe" ] } - ] + ], + redirect = Nothing } testParseYAMLFileDescription :: IO () @@ -157,6 +163,18 @@ testSerializeFileDescription = withRemoveTmpFile $ do fdExp <- B.readFile fileDescPath fdSer `shouldBe` fdExp +testFileDescriptionURI :: IO () +testFileDescriptionURI = do + vfd <- either fail pure $ validateFileDescription fileDesc + let descr = FileDescriptionURI SSSimplex vfd mempty + strDecode (strEncode descr) `shouldBe` Right descr + +testFileDescriptionURIExtras :: IO () +testFileDescriptionURIExtras = do + vfd <- either fail pure $ validateFileDescription fileDesc + let descr = FileDescriptionURI SSSimplex vfd $ Just "{\"something\":\"extra\",\"more\":true}" + strDecode (strEncode descr) `shouldBe` Right descr + withRemoveTmpFile :: IO () -> IO () withRemoveTmpFile = bracket_ diff --git a/tests/XFTPAgent.hs b/tests/XFTPAgent.hs index 07ac62da6..69f067731 100644 --- a/tests/XFTPAgent.hs +++ b/tests/XFTPAgent.hs @@ -1,14 +1,15 @@ {-# LANGUAGE DataKinds #-} {-# LANGUAGE GADTs #-} +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PatternSynonyms #-} {-# LANGUAGE ScopedTypeVariables #-} module XFTPAgent where import AgentTests.FunctionalAPITests (get, getSMPAgentClient', rfGet, runRight, runRight_, sfGet) -import Control.Concurrent (threadDelay) -import Control.Concurrent.STM + import Control.Logger.Simple import Control.Monad import Control.Monad.Except @@ -19,10 +20,10 @@ import Data.Int (Int64) import Data.List (find, isSuffixOf) import Data.Maybe (fromJust) import SMPAgentClient (agentCfg, initAgentServers, testDB, testDB2, testDB3) -import Simplex.FileTransfer.Description +import Simplex.FileTransfer.Description (FileDescription (..), FileDescriptionURI (..), ValidFileDescription, fileDescriptionURI, mb, qrSizeLimit, pattern ValidFileDescription) import Simplex.FileTransfer.Protocol (FileParty (..), XFTPErrorType (AUTH)) import Simplex.FileTransfer.Server.Env (XFTPServerConfig (..)) -import Simplex.Messaging.Agent (AgentClient, disconnectAgentClient, testProtocolServer, xftpDeleteRcvFile, xftpDeleteSndFileInternal, xftpDeleteSndFileRemote, xftpReceiveFile, xftpSendFile, xftpStartWorkers) +import Simplex.Messaging.Agent (AgentClient, disconnectAgentClient, testProtocolServer, xftpDeleteRcvFile, xftpDeleteSndFileInternal, xftpDeleteSndFileRemote, xftpReceiveFile, xftpSendDescription, xftpSendFile, xftpStartWorkers) import Simplex.Messaging.Agent.Client (ProtocolTestFailure (..), ProtocolTestStep (..)) import Simplex.Messaging.Agent.Protocol (ACommand (..), AgentErrorType (..), BrokerErrorType (..), RcvFileId, SndFileId, noAuthSrv) import qualified Simplex.Messaging.Crypto as C @@ -31,10 +32,12 @@ import qualified Simplex.Messaging.Crypto.File as CF import Simplex.Messaging.Encoding.String (StrEncoding (..)) import Simplex.Messaging.Protocol (BasicAuth, ProtoServerWithAuth (..), ProtocolServer (..), XFTPServerWithAuth) import Simplex.Messaging.Server.Expiration (ExpirationConfig (..)) +import Simplex.Messaging.Util (tshow) import System.Directory (doesDirectoryExist, doesFileExist, getFileSize, listDirectory, removeFile) import System.FilePath (()) -import System.Timeout (timeout) import Test.Hspec +import UnliftIO +import UnliftIO.Concurrent import XFTPCLI import XFTPClient @@ -42,6 +45,8 @@ xftpAgentTests :: Spec xftpAgentTests = around_ testBracket . describe "agent XFTP API" $ do it "should send and receive file" testXFTPAgentSendReceive it "should send and receive with encrypted local files" testXFTPAgentSendReceiveEncrypted + it "should send and receive large file with a redirect" testXFTPAgentSendReceiveRedirect + it "should send and receive small file without a redirect" testXFTPAgentSendReceiveNoRedirect it "should resume receiving file after restart" testXFTPAgentReceiveRestore it "should cleanup rcv tmp path after permanent error" testXFTPAgentReceiveCleanup it "should resume sending file after restart" testXFTPAgentSendRestore @@ -135,14 +140,112 @@ testXFTPAgentSendReceiveEncrypted = withXFTPServer $ do xftpDeleteRcvFile rcp rfId disconnectAgentClient rcp +testXFTPAgentSendReceiveRedirect :: HasCallStack => IO () +testXFTPAgentSendReceiveRedirect = withXFTPServer $ do + --- sender + filePathIn <- createRandomFile + let fileSize = mb 17 + totalSize = fileSize + mb 1 + sndr <- getSMPAgentClient' 1 agentCfg initAgentServers testDB + directFileId <- runRight $ xftpSendFile sndr 1 (CryptoFile filePathIn Nothing) 1 + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 4194304 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 8388608 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 12582912 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 16777216 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 17825792 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG totalSize totalSize) + vfdDirect <- + sfGet sndr >>= \case + (_, _, SFDONE _snd (vfd : _)) -> pure vfd + r -> error $ "Expected SFDONE, got " <> show r + redirectFileId <- runRight $ xftpSendDescription sndr 1 vfdDirect + logInfo $ "File sent, sending redirect: " <> tshow redirectFileId + sfGet sndr `shouldReturn` ("", redirectFileId, SFPROG 65536 65536) + vfdRedirect@(ValidFileDescription fdRedirect) <- + sfGet sndr >>= \case + (_, _, SFDONE _snd (vfd : _)) -> pure vfd + r -> error $ "Expected SFDONE, got " <> show r + case fdRedirect of + FileDescription {redirect = Just _} -> pure () + _ -> error "missing RedirectFileInfo" + let uri = strEncode $ fileDescriptionURI vfdRedirect + case strDecode uri of + Left err -> fail err + Right ok -> ok `shouldBe` fileDescriptionURI vfdRedirect + disconnectAgentClient sndr + --- recipient + rcp <- getSMPAgentClient' 2 agentCfg initAgentServers testDB2 + FileDescriptionURI {description} <- either fail pure $ strDecode uri + + rcvFileId <- runRight $ xftpReceiveFile rcp 1 description Nothing + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 65536 totalSize) -- extra RFPROG before switching to real file + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 4194304 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 8388608 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 12582912 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 16777216 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 17825792 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG totalSize totalSize) + out <- + rfGet rcp >>= \case + (_, _, RFDONE out) -> pure out + r -> error $ "Expected RFDONE, got " <> show r + disconnectAgentClient rcp + + inBytes <- B.readFile filePathIn + B.readFile out `shouldReturn` inBytes + +testXFTPAgentSendReceiveNoRedirect :: HasCallStack => IO () +testXFTPAgentSendReceiveNoRedirect = withXFTPServer $ do + --- sender + let fileSize = mb 5 + filePathIn <- createRandomFile_ fileSize "testfile" + sndr <- getSMPAgentClient' 1 agentCfg initAgentServers testDB + directFileId <- runRight $ xftpSendFile sndr 1 (CryptoFile filePathIn Nothing) 1 + let totalSize = fileSize + mb 1 + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 4194304 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG 5242880 totalSize) + sfGet sndr `shouldReturn` ("", directFileId, SFPROG totalSize totalSize) + vfdDirect <- + sfGet sndr >>= \case + (_, _, SFDONE _snd (vfd : _)) -> pure vfd + r -> error $ "Expected SFDONE, got " <> show r + B.putStrLn $ strEncode vfdDirect + let uri = strEncode $ fileDescriptionURI vfdDirect + B.length uri `shouldSatisfy` (< qrSizeLimit) + case strDecode uri of + Left err -> fail err + Right ok -> ok `shouldBe` fileDescriptionURI vfdDirect + disconnectAgentClient sndr + --- recipient + rcp <- getSMPAgentClient' 2 agentCfg initAgentServers testDB2 + FileDescriptionURI {description} <- either fail pure $ strDecode uri + let ValidFileDescription FileDescription {redirect} = description + redirect `shouldBe` Nothing + rcvFileId <- runRight $ xftpReceiveFile rcp 1 description Nothing + -- NO extra "RFPROG 65k 65k" before switching to real file + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 4194304 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG 5242880 totalSize) + rfGet rcp `shouldReturn` ("", rcvFileId, RFPROG totalSize totalSize) + out <- + rfGet rcp >>= \case + (_, _, RFDONE out) -> pure out + r -> error $ "Expected RFDONE, got " <> show r + disconnectAgentClient rcp + + inBytes <- B.readFile filePathIn + B.readFile out `shouldReturn` inBytes + createRandomFile :: HasCallStack => IO FilePath createRandomFile = createRandomFile' "testfile" createRandomFile' :: HasCallStack => FilePath -> IO FilePath -createRandomFile' fileName = do +createRandomFile' = createRandomFile_ (mb 17 :: Integer) + +createRandomFile_ :: (HasCallStack, Integral s, Show s) => s -> FilePath -> IO FilePath +createRandomFile_ size fileName = do let filePath = senderFiles fileName - xftpCLI ["rand", filePath, "17mb"] `shouldReturn` ["File created: " <> filePath] - getFileSize filePath `shouldReturn` mb 17 + xftpCLI ["rand", filePath, show size] `shouldReturn` ["File created: " <> filePath] + getFileSize filePath `shouldReturn` toInteger size pure filePath testSend :: HasCallStack => AgentClient -> FilePath -> ExceptT AgentErrorType IO (SndFileId, ValidFileDescription 'FSender, ValidFileDescription 'FRecipient, ValidFileDescription 'FRecipient) diff --git a/tests/XFTPCLI.hs b/tests/XFTPCLI.hs index 14c62aea5..567db6f9b 100644 --- a/tests/XFTPCLI.hs +++ b/tests/XFTPCLI.hs @@ -152,7 +152,7 @@ testPrepareChunkSizes = do prepareChunkSizes (mb 2 + 1) `shouldBe` [mb 1, mb 1, kb 256] prepareChunkSizes (3 * kb 256 + 1) `shouldBe` [mb 1] prepareChunkSizes (3 * kb 256) `shouldBe` r3 (kb 256) - prepareChunkSizes 1 `shouldBe` [kb 256] + prepareChunkSizes 1 `shouldBe` [kb 64] where r3 = replicate 3 diff --git a/tests/XFTPClient.hs b/tests/XFTPClient.hs index 312135072..af4e6dd36 100644 --- a/tests/XFTPClient.hs +++ b/tests/XFTPClient.hs @@ -102,7 +102,7 @@ testXFTPServerConfig = storeLogFile = Nothing, filesPath = xftpServerFiles, fileSizeQuota = Nothing, - allowedChunkSizes = [kb 128, kb 256, mb 1, mb 4], + allowedChunkSizes = [kb 64, kb 128, kb 256, mb 1, mb 4], allowNewFiles = True, newFileBasicAuth = Nothing, fileExpiration = Just defaultFileExpiration, From 004597c764a8f8d5dc1e03fbad713b02a878c2ed Mon Sep 17 00:00:00 2001 From: spaced4ndy <8711996+spaced4ndy@users.noreply.github.com> Date: Wed, 14 Feb 2024 18:47:39 +0400 Subject: [PATCH 4/8] agent: add index to file redirect migration (#988) --- .../Agent/Store/SQLite/Migrations/M20240124_file_redirect.hs | 4 ++++ .../Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql | 1 + 2 files changed, 5 insertions(+) diff --git a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20240124_file_redirect.hs b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20240124_file_redirect.hs index 2062d5d3d..e02932d63 100644 --- a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20240124_file_redirect.hs +++ b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20240124_file_redirect.hs @@ -15,11 +15,15 @@ ALTER TABLE rcv_files ADD COLUMN redirect_id INTEGER REFERENCES rcv_files ON DEL ALTER TABLE rcv_files ADD COLUMN redirect_entity_id BLOB; ALTER TABLE rcv_files ADD COLUMN redirect_size INTEGER; ALTER TABLE rcv_files ADD COLUMN redirect_digest BLOB; + +CREATE INDEX idx_rcv_files_redirect_id on rcv_files(redirect_id); |] down_m20240124_file_redirect :: Query down_m20240124_file_redirect = [sql| +DROP INDEX idx_rcv_files_redirect_id; + ALTER TABLE snd_files DROP COLUMN redirect_size; ALTER TABLE snd_files DROP COLUMN redirect_digest; diff --git a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql index 66acc5288..32df1520d 100644 --- a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql +++ b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql @@ -514,3 +514,4 @@ CREATE INDEX idx_snd_message_deliveries_expired ON snd_message_deliveries( failed, internal_id ); +CREATE INDEX idx_rcv_files_redirect_id on rcv_files(redirect_id); From 7275714b8e5c6e21d7ed326cb8273cf9a8053de7 Mon Sep 17 00:00:00 2001 From: Alexander Bondarenko <486682+dpwiz@users.noreply.github.com> Date: Wed, 14 Feb 2024 22:31:06 +0200 Subject: [PATCH 5/8] cli: configure server paths from env (#992) --- apps/ntf-server/Main.hs | 11 +++++++---- apps/smp-server/Main.hs | 5 +---- apps/xftp-server/Main.hs | 11 +++++++---- src/Simplex/Messaging/Server/CLI.hs | 4 ++++ 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/apps/ntf-server/Main.hs b/apps/ntf-server/Main.hs index aa62d35d8..31dbaec07 100644 --- a/apps/ntf-server/Main.hs +++ b/apps/ntf-server/Main.hs @@ -1,13 +1,14 @@ module Main where import Control.Logger.Simple +import Simplex.Messaging.Server.CLI (getEnvPath) import Simplex.Messaging.Notifications.Server.Main -cfgPath :: FilePath -cfgPath = "/etc/opt/simplex-notifications" +defaultCfgPath :: FilePath +defaultCfgPath = "/etc/opt/simplex-notifications" -logPath :: FilePath -logPath = "/var/opt/simplex-notifications" +defaultLogPath :: FilePath +defaultLogPath = "/var/opt/simplex-notifications" logCfg :: LogConfig logCfg = LogConfig {lc_file = Nothing, lc_stderr = True} @@ -15,4 +16,6 @@ logCfg = LogConfig {lc_file = Nothing, lc_stderr = True} main :: IO () main = do setLogLevel LogDebug -- change to LogError in production + cfgPath <- getEnvPath "NTF_SERVER_CFG_PATH" defaultCfgPath + logPath <- getEnvPath "NTF_SERVER_LOG_PATH" defaultLogPath withGlobalLogging logCfg $ ntfServerCLI cfgPath logPath diff --git a/apps/smp-server/Main.hs b/apps/smp-server/Main.hs index 92bdfded4..d5cc5e732 100644 --- a/apps/smp-server/Main.hs +++ b/apps/smp-server/Main.hs @@ -3,8 +3,8 @@ module Main where import Control.Logger.Simple +import Simplex.Messaging.Server.CLI (getEnvPath) import Simplex.Messaging.Server.Main -import System.Environment defaultCfgPath :: FilePath defaultCfgPath = "/etc/opt/simplex" @@ -21,6 +21,3 @@ main = do cfgPath <- getEnvPath "SMP_SERVER_CFG_PATH" defaultCfgPath logPath <- getEnvPath "SMP_SERVER_LOG_PATH" defaultLogPath withGlobalLogging logCfg $ smpServerCLI cfgPath logPath - -getEnvPath :: String -> FilePath -> IO FilePath -getEnvPath name def = maybe def (\case "" -> def; f -> f) <$> lookupEnv name diff --git a/apps/xftp-server/Main.hs b/apps/xftp-server/Main.hs index ca3528872..269156c93 100644 --- a/apps/xftp-server/Main.hs +++ b/apps/xftp-server/Main.hs @@ -1,13 +1,14 @@ module Main where import Control.Logger.Simple +import Simplex.Messaging.Server.CLI (getEnvPath) import Simplex.FileTransfer.Server.Main -cfgPath :: FilePath -cfgPath = "/etc/opt/simplex-xftp" +defaultCfgPath :: FilePath +defaultCfgPath = "/etc/opt/simplex-xftp" -logPath :: FilePath -logPath = "/var/opt/simplex-xftp" +defaultLogPath :: FilePath +defaultLogPath = "/var/opt/simplex-xftp" logCfg :: LogConfig logCfg = LogConfig {lc_file = Nothing, lc_stderr = True} @@ -15,4 +16,6 @@ logCfg = LogConfig {lc_file = Nothing, lc_stderr = True} main :: IO () main = do setLogLevel LogDebug -- change to LogError in production + cfgPath <- getEnvPath "XFTP_SERVER_CFG_PATH" defaultCfgPath + logPath <- getEnvPath "XFTP_SERVER_LOG_PATH" defaultLogPath withGlobalLogging logCfg $ xftpServerCLI cfgPath logPath diff --git a/src/Simplex/Messaging/Server/CLI.hs b/src/Simplex/Messaging/Server/CLI.hs index 26e3dfa42..f9dcc945a 100644 --- a/src/Simplex/Messaging/Server/CLI.hs +++ b/src/Simplex/Messaging/Server/CLI.hs @@ -27,6 +27,7 @@ import Simplex.Messaging.Transport.Server (loadFingerprint) import Simplex.Messaging.Transport.WebSockets (WS) import Simplex.Messaging.Util (eitherToMaybe, whenM) import System.Directory (doesDirectoryExist, listDirectory, removeDirectoryRecursive, removePathForcibly) +import System.Environment (lookupEnv) import System.Exit (exitFailure) import System.FilePath (combine) import System.IO (IOMode (..), hFlush, hGetLine, stdout, withFile) @@ -239,3 +240,6 @@ printServiceInfo serverVersion srv@(ProtoServerWithAuth ProtocolServer {keyHash} clearDirIfExists :: FilePath -> IO () clearDirIfExists path = whenM (doesDirectoryExist path) $ listDirectory path >>= mapM_ (removePathForcibly . combine path) + +getEnvPath :: String -> FilePath -> IO FilePath +getEnvPath name def = maybe def (\case "" -> def; f -> f) <$> lookupEnv name From 6f62d7ff05b9d25bca4f822c04ee83a102c34e42 Mon Sep 17 00:00:00 2001 From: spaced4ndy <8711996+spaced4ndy@users.noreply.github.com> Date: Thu, 15 Feb 2024 13:24:46 +0400 Subject: [PATCH 6/8] agent: add numRecipients parameter to send description (#993) --- src/Simplex/FileTransfer/Agent.hs | 6 +++--- src/Simplex/Messaging/Agent.hs | 4 ++-- tests/XFTPAgent.hs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Simplex/FileTransfer/Agent.hs b/src/Simplex/FileTransfer/Agent.hs index 18f39de42..3bed1ce21 100644 --- a/src/Simplex/FileTransfer/Agent.hs +++ b/src/Simplex/FileTransfer/Agent.hs @@ -311,8 +311,8 @@ xftpSendFile' c userId file numRecipients = do void $ getXFTPSndWorker True c Nothing pure fId -xftpSendDescription' :: forall m. AgentMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> m SndFileId -xftpSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) = do +xftpSendDescription' :: forall m. AgentMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> m SndFileId +xftpSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) numRecipients = do g <- asks random prefixPath <- getPrefixPath "snd.xftp" createDirectory prefixPath @@ -323,7 +323,7 @@ xftpSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {si liftError (INTERNAL . show) $ CF.writeFile file (LB.fromStrict $ strEncode fdDirect) key <- atomically $ C.randomSbKey g nonce <- atomically $ C.randomCbNonce g - fId <- withStore c $ \db -> createSndFile db g userId file 1 relPrefixPath key nonce $ Just RedirectFileInfo {size, digest} + fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce $ Just RedirectFileInfo {size, digest} void $ getXFTPSndWorker True c Nothing pure fId diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs index 911358992..da309d545 100644 --- a/src/Simplex/Messaging/Agent.hs +++ b/src/Simplex/Messaging/Agent.hs @@ -403,8 +403,8 @@ xftpSendFile :: AgentErrorMonad m => AgentClient -> UserId -> CryptoFile -> Int xftpSendFile c = withAgentEnv c .:. xftpSendFile' c -- | Send XFTP file -xftpSendDescription :: AgentErrorMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> m SndFileId -xftpSendDescription c = withAgentEnv c .: xftpSendDescription' c +xftpSendDescription :: AgentErrorMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> m SndFileId +xftpSendDescription c = withAgentEnv c .:. xftpSendDescription' c -- | Delete XFTP snd file internally (deletes work files from file system and db records) xftpDeleteSndFileInternal :: AgentErrorMonad m => AgentClient -> SndFileId -> m () diff --git a/tests/XFTPAgent.hs b/tests/XFTPAgent.hs index 69f067731..6ac64a4c1 100644 --- a/tests/XFTPAgent.hs +++ b/tests/XFTPAgent.hs @@ -158,7 +158,7 @@ testXFTPAgentSendReceiveRedirect = withXFTPServer $ do sfGet sndr >>= \case (_, _, SFDONE _snd (vfd : _)) -> pure vfd r -> error $ "Expected SFDONE, got " <> show r - redirectFileId <- runRight $ xftpSendDescription sndr 1 vfdDirect + redirectFileId <- runRight $ xftpSendDescription sndr 1 vfdDirect 1 logInfo $ "File sent, sending redirect: " <> tshow redirectFileId sfGet sndr `shouldReturn` ("", redirectFileId, SFPROG 65536 65536) vfdRedirect@(ValidFileDescription fdRedirect) <- From 9ab34bca7d706339c7c0733aebbd2dab96a3c17c Mon Sep 17 00:00:00 2001 From: Alexander Bondarenko <486682+dpwiz@users.noreply.github.com> Date: Fri, 16 Feb 2024 12:56:54 +0200 Subject: [PATCH 7/8] cli: add cert command to xftp and ntf servers (#991) Co-authored-by: Evgeny Poberezkin --- src/Simplex/FileTransfer/Server/Main.hs | 6 ++ .../Messaging/Notifications/Server/Main.hs | 6 ++ src/Simplex/Messaging/Server/CLI.hs | 58 ++++++++++++++++++ src/Simplex/Messaging/Server/Main.hs | 60 +------------------ 4 files changed, 73 insertions(+), 57 deletions(-) diff --git a/src/Simplex/FileTransfer/Server/Main.hs b/src/Simplex/FileTransfer/Server/Main.hs index 4f5d0558c..9d55f250d 100644 --- a/src/Simplex/FileTransfer/Server/Main.hs +++ b/src/Simplex/FileTransfer/Server/Main.hs @@ -42,6 +42,10 @@ xftpServerCLI cfgPath logPath = do doesFileExist iniFile >>= \case True -> exitError $ "Error: server is already initialized (" <> iniFile <> " exists).\nRun `" <> executableName <> " start`." _ -> initializeServer opts + OnlineCert certOpts -> + doesFileExist iniFile >>= \case + True -> genOnline cfgPath certOpts + _ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`." Start -> doesFileExist iniFile >>= \case True -> readIniFile iniFile >>= either exitError runServer @@ -179,6 +183,7 @@ xftpServerCLI cfgPath logPath = do data CliCommand = Init InitOptions + | OnlineCert CertOptions | Start | Delete @@ -196,6 +201,7 @@ cliCommandP :: FilePath -> FilePath -> FilePath -> Parser CliCommand cliCommandP cfgPath logPath iniFile = hsubparser ( command "init" (info (Init <$> initP) (progDesc $ "Initialize server - creates " <> cfgPath <> " and " <> logPath <> " directories and configuration files")) + <> command "cert" (info (OnlineCert <$> certOptionsP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")")) <> command "start" (info (pure Start) (progDesc $ "Start server (configuration: " <> iniFile <> ")")) <> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files")) ) diff --git a/src/Simplex/Messaging/Notifications/Server/Main.hs b/src/Simplex/Messaging/Notifications/Server/Main.hs index 53d134abd..f3d47e5c3 100644 --- a/src/Simplex/Messaging/Notifications/Server/Main.hs +++ b/src/Simplex/Messaging/Notifications/Server/Main.hs @@ -42,6 +42,10 @@ ntfServerCLI cfgPath logPath = doesFileExist iniFile >>= \case True -> exitError $ "Error: server is already initialized (" <> iniFile <> " exists).\nRun `" <> executableName <> " start`." _ -> initializeServer opts + OnlineCert certOpts -> + doesFileExist iniFile >>= \case + True -> genOnline cfgPath certOpts + _ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`." Start -> doesFileExist iniFile >>= \case True -> readIniFile iniFile >>= either exitError runServer @@ -143,6 +147,7 @@ ntfServerCLI cfgPath logPath = data CliCommand = Init InitOptions + | OnlineCert CertOptions | Start | Delete @@ -158,6 +163,7 @@ cliCommandP :: FilePath -> FilePath -> FilePath -> Parser CliCommand cliCommandP cfgPath logPath iniFile = hsubparser ( command "init" (info (Init <$> initP) (progDesc $ "Initialize server - creates " <> cfgPath <> " and " <> logPath <> " directories and configuration files")) + <> command "cert" (info (OnlineCert <$> certOptionsP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")")) <> command "start" (info (pure Start) (progDesc $ "Start server (configuration: " <> iniFile <> ")")) <> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files")) ) diff --git a/src/Simplex/Messaging/Server/CLI.hs b/src/Simplex/Messaging/Server/CLI.hs index f9dcc945a..9531a2ca5 100644 --- a/src/Simplex/Messaging/Server/CLI.hs +++ b/src/Simplex/Messaging/Server/CLI.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE ApplicativeDo #-} {-# LANGUAGE DataKinds #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE FlexibleContexts #-} @@ -10,6 +11,7 @@ module Simplex.Messaging.Server.CLI where import Control.Monad +import Data.ASN1.Types (asn1CharacterToString) import Data.ByteString.Char8 (ByteString) import qualified Data.ByteString.Char8 as B import Data.Either (fromRight) @@ -17,6 +19,8 @@ import Data.Ini (Ini, lookupValue) import Data.Text (Text) import qualified Data.Text as T import Data.Text.Encoding (encodeUtf8) +import qualified Data.X509 as X +import qualified Data.X509.File as XF import Data.X509.Validation (Fingerprint (..)) import Network.Socket (HostName, ServiceName) import Options.Applicative @@ -32,6 +36,7 @@ import System.Exit (exitFailure) import System.FilePath (combine) import System.IO (IOMode (..), hFlush, hGetLine, stdout, withFile) import System.Process (readCreateProcess, shell) +import Text.Read (readMaybe) exitError :: String -> IO a exitError msg = putStrLn msg >> exitFailure @@ -136,6 +141,59 @@ createServerX509_ createCA cfgPath x509cfg = do withFile (c fingerprintFile) WriteMode (`B.hPutStrLn` strEncode fp) pure fp +data CertOptions = CertOptions + { signAlgorithm_ :: Maybe SignAlgorithm, + commonName_ :: Maybe HostName + } + deriving (Show) + +certOptionsP :: Parser CertOptions +certOptionsP = do + signAlgorithm_ <- + optional $ + option + (maybeReader readMaybe) + ( long "sign-algorithm" + <> short 'a' + <> help "Set new signature algorithm used for TLS certificates: ED25519, ED448" + <> metavar "ALG" + ) + commonName_ <- + optional $ + strOption + ( long "cn" + <> help + "Set new Common Name for TLS online certificate" + <> metavar "FQDN" + ) + pure CertOptions {signAlgorithm_, commonName_} + +genOnline :: FilePath -> CertOptions -> IO () +genOnline cfgPath CertOptions {signAlgorithm_, commonName_} = do + (signAlgorithm, commonName) <- + case (signAlgorithm_, commonName_) of + (Just alg, Just cn) -> pure (alg, cn) + _ -> + XF.readSignedObject certPath >>= \case + [old] -> either exitError pure . fromX509 . X.signedObject $ X.getSigned old + [] -> exitError $ "No certificate found at " <> certPath + _ -> exitError $ "Too many certificates at " <> certPath + let x509cfg = defaultX509Config {signAlgorithm, commonName} + void $ createServerX509_ False cfgPath x509cfg + putStrLn "Generated new server credentials" + warnCAPrivateKeyFile cfgPath x509cfg + where + certPath = combine cfgPath $ serverCrtFile defaultX509Config + fromX509 X.Certificate {certSignatureAlg, certSubjectDN} = (,) <$> maybe oldAlg Right signAlgorithm_ <*> maybe oldCN Right commonName_ + where + oldAlg = case certSignatureAlg of + X.SignatureALG_IntrinsicHash X.PubKeyALG_Ed448 -> Right ED448 + X.SignatureALG_IntrinsicHash X.PubKeyALG_Ed25519 -> Right ED25519 + alg -> Left $ "Unexpected signature algorithm " <> show alg + oldCN = case X.getDnElement X.DnCommonName certSubjectDN of + Nothing -> Left "Certificate subject has no CN element" + Just cn -> maybe (Left "Certificate subject CN decoding failed") Right $ asn1CharacterToString cn + warnCAPrivateKeyFile :: FilePath -> X509Config -> IO () warnCAPrivateKeyFile cfgPath X509Config {caKeyFile} = putStrLn $ diff --git a/src/Simplex/Messaging/Server/Main.hs b/src/Simplex/Messaging/Server/Main.hs index f52f0311b..13b2af4a3 100644 --- a/src/Simplex/Messaging/Server/Main.hs +++ b/src/Simplex/Messaging/Server/Main.hs @@ -10,15 +10,12 @@ module Simplex.Messaging.Server.Main where import Control.Concurrent.STM import Control.Monad (void) -import Data.ASN1.Types.String (asn1CharacterToString) import qualified Data.ByteString.Char8 as B import Data.Functor (($>)) import Data.Ini (lookupValue, readIniFile) import Data.Maybe (fromMaybe) import qualified Data.Text as T import Data.Text.Encoding (encodeUtf8) -import qualified Data.X509 as X -import qualified Data.X509.File as XF import Network.Socket (HostName) import Options.Applicative import qualified Simplex.Messaging.Crypto as C @@ -46,7 +43,7 @@ smpServerCLI cfgPath logPath = _ -> initializeServer opts OnlineCert certOpts -> doesFileExist iniFile >>= \case - True -> genOnline certOpts + True -> genOnline cfgPath certOpts _ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`." Start -> doesFileExist iniFile >>= \case @@ -143,30 +140,6 @@ smpServerCLI cfgPath logPath = \disconnect: off\n" <> ("# ttl: " <> show (ttl defaultInactiveClientExpiration) <> "\n") <> ("# check_interval: " <> show (checkInterval defaultInactiveClientExpiration) <> "\n") - genOnline CertOptions {signAlgorithm_, commonName_} = do - (signAlgorithm, commonName) <- - case (signAlgorithm_, commonName_) of - (Just alg, Just cn) -> pure (alg, cn) - _ -> - XF.readSignedObject certPath >>= \case - [old] -> either exitError pure . fromX509 . X.signedObject $ X.getSigned old - [] -> exitError $ "No certificate found at " <> certPath - _ -> exitError $ "Too many certificates at " <> certPath - let x509cfg = defaultX509Config {signAlgorithm, commonName} - createServerX509_ False cfgPath x509cfg - putStrLn "Generated new server credentials" - warnCAPrivateKeyFile cfgPath x509cfg - where - certPath = combine cfgPath $ serverCrtFile defaultX509Config - fromX509 X.Certificate {certSignatureAlg, certSubjectDN} = (,) <$> maybe oldAlg Right signAlgorithm_ <*> maybe oldCN Right commonName_ - where - oldAlg = case certSignatureAlg of - X.SignatureALG_IntrinsicHash X.PubKeyALG_Ed448 -> Right ED448 - X.SignatureALG_IntrinsicHash X.PubKeyALG_Ed25519 -> Right ED25519 - alg -> Left $ "Unexpected signature algorithm " <> show alg - oldCN = case X.getDnElement X.DnCommonName certSubjectDN of - Nothing -> Left "Certificate subject has no CN element" - Just cn -> maybe (Left "Certificate subject CN decoding failed") Right $ asn1CharacterToString cn runServer ini = do hSetBuffering stdout LineBuffering hSetBuffering stderr LineBuffering @@ -259,17 +232,11 @@ data InitOptions = InitOptions data ServerPassword = ServerPassword BasicAuth | SPRandom deriving (Show) -data CertOptions = CertOptions - { signAlgorithm_ :: Maybe SignAlgorithm, - commonName_ :: Maybe HostName - } - deriving (Show) - cliCommandP :: FilePath -> FilePath -> FilePath -> Parser CliCommand cliCommandP cfgPath logPath iniFile = hsubparser ( command "init" (info (Init <$> initP) (progDesc $ "Initialize server - creates " <> cfgPath <> " and " <> logPath <> " directories and configuration files")) - <> command "cert" (info (OnlineCert <$> certP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")")) + <> command "cert" (info (OnlineCert <$> certOptionsP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")")) <> command "start" (info (pure Start) (progDesc $ "Start server (configuration: " <> iniFile <> ")")) <> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files")) ) @@ -332,27 +299,6 @@ cliCommandP cfgPath logPath iniFile = <> help "Non-interactive initialization using command-line options" ) pure InitOptions {enableStoreLog, logStats, signAlgorithm, ip, fqdn, password, scripted} - certP :: Parser CertOptions - certP = do - signAlgorithm_ <- - optional $ - option - (maybeReader readMaybe) - ( long "sign-algorithm" - <> short 'a' - <> help "Set new signature algorithm used for TLS certificates: ED25519, ED448" - <> showDefault - <> metavar "ALG" - ) - commonName_ <- - optional $ - strOption - ( long "cn" - <> help - "Set new Common Name for TLS online certificate" - <> showDefault - <> metavar "FQDN" - ) - pure CertOptions {signAlgorithm_, commonName_} parseBasicAuth :: ReadM ServerPassword parseBasicAuth = eitherReader $ fmap ServerPassword . strDecode . B.pack + From 9254d8dac567035b2ddcf9908c7b75884cf8672b Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin Date: Fri, 16 Feb 2024 11:33:56 +0000 Subject: [PATCH 8/8] v5.5.3 --- CHANGELOG.md | 10 ++++++++++ package.yaml | 2 +- simplexmq.cabal | 2 +- src/Simplex/FileTransfer/Server/Main.hs | 2 +- src/Simplex/Messaging/Notifications/Server/Main.hs | 2 +- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5f37cf73..3311f2681 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +# 5.5.3 + +Agent: +- notification token API also returns active notifications server. +- support file descriptions with redirection and file URIs. + +Servers: +- CLI commands for online key and certificate rotation. +- Configure config and log paths via environment variables. + # 5.5.2 Extensible handshake for clients and SMP/NTF servers (ignore extra data). diff --git a/package.yaml b/package.yaml index 243545478..13109241f 100644 --- a/package.yaml +++ b/package.yaml @@ -1,5 +1,5 @@ name: simplexmq -version: 5.5.2.0 +version: 5.5.3.0 synopsis: SimpleXMQ message broker description: | This package includes <./docs/Simplex-Messaging-Server.html server>, diff --git a/simplexmq.cabal b/simplexmq.cabal index dd01a2849..1f7027b5f 100644 --- a/simplexmq.cabal +++ b/simplexmq.cabal @@ -5,7 +5,7 @@ cabal-version: 1.12 -- see: https://github.com/sol/hpack name: simplexmq -version: 5.5.2.0 +version: 5.5.3.0 synopsis: SimpleXMQ message broker description: This package includes <./docs/Simplex-Messaging-Server.html server>, <./docs/Simplex-Messaging-Client.html client> and diff --git a/src/Simplex/FileTransfer/Server/Main.hs b/src/Simplex/FileTransfer/Server/Main.hs index 9d55f250d..33975298e 100644 --- a/src/Simplex/FileTransfer/Server/Main.hs +++ b/src/Simplex/FileTransfer/Server/Main.hs @@ -33,7 +33,7 @@ import System.IO (BufferMode (..), hSetBuffering, stderr, stdout) import Text.Read (readMaybe) xftpServerVersion :: String -xftpServerVersion = "1.2.2.0" +xftpServerVersion = "1.2.3.0" xftpServerCLI :: FilePath -> FilePath -> IO () xftpServerCLI cfgPath logPath = do diff --git a/src/Simplex/Messaging/Notifications/Server/Main.hs b/src/Simplex/Messaging/Notifications/Server/Main.hs index f3d47e5c3..0462cf154 100644 --- a/src/Simplex/Messaging/Notifications/Server/Main.hs +++ b/src/Simplex/Messaging/Notifications/Server/Main.hs @@ -30,7 +30,7 @@ import System.IO (BufferMode (..), hSetBuffering, stderr, stdout) import Text.Read (readMaybe) ntfServerVersion :: String -ntfServerVersion = "1.7.2.0" +ntfServerVersion = "1.7.3.0" defaultSMPBatchDelay :: Int defaultSMPBatchDelay = 10000