Merge branch 'stable' into v5521

This commit is contained in:
Evgeny Poberezkin
2024-02-21 19:28:30 +00:00
31 changed files with 715 additions and 158 deletions
+76 -21
View File
@@ -7,6 +7,7 @@
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE RankNTypes #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TupleSections #-}
{-# LANGUAGE TypeApplications #-}
{-# OPTIONS_GHC -fno-warn-ambiguous-fields #-}
@@ -19,6 +20,7 @@ module Simplex.FileTransfer.Agent
xftpDeleteRcvFile',
-- Sending files
xftpSendFile',
xftpSendDescription',
deleteSndFileInternal,
deleteSndFileRemote,
)
@@ -44,6 +46,7 @@ import Simplex.FileTransfer.Client.Main
import Simplex.FileTransfer.Crypto
import Simplex.FileTransfer.Description
import Simplex.FileTransfer.Protocol (FileParty (..), SFileParty (..))
import qualified Simplex.FileTransfer.Protocol as XFTP
import Simplex.FileTransfer.Transport (XFTPRcvChunkSpec (..))
import Simplex.FileTransfer.Types
import Simplex.FileTransfer.Util (removePath, uniqueCombine)
@@ -57,6 +60,7 @@ import Simplex.Messaging.Crypto.File (CryptoFile (..), CryptoFileArgs)
import qualified Simplex.Messaging.Crypto.File as CF
import qualified Simplex.Messaging.Crypto.Lazy as LC
import Simplex.Messaging.Encoding
import Simplex.Messaging.Encoding.String (strDecode, strEncode)
import Simplex.Messaging.Protocol (EntityId, XFTPServer)
import Simplex.Messaging.Util (liftError, tshow, unlessM, whenM)
import System.FilePath (takeFileName, (</>))
@@ -97,7 +101,7 @@ closeXFTPAgent a = do
stopWorkers workers = atomically (swapTVar workers M.empty) >>= mapM_ (liftIO . cancelWorker)
xftpReceiveFile' :: AgentMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Maybe CryptoFileArgs -> m RcvFileId
xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks}) cfArgs = do
xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks, redirect}) cfArgs = do
g <- asks random
prefixPath <- getPrefixPath "rcv.xftp"
createDirectory prefixPath
@@ -107,14 +111,25 @@ xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks}) cfA
createDirectory =<< toFSFilePath relTmpPath
createEmptyFile =<< toFSFilePath relSavePath
let saveFile = CryptoFile relSavePath cfArgs
fId <- withStore c $ \db -> createRcvFile db g userId fd relPrefixPath relTmpPath saveFile
forM_ chunks downloadChunk
fId <- case redirect of
Nothing -> withStore c $ \db -> createRcvFile db g userId fd relPrefixPath relTmpPath saveFile
Just _ -> do
-- prepare description paths
let relTmpPathRedirect = relPrefixPath </> "xftp.redirect-encrypted"
relSavePathRedirect = relPrefixPath </> "xftp.redirect-decrypted"
createDirectory =<< toFSFilePath relTmpPathRedirect
createEmptyFile =<< toFSFilePath relSavePathRedirect
cfArgsRedirect <- atomically $ CF.randomArgs g
let saveFileRedirect = CryptoFile relSavePathRedirect $ Just cfArgsRedirect
-- create download tasks
withStore c $ \db -> createRcvFileRedirect db g userId fd relPrefixPath relTmpPathRedirect saveFileRedirect relTmpPath saveFile
forM_ chunks (downloadChunk c)
pure fId
where
downloadChunk :: AgentMonad m => FileChunk -> m ()
downloadChunk FileChunk {replicas = (FileChunkReplica {server} : _)} = do
void $ getXFTPRcvWorker True c (Just server)
downloadChunk _ = throwError $ INTERNAL "no replicas"
downloadChunk :: AgentMonad m => AgentClient -> FileChunk -> m ()
downloadChunk c FileChunk {replicas = (FileChunkReplica {server} : _)} = do
void $ getXFTPRcvWorker True c (Just server)
downloadChunk _ _ = throwError $ INTERNAL "no replicas"
getPrefixPath :: AgentMonad m => String -> m FilePath
getPrefixPath suffix = do
@@ -172,14 +187,17 @@ runXFTPRcvWorker c srv Worker {doWork} = do
relChunkPath = fileTmpPath </> takeFileName chunkPath
agentXFTPDownloadChunk c userId digest replica chunkSpec
atomically $ waitUntilForeground c
(complete, progress) <- withStore c $ \db -> runExceptT $ do
(entityId, complete, progress) <- withStore c $ \db -> runExceptT $ do
liftIO $ updateRcvFileChunkReceived db (rcvChunkReplicaId replica) rcvChunkId relChunkPath
RcvFile {size = FileSize total, chunks} <- ExceptT $ getRcvFile db rcvFileId
RcvFile {size = FileSize currentSize, chunks, redirect} <- ExceptT $ getRcvFile db rcvFileId
let rcvd = receivedSize chunks
complete = all chunkReceived chunks
(entityId, total) = case redirect of
Nothing -> (rcvFileEntityId, currentSize)
Just RcvFileRedirect {redirectFileInfo = RedirectFileInfo {size = FileSize finalSize}, redirectEntityId} -> (redirectEntityId, finalSize)
liftIO . when complete $ updateRcvFileStatus db rcvFileId RFSReceived
pure (complete, RFPROG rcvd total)
notify c rcvFileEntityId progress
pure (entityId, complete, RFPROG rcvd total)
notify c entityId progress
when complete . void $
getXFTPRcvWorker True c Nothing
where
@@ -223,7 +241,7 @@ runXFTPRcvLocalWorker c Worker {doWork} = do
\f@RcvFile {rcvFileId, rcvFileEntityId, tmpPath} ->
decryptFile f `catchAgentError` (rcvWorkerInternalError c rcvFileId rcvFileEntityId tmpPath . show)
decryptFile :: RcvFile -> m ()
decryptFile RcvFile {rcvFileId, rcvFileEntityId, key, nonce, tmpPath, saveFile, status, chunks} = do
decryptFile RcvFile {rcvFileId, rcvFileEntityId, size, digest, key, nonce, tmpPath, saveFile, status, chunks, redirect} = do
let CryptoFile savePath cfArgs = saveFile
fsSavePath <- toFSFilePath savePath
when (status == RFSDecrypting) $
@@ -231,12 +249,33 @@ runXFTPRcvLocalWorker c Worker {doWork} = do
withStore' c $ \db -> updateRcvFileStatus db rcvFileId RFSDecrypting
chunkPaths <- getChunkPaths chunks
encSize <- liftIO $ foldM (\s path -> (s +) . fromIntegral <$> getFileSize path) 0 chunkPaths
when (FileSize encSize /= size) $ throwError $ XFTP XFTP.SIZE
encDigest <- liftIO $ LC.sha512Hash <$> readChunks chunkPaths
when (FileDigest encDigest /= digest) $ throwError $ XFTP XFTP.DIGEST
let destFile = CryptoFile fsSavePath cfArgs
void $ liftError (INTERNAL . show) $ decryptChunks encSize chunkPaths key nonce $ \_ -> pure destFile
notify c rcvFileEntityId $ RFDONE fsSavePath
forM_ tmpPath (removePath <=< toFSFilePath)
atomically $ waitUntilForeground c
withStore' c (`updateRcvFileComplete` rcvFileId)
case redirect of
Nothing -> do
notify c rcvFileEntityId $ RFDONE fsSavePath
forM_ tmpPath (removePath <=< toFSFilePath)
atomically $ waitUntilForeground c
withStore' c (`updateRcvFileComplete` rcvFileId)
Just RcvFileRedirect {redirectFileInfo, redirectDbId} -> do
let RedirectFileInfo {size = redirectSize, digest = redirectDigest} = redirectFileInfo
forM_ tmpPath (removePath <=< toFSFilePath)
atomically $ waitUntilForeground c
withStore' c (`updateRcvFileComplete` rcvFileId)
-- proceed with redirect
yaml <- liftError (INTERNAL . show) (CF.readFile $ CryptoFile fsSavePath cfArgs) `finally` (toFSFilePath fsSavePath >>= removePath)
next@FileDescription {chunks = nextChunks} <- case strDecode (LB.toStrict yaml) of
Left _ -> throwError . XFTP $ XFTP.REDIRECT "decode error"
Right (ValidFileDescription fd@FileDescription {size = dstSize, digest = dstDigest})
| dstSize /= redirectSize -> throwError . XFTP $ XFTP.REDIRECT "size mismatch"
| dstDigest /= redirectDigest -> throwError . XFTP $ XFTP.REDIRECT "digest mismatch"
| otherwise -> pure fd
-- register and download chunks from the actual file
withStore c $ \db -> updateRcvFileRedirect db redirectDbId next
forM_ nextChunks (downloadChunk c)
where
getChunkPaths :: [RcvFileChunk] -> m [FilePath]
getChunkPaths [] = pure []
@@ -268,7 +307,23 @@ xftpSendFile' c userId file numRecipients = do
key <- atomically $ C.randomSbKey g
nonce <- atomically $ C.randomCbNonce g
-- saving absolute filePath will not allow to restore file encryption after app update, but it's a short window
fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce
fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce Nothing
void $ getXFTPSndWorker True c Nothing
pure fId
xftpSendDescription' :: forall m. AgentMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> m SndFileId
xftpSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) numRecipients = do
g <- asks random
prefixPath <- getPrefixPath "snd.xftp"
createDirectory prefixPath
let relPrefixPath = takeFileName prefixPath
let directYaml = prefixPath </> "direct.yaml"
cfArgs <- atomically $ CF.randomArgs g
let file = CryptoFile directYaml (Just cfArgs)
liftError (INTERNAL . show) $ CF.writeFile file (LB.fromStrict $ strEncode fdDirect)
key <- atomically $ C.randomSbKey g
nonce <- atomically $ C.randomCbNonce g
fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce $ Just RedirectFileInfo {size, digest}
void $ getXFTPSndWorker True c Nothing
pure fId
@@ -423,15 +478,15 @@ runXFTPSndWorker c srv Worker {doWork} = do
sndFileToDescrs :: SndFile -> m (ValidFileDescription 'FSender, [ValidFileDescription 'FRecipient])
sndFileToDescrs SndFile {digest = Nothing} = throwError $ INTERNAL "snd file has no digest"
sndFileToDescrs SndFile {chunks = []} = throwError $ INTERNAL "snd file has no chunks"
sndFileToDescrs SndFile {digest = Just digest, key, nonce, chunks = chunks@(fstChunk : _)} = do
sndFileToDescrs SndFile {digest = Just digest, key, nonce, chunks = chunks@(fstChunk : _), redirect} = do
let chunkSize = FileSize $ sndChunkSize fstChunk
size = FileSize $ sum $ map (fromIntegral . sndChunkSize) chunks
-- snd description
sndDescrChunks <- mapM toSndDescrChunk chunks
let fdSnd = FileDescription {party = SFSender, size, digest, key, nonce, chunkSize, chunks = sndDescrChunks}
let fdSnd = FileDescription {party = SFSender, size, digest, key, nonce, chunkSize, chunks = sndDescrChunks, redirect = Nothing}
validFdSnd <- either (throwError . INTERNAL) pure $ validateFileDescription fdSnd
-- rcv descriptions
let fdRcv = FileDescription {party = SFRecipient, size, digest, key, nonce, chunkSize, chunks = []}
let fdRcv = FileDescription {party = SFRecipient, size, digest, key, nonce, chunkSize, chunks = [], redirect}
fdRcvs = createRcvFileDescriptions fdRcv chunks
validFdRcvs <- either (throwError . INTERNAL) pure $ mapM validateFileDescription fdRcvs
pure (validFdSnd, validFdRcvs)
+5 -5
View File
@@ -15,6 +15,7 @@ module Simplex.FileTransfer.Client.Main
CLIError (..),
xftpClientCLI,
cliSendFile,
cliSendFileOpts,
prepareChunkSizes,
prepareChunkSpecs,
maxFileSize,
@@ -297,8 +298,8 @@ cliSendFileOpts SendOptions {filePath, outputDir, numRecipients, xftpServers, re
withExceptT (CLIError . show) $ encryptFile srcFile fileHdr key nonce fileSize' encSize encPath
digest <- liftIO $ LC.sha512Hash <$> LB.readFile encPath
let chunkSpecs = prepareChunkSpecs encPath chunkSizes
fdRcv = FileDescription {party = SFRecipient, size = FileSize encSize, digest = FileDigest digest, key, nonce, chunkSize = FileSize defChunkSize, chunks = []}
fdSnd = FileDescription {party = SFSender, size = FileSize encSize, digest = FileDigest digest, key, nonce, chunkSize = FileSize defChunkSize, chunks = []}
fdRcv = FileDescription {party = SFRecipient, size = FileSize encSize, digest = FileDigest digest, key, nonce, chunkSize = FileSize defChunkSize, chunks = [], redirect = Nothing}
fdSnd = FileDescription {party = SFSender, size = FileSize encSize, digest = FileDigest digest, key, nonce, chunkSize = FileSize defChunkSize, chunks = [], redirect = Nothing}
logInfo $ "encrypted file to " <> tshow encPath
pure (encPath, fdRcv, fdSnd, chunkSpecs, encSize)
uploadFile :: TVar ChaChaDRG -> [XFTPChunkSpec] -> TVar [Int64] -> Int64 -> ExceptT CLIError IO [SentFileChunk]
@@ -526,9 +527,8 @@ prepareChunkSizes size' = prepareSizes size'
where
(smallSize, bigSize)
| size' > size34 chunkSize3 = (chunkSize2, chunkSize3)
| otherwise = (chunkSize1, chunkSize2)
-- | size' > size34 chunkSize2 = (chunkSize1, chunkSize2)
-- | otherwise = (chunkSize0, chunkSize1)
| size' > size34 chunkSize2 = (chunkSize1, chunkSize2)
| otherwise = (chunkSize0, chunkSize1)
size34 sz = (fromIntegral sz * 3) `div` 4
prepareSizes 0 = []
prepareSizes size
+63 -7
View File
@@ -12,6 +12,7 @@
module Simplex.FileTransfer.Description
( FileDescription (..),
RedirectFileInfo (..),
AFileDescription (..),
ValidFileDescription, -- constructor is not exported, use pattern
pattern ValidFileDescription,
@@ -30,12 +31,17 @@ module Simplex.FileTransfer.Description
kb,
mb,
gb,
FileDescriptionURI (..),
FileClientData,
fileDescriptionURI,
qrSizeLimit,
)
where
import Control.Applicative (optional)
import Control.Monad ((<=<))
import Data.Aeson (FromJSON, ToJSON)
import qualified Data.Aeson as J
import qualified Data.Aeson.TH as J
import Data.Attoparsec.ByteString.Char8 (Parser)
import qualified Data.Attoparsec.ByteString.Char8 as A
@@ -50,17 +56,21 @@ import Data.Map (Map)
import qualified Data.Map as M
import Data.Maybe (fromMaybe)
import Data.String
import Data.Text (Text)
import Data.Text.Encoding (encodeUtf8)
import Data.Word (Word32)
import qualified Data.Yaml as Y
import Database.SQLite.Simple.FromField (FromField (..))
import Database.SQLite.Simple.ToField (ToField (..))
import Simplex.FileTransfer.Chunks
import Simplex.FileTransfer.Protocol
import Simplex.Messaging.Agent.QueryString
import qualified Simplex.Messaging.Crypto as C
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Parsers (defaultJSON, parseAll)
import Simplex.Messaging.Protocol (XFTPServer)
import Simplex.Messaging.Util (bshow, (<$?>))
import Simplex.Messaging.ServiceScheme (ServiceScheme (..))
import Simplex.Messaging.Util (bshow, safeDecodeUtf8, (<$?>))
data FileDescription (p :: FileParty) = FileDescription
{ party :: SFileParty p,
@@ -69,7 +79,14 @@ data FileDescription (p :: FileParty) = FileDescription
key :: C.SbKey,
nonce :: C.CbNonce,
chunkSize :: FileSize Word32,
chunks :: [FileChunk]
chunks :: [FileChunk],
redirect :: Maybe RedirectFileInfo
}
deriving (Eq, Show)
data RedirectFileInfo = RedirectFileInfo
{ size :: FileSize Int64,
digest :: FileDigest
}
deriving (Eq, Show)
@@ -147,7 +164,8 @@ data YAMLFileDescription = YAMLFileDescription
key :: C.SbKey,
nonce :: C.CbNonce,
chunkSize :: String,
replicas :: [YAMLServerReplicas]
replicas :: [YAMLServerReplicas],
redirect :: Maybe RedirectFileInfo
}
deriving (Eq, Show)
@@ -170,8 +188,16 @@ data FileServerReplica = FileServerReplica
newtype FileSize a = FileSize {unFileSize :: a}
deriving (Eq, Show)
instance FromJSON a => FromJSON (FileSize a) where
parseJSON v = FileSize <$> Y.parseJSON v
instance ToJSON a => ToJSON (FileSize a) where
toJSON = Y.toJSON . unFileSize
$(J.deriveJSON defaultJSON ''YAMLServerReplicas)
$(J.deriveJSON defaultJSON ''RedirectFileInfo)
$(J.deriveJSON defaultJSON ''YAMLFileDescription)
instance FilePartyI p => StrEncoding (ValidFileDescription p) where
@@ -204,7 +230,7 @@ validateFileDescription fd@FileDescription {size, chunks}
chunksSize = fromIntegral . foldl' (\s FileChunk {chunkSize} -> s + unFileSize chunkSize) 0
encodeFileDescription :: FileDescription p -> YAMLFileDescription
encodeFileDescription FileDescription {party, size, digest, key, nonce, chunkSize, chunks} =
encodeFileDescription FileDescription {party, size, digest, key, nonce, chunkSize, chunks, redirect} =
YAMLFileDescription
{ party = toFileParty party,
size = B.unpack $ strEncode size,
@@ -212,9 +238,39 @@ encodeFileDescription FileDescription {party, size, digest, key, nonce, chunkSiz
key,
nonce,
chunkSize = B.unpack $ strEncode chunkSize,
replicas = encodeFileReplicas chunkSize chunks
replicas = encodeFileReplicas chunkSize chunks,
redirect
}
data FileDescriptionURI = FileDescriptionURI
{ scheme :: ServiceScheme,
description :: ValidFileDescription 'FRecipient,
clientData :: Maybe FileClientData -- JSON-encoded extensions to pass in a link
}
deriving (Eq, Show)
type FileClientData = Text
fileDescriptionURI :: ValidFileDescription 'FRecipient -> FileDescriptionURI
fileDescriptionURI vfd = FileDescriptionURI SSSimplex vfd mempty
instance StrEncoding FileDescriptionURI where
strEncode FileDescriptionURI {scheme, description, clientData} = mconcat [strEncode scheme, "/file", "#/?", queryStr]
where
queryStr = strEncode $ QSP QEscape qs
qs = ("desc", strEncode description) : maybe [] (\cd -> [("data", encodeUtf8 cd)]) clientData
strP = do
scheme <- strP
_ <- "/file" <* optional (A.char '/') <* "#/?"
query <- strP
description <- queryParam "desc" query
let clientData = safeDecodeUtf8 <$> queryParamStr "data" query
pure FileDescriptionURI {scheme, description, clientData}
-- | URL length in QR code before jumping up to a next size.
qrSizeLimit :: Int
qrSizeLimit = 1002 -- ~2 chunks in URLencoded YAML with some spare size for server hosts
instance (Integral a, Show a) => StrEncoding (FileSize a) where
strEncode (FileSize b)
| b' /= 0 = bshow b
@@ -285,13 +341,13 @@ unfoldChunksToReplicas defChunkSize = concatMap chunkReplicas
in FileServerReplica {chunkNo, server, replicaId, replicaKey, digest = digest', chunkSize = chunkSize'}
decodeFileDescription :: YAMLFileDescription -> Either String AFileDescription
decodeFileDescription YAMLFileDescription {party, size, digest, key, nonce, chunkSize, replicas} = do
decodeFileDescription YAMLFileDescription {party, size, digest, key, nonce, chunkSize, replicas, redirect} = do
size' <- strDecode $ B.pack size
chunkSize' <- strDecode $ B.pack chunkSize
replicas' <- decodeFileParts replicas
chunks <- foldReplicasToChunks chunkSize' replicas'
pure $ case aFileParty party of
AFP party' -> AFD FileDescription {party = party', size = size', digest, key, nonce, chunkSize = chunkSize', chunks}
AFP party' -> AFD FileDescription {party = party', size = size', digest, key, nonce, chunkSize = chunkSize', chunks, redirect}
where
decodeFileParts = fmap concat . mapM decodeYAMLServerReplicas
+9 -1
View File
@@ -338,6 +338,8 @@ data XFTPErrorType
HAS_FILE
| -- | file IO error
FILE_IO
| -- | bad redirect data
REDIRECT {redirectError :: String}
| -- | internal server error
INTERNAL
| -- | used internally, never returned by the server (to be removed)
@@ -347,8 +349,12 @@ data XFTPErrorType
instance StrEncoding XFTPErrorType where
strEncode = \case
CMD e -> "CMD " <> bshow e
REDIRECT e -> "REDIRECT " <> bshow e
e -> bshow e
strP = "CMD " *> (CMD <$> parseRead1) <|> parseRead1
strP =
"CMD " *> (CMD <$> parseRead1)
<|> "REDIRECT " *> (REDIRECT <$> parseRead A.takeByteString)
<|> parseRead1
instance Encoding XFTPErrorType where
smpEncode = \case
@@ -363,6 +369,7 @@ instance Encoding XFTPErrorType where
NO_FILE -> "NO_FILE"
HAS_FILE -> "HAS_FILE"
FILE_IO -> "FILE_IO"
REDIRECT err -> "REDIRECT " <> smpEncode err
INTERNAL -> "INTERNAL"
DUPLICATE_ -> "DUPLICATE_"
@@ -379,6 +386,7 @@ instance Encoding XFTPErrorType where
"NO_FILE" -> pure NO_FILE
"HAS_FILE" -> pure HAS_FILE
"FILE_IO" -> pure FILE_IO
"REDIRECT" -> REDIRECT <$> _smpP
"INTERNAL" -> pure INTERNAL
"DUPLICATE_" -> pure DUPLICATE_
_ -> fail "bad error type"
+7 -1
View File
@@ -33,7 +33,7 @@ import System.IO (BufferMode (..), hSetBuffering, stderr, stdout)
import Text.Read (readMaybe)
xftpServerVersion :: String
xftpServerVersion = "1.2.2.0"
xftpServerVersion = "1.2.3.0"
xftpServerCLI :: FilePath -> FilePath -> IO ()
xftpServerCLI cfgPath logPath = do
@@ -42,6 +42,10 @@ xftpServerCLI cfgPath logPath = do
doesFileExist iniFile >>= \case
True -> exitError $ "Error: server is already initialized (" <> iniFile <> " exists).\nRun `" <> executableName <> " start`."
_ -> initializeServer opts
OnlineCert certOpts ->
doesFileExist iniFile >>= \case
True -> genOnline cfgPath certOpts
_ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`."
Start ->
doesFileExist iniFile >>= \case
True -> readIniFile iniFile >>= either exitError runServer
@@ -179,6 +183,7 @@ xftpServerCLI cfgPath logPath = do
data CliCommand
= Init InitOptions
| OnlineCert CertOptions
| Start
| Delete
@@ -196,6 +201,7 @@ cliCommandP :: FilePath -> FilePath -> FilePath -> Parser CliCommand
cliCommandP cfgPath logPath iniFile =
hsubparser
( command "init" (info (Init <$> initP) (progDesc $ "Initialize server - creates " <> cfgPath <> " and " <> logPath <> " directories and configuration files"))
<> command "cert" (info (OnlineCert <$> certOptionsP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")"))
<> command "start" (info (pure Start) (progDesc $ "Start server (configuration: " <> iniFile <> ")"))
<> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files"))
)
+10 -1
View File
@@ -47,6 +47,7 @@ data RcvFile = RcvFile
key :: C.SbKey,
nonce :: C.CbNonce,
chunkSize :: FileSize Word32,
redirect :: Maybe RcvFileRedirect,
chunks :: [RcvFileChunk],
prefixPath :: FilePath,
tmpPath :: Maybe FilePath,
@@ -108,6 +109,13 @@ data RcvFileChunkReplica = RcvFileChunkReplica
}
deriving (Eq, Show)
data RcvFileRedirect = RcvFileRedirect
{ redirectDbId :: DBRcvFileId,
redirectEntityId :: RcvFileId,
redirectFileInfo :: RedirectFileInfo
}
deriving (Eq, Show)
-- Sending files
type DBSndFileId = Int64
@@ -124,7 +132,8 @@ data SndFile = SndFile
srcFile :: CryptoFile,
prefixPath :: Maybe FilePath,
status :: SndFileStatus,
deleted :: Bool
deleted :: Bool,
redirect :: Maybe RedirectFileInfo
}
deriving (Eq, Show)
+11 -5
View File
@@ -93,6 +93,7 @@ module Simplex.Messaging.Agent
xftpReceiveFile,
xftpDeleteRcvFile,
xftpSendFile,
xftpSendDescription,
xftpDeleteSndFileInternal,
xftpDeleteSndFileRemote,
rcNewHostPairing,
@@ -137,7 +138,7 @@ import qualified Data.Text as T
import Data.Time.Clock
import Data.Time.Clock.System (systemToUTCTime)
import Data.Word (Word16)
import Simplex.FileTransfer.Agent (closeXFTPAgent, deleteSndFileInternal, deleteSndFileRemote, startXFTPWorkers, toFSFilePath, xftpDeleteRcvFile', xftpReceiveFile', xftpSendFile')
import Simplex.FileTransfer.Agent (closeXFTPAgent, deleteSndFileInternal, deleteSndFileRemote, startXFTPWorkers, toFSFilePath, xftpDeleteRcvFile', xftpReceiveFile', xftpSendDescription', xftpSendFile')
import Simplex.FileTransfer.Description (ValidFileDescription)
import Simplex.FileTransfer.Protocol (FileParty (..))
import Simplex.FileTransfer.Util (removePath)
@@ -163,6 +164,7 @@ import Simplex.Messaging.Notifications.Types
import Simplex.Messaging.Parsers (parse)
import Simplex.Messaging.Protocol (BrokerMsg, EntityId, ErrorType (AUTH), MsgBody, MsgFlags (..), NtfServer, ProtoServerWithAuth, ProtocolTypeI (..), SMPMsgMeta, SProtocolType (..), SndPublicVerifyKey, SubscriptionMode (..), UserProtocol, XFTPServerWithAuth)
import qualified Simplex.Messaging.Protocol as SMP
import Simplex.Messaging.ServiceScheme (ServiceScheme (..))
import qualified Simplex.Messaging.TMap as TM
import Simplex.Messaging.Util
import Simplex.Messaging.Version
@@ -375,7 +377,7 @@ checkNtfToken c = withAgentEnv c . checkNtfToken' c
deleteNtfToken :: AgentErrorMonad m => AgentClient -> DeviceToken -> m ()
deleteNtfToken c = withAgentEnv c . deleteNtfToken' c
getNtfToken :: AgentErrorMonad m => AgentClient -> m (DeviceToken, NtfTknStatus, NotificationsMode)
getNtfToken :: AgentErrorMonad m => AgentClient -> m (DeviceToken, NtfTknStatus, NotificationsMode, NtfServer)
getNtfToken c = withAgentEnv c $ getNtfToken' c
getNtfTokenData :: AgentErrorMonad m => AgentClient -> m NtfToken
@@ -400,6 +402,10 @@ xftpDeleteRcvFile c = withAgentEnv c . xftpDeleteRcvFile' c
xftpSendFile :: AgentErrorMonad m => AgentClient -> UserId -> CryptoFile -> Int -> m SndFileId
xftpSendFile c = withAgentEnv c .:. xftpSendFile' c
-- | Send XFTP file
xftpSendDescription :: AgentErrorMonad m => AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> m SndFileId
xftpSendDescription c = withAgentEnv c .:. xftpSendDescription' c
-- | Delete XFTP snd file internally (deletes work files from file system and db records)
xftpDeleteSndFileInternal :: AgentErrorMonad m => AgentClient -> SndFileId -> m ()
xftpDeleteSndFileInternal c = withAgentEnv c . deleteSndFileInternal c
@@ -643,7 +649,7 @@ newRcvConnSrv c userId connId enableNtfs cMode clientData subMode srv = do
when enableNtfs $ do
ns <- asks ntfSupervisor
atomically $ sendNtfSubCommand ns (connId, NSCCreate)
let crData = ConnReqUriData CRSSimplex smpAgentVRange [qUri] clientData
let crData = ConnReqUriData SSSimplex smpAgentVRange [qUri] clientData
case cMode of
SCMContact -> pure (connId, CRContactUri crData)
SCMInvitation -> do
@@ -1685,10 +1691,10 @@ deleteNtfToken' c deviceToken =
deleteNtfSubs c NSCSmpDelete
_ -> throwError $ CMD PROHIBITED
getNtfToken' :: AgentMonad m => AgentClient -> m (DeviceToken, NtfTknStatus, NotificationsMode)
getNtfToken' :: AgentMonad m => AgentClient -> m (DeviceToken, NtfTknStatus, NotificationsMode, NtfServer)
getNtfToken' c =
withStore' c getSavedNtfToken >>= \case
Just NtfToken {deviceToken, ntfTknStatus, ntfMode} -> pure (deviceToken, ntfTknStatus, ntfMode)
Just NtfToken {deviceToken, ntfTknStatus, ntfMode, ntfServer} -> pure (deviceToken, ntfTknStatus, ntfMode, ntfServer)
_ -> throwError $ CMD PROHIBITED
getNtfTokenData' :: AgentMonad m => AgentClient -> m NtfToken
+5 -19
View File
@@ -97,7 +97,7 @@ module Simplex.Messaging.Agent.Protocol
AConnectionRequestUri (..),
ConnReqUriData (..),
CRClientData,
ConnReqScheme (..),
ServiceScheme,
simplexChat,
AgentErrorType (..),
CommandErrorType (..),
@@ -197,7 +197,6 @@ import Simplex.Messaging.Protocol
SMPServer,
SMPServerWithAuth,
SndPublicVerifyKey,
SrvLoc (..),
SubscriptionMode,
legacyEncodeServer,
legacyServerP,
@@ -208,6 +207,7 @@ import Simplex.Messaging.Protocol
pattern SMPServer,
)
import qualified Simplex.Messaging.Protocol as SMP
import Simplex.Messaging.ServiceScheme
import Simplex.Messaging.Transport (Transport (..), TransportError, serializeTransportError, transportErrorP)
import Simplex.Messaging.Transport.Client (TransportHost, TransportHosts_ (..))
import Simplex.Messaging.Util
@@ -1123,13 +1123,13 @@ instance forall m. ConnectionModeI m => StrEncoding (ConnectionRequestUri m) whe
instance StrEncoding AConnectionRequestUri where
strEncode (ACR _ cr) = strEncode cr
strP = do
_crScheme :: ConnReqScheme <- strP
_crScheme :: ServiceScheme <- strP
crMode <- A.char '/' *> crModeP <* optional (A.char '/') <* "#/?"
query <- strP
crAgentVRange <- queryParam "v" query
crSmpQueues <- queryParam "smp" query
let crClientData = safeDecodeUtf8 <$> queryParamStr "data" query
let crData = ConnReqUriData {crScheme = CRSSimplex, crAgentVRange, crSmpQueues, crClientData}
let crData = ConnReqUriData {crScheme = SSSimplex, crAgentVRange, crSmpQueues, crClientData}
case crMode of
CMInvitation -> do
crE2eParams <- queryParam "e2e" query
@@ -1328,7 +1328,7 @@ instance Eq AConnectionRequestUri where
deriving instance Show AConnectionRequestUri
data ConnReqUriData = ConnReqUriData
{ crScheme :: ConnReqScheme,
{ crScheme :: ServiceScheme,
crAgentVRange :: VersionRange,
crSmpQueues :: NonEmpty SMPQueueUri,
crClientData :: Maybe CRClientData
@@ -1337,20 +1337,6 @@ data ConnReqUriData = ConnReqUriData
type CRClientData = Text
data ConnReqScheme = CRSSimplex | CRSAppServer SrvLoc
deriving (Eq, Show)
instance StrEncoding ConnReqScheme where
strEncode = \case
CRSSimplex -> "simplex:"
CRSAppServer srv -> "https://" <> strEncode srv
strP =
"simplex:" $> CRSSimplex
<|> "https://" *> (CRSAppServer <$> strP)
simplexChat :: ConnReqScheme
simplexChat = CRSAppServer $ SrvLoc "simplex.chat" ""
-- | SMP queue status.
data QueueStatus
= -- | queue is created
+88 -39
View File
@@ -164,6 +164,7 @@ module Simplex.Messaging.Agent.Store.SQLite
-- Rcv files
createRcvFile,
createRcvFileRedirect,
getRcvFile,
getRcvFileByEntityId,
updateRcvChunkReplicaDelay,
@@ -171,6 +172,7 @@ module Simplex.Messaging.Agent.Store.SQLite
updateRcvFileStatus,
updateRcvFileError,
updateRcvFileComplete,
updateRcvFileRedirect,
updateRcvFileNoTmpPath,
updateRcvFileDeleted,
deleteRcvFile',
@@ -231,6 +233,7 @@ import Data.ByteArray (ScrubbedBytes)
import qualified Data.ByteArray as BA
import Data.ByteString (ByteString)
import qualified Data.ByteString.Base64.URL as U
import qualified Data.ByteString.Char8 as B
import Data.Char (toLower)
import Data.Functor (($>))
import Data.IORef
@@ -255,7 +258,7 @@ import qualified Database.SQLite3 as SQLite3
import Network.Socket (ServiceName)
import Simplex.FileTransfer.Client (XFTPChunkSpec (..))
import Simplex.FileTransfer.Description
import Simplex.FileTransfer.Protocol (FileParty (..))
import Simplex.FileTransfer.Protocol (FileParty (..), SFileParty (..))
import Simplex.FileTransfer.Types
import Simplex.Messaging.Agent.Protocol
import Simplex.Messaging.Agent.RetryInterval (RI2State (..))
@@ -2263,38 +2266,66 @@ getXFTPServerId_ db ProtocolServer {host, port, keyHash} = do
DB.query db "SELECT xftp_server_id FROM xftp_servers WHERE xftp_host = ? AND xftp_port = ? AND xftp_key_hash = ?" (host, port, keyHash)
createRcvFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription 'FRecipient -> FilePath -> FilePath -> CryptoFile -> IO (Either StoreError RcvFileId)
createRcvFile db gVar userId fd@FileDescription {chunks} prefixPath tmpPath (CryptoFile savePath cfArgs) = runExceptT $ do
(rcvFileEntityId, rcvFileId) <- ExceptT $ insertRcvFile fd
createRcvFile db gVar userId fd@FileDescription {chunks} prefixPath tmpPath file = runExceptT $ do
(rcvFileEntityId, rcvFileId) <- ExceptT $ insertRcvFile db gVar userId fd prefixPath tmpPath file Nothing Nothing
liftIO $
forM_ chunks $ \fc@FileChunk {replicas} -> do
chunkId <- insertChunk fc rcvFileId
forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertReplica rno replica chunkId
chunkId <- insertRcvFileChunk db fc rcvFileId
forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertRcvFileChunkReplica db rno replica chunkId
pure rcvFileEntityId
createRcvFileRedirect :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription FRecipient -> FilePath -> FilePath -> CryptoFile -> FilePath -> CryptoFile -> IO (Either StoreError RcvFileId)
createRcvFileRedirect _ _ _ FileDescription {redirect = Nothing} _ _ _ _ _ = pure $ Left $ SEInternal "createRcvFileRedirect called without redirect"
createRcvFileRedirect db gVar userId redirectFd@FileDescription {chunks = redirectChunks, redirect = Just RedirectFileInfo {size, digest}} prefixPath redirectPath redirectFile dstPath dstFile = runExceptT $ do
(dstEntityId, dstId) <- ExceptT $ insertRcvFile db gVar userId dummyDst prefixPath dstPath dstFile Nothing Nothing
(_, redirectId) <- ExceptT $ insertRcvFile db gVar userId redirectFd prefixPath redirectPath redirectFile (Just dstId) (Just dstEntityId)
liftIO $
forM_ redirectChunks $ \fc@FileChunk {replicas} -> do
chunkId <- insertRcvFileChunk db fc redirectId
forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertRcvFileChunkReplica db rno replica chunkId
pure dstEntityId
where
insertRcvFile :: FileDescription 'FRecipient -> IO (Either StoreError (RcvFileId, DBRcvFileId))
insertRcvFile FileDescription {size, digest, key, nonce, chunkSize} = runExceptT $ do
rcvFileEntityId <- ExceptT $
createWithRandomId gVar $ \rcvFileEntityId ->
DB.execute
db
"INSERT INTO rcv_files (rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"
((rcvFileEntityId, userId, size, digest, key, nonce, chunkSize) :. (prefixPath, tmpPath, savePath, fileKey <$> cfArgs, fileNonce <$> cfArgs, RFSReceiving))
rcvFileId <- liftIO $ insertedRowId db
pure (rcvFileEntityId, rcvFileId)
insertChunk :: FileChunk -> DBRcvFileId -> IO Int64
insertChunk FileChunk {chunkNo, chunkSize, digest} rcvFileId = do
dummyDst = FileDescription
{ party = SFRecipient,
size,
digest,
redirect = Nothing,
-- updated later with updateRcvFileRedirect
key = C.unsafeSbKey $ B.replicate 32 '#',
nonce = C.cbNonce "",
chunkSize = FileSize 0,
chunks = []
}
insertRcvFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription 'FRecipient -> FilePath -> FilePath -> CryptoFile -> Maybe DBRcvFileId -> Maybe RcvFileId -> IO (Either StoreError (RcvFileId, DBRcvFileId))
insertRcvFile db gVar userId FileDescription {size, digest, key, nonce, chunkSize, redirect} prefixPath tmpPath (CryptoFile savePath cfArgs) redirectId_ redirectEntityId_ = runExceptT $ do
let (redirectDigest_, redirectSize_) = case redirect of
Just RedirectFileInfo {digest = d, size = s} -> (Just d, Just s)
Nothing -> (Nothing, Nothing)
rcvFileEntityId <- ExceptT $
createWithRandomId gVar $ \rcvFileEntityId ->
DB.execute
db
"INSERT INTO rcv_file_chunks (rcv_file_id, chunk_no, chunk_size, digest) VALUES (?,?,?,?)"
(rcvFileId, chunkNo, chunkSize, digest)
insertedRowId db
insertReplica :: Int -> FileChunkReplica -> Int64 -> IO ()
insertReplica replicaNo FileChunkReplica {server, replicaId, replicaKey} chunkId = do
srvId <- createXFTPServer_ db server
DB.execute
db
"INSERT INTO rcv_file_chunk_replicas (replica_number, rcv_file_chunk_id, xftp_server_id, replica_id, replica_key) VALUES (?,?,?,?,?)"
(replicaNo, chunkId, srvId, replicaId, replicaKey)
"INSERT INTO rcv_files (rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, redirect_id, redirect_entity_id, redirect_digest, redirect_size) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
((rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, fileKey <$> cfArgs, fileNonce <$> cfArgs, RFSReceiving, redirectId_, redirectEntityId_, redirectDigest_, redirectSize_))
rcvFileId <- liftIO $ insertedRowId db
pure (rcvFileEntityId, rcvFileId)
insertRcvFileChunk :: DB.Connection -> FileChunk -> DBRcvFileId -> IO Int64
insertRcvFileChunk db FileChunk {chunkNo, chunkSize, digest} rcvFileId = do
DB.execute
db
"INSERT INTO rcv_file_chunks (rcv_file_id, chunk_no, chunk_size, digest) VALUES (?,?,?,?)"
(rcvFileId, chunkNo, chunkSize, digest)
insertedRowId db
insertRcvFileChunkReplica :: DB.Connection -> Int -> FileChunkReplica -> Int64 -> IO ()
insertRcvFileChunkReplica db replicaNo FileChunkReplica {server, replicaId, replicaKey} chunkId = do
srvId <- createXFTPServer_ db server
DB.execute
db
"INSERT INTO rcv_file_chunk_replicas (replica_number, rcv_file_chunk_id, xftp_server_id, replica_id, replica_key) VALUES (?,?,?,?,?)"
(replicaNo, chunkId, srvId, replicaId, replicaKey)
getRcvFileByEntityId :: DB.Connection -> RcvFileId -> IO (Either StoreError RcvFile)
getRcvFileByEntityId db rcvFileEntityId = runExceptT $ do
@@ -2318,17 +2349,21 @@ getRcvFile db rcvFileId = runExceptT $ do
DB.query
db
[sql|
SELECT rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, deleted
SELECT rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, deleted, redirect_id, redirect_entity_id, redirect_size, redirect_digest
FROM rcv_files
WHERE rcv_file_id = ?
|]
(Only rcvFileId)
where
toFile :: (RcvFileId, UserId, FileSize Int64, FileDigest, C.SbKey, C.CbNonce, FileSize Word32, FilePath, Maybe FilePath) :. (FilePath, Maybe C.SbKey, Maybe C.CbNonce, RcvFileStatus, Bool) -> RcvFile
toFile ((rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, saveKey_, saveNonce_, status, deleted)) =
toFile :: (RcvFileId, UserId, FileSize Int64, FileDigest, C.SbKey, C.CbNonce, FileSize Word32, FilePath, Maybe FilePath) :. (FilePath, Maybe C.SbKey, Maybe C.CbNonce, RcvFileStatus, Bool, Maybe DBRcvFileId, Maybe RcvFileId, Maybe (FileSize Int64), Maybe FileDigest) -> RcvFile
toFile ((rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, saveKey_, saveNonce_, status, deleted, redirectDbId, redirectEntityId, redirectSize_, redirectDigest_)) =
let cfArgs = CFArgs <$> saveKey_ <*> saveNonce_
saveFile = CryptoFile savePath cfArgs
in RcvFile {rcvFileId, rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath, saveFile, status, deleted, chunks = []}
redirect = RcvFileRedirect
<$> redirectDbId
<*> redirectEntityId
<*> (RedirectFileInfo <$> redirectSize_ <*> redirectDigest_)
in RcvFile {rcvFileId, rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, redirect, prefixPath, tmpPath, saveFile, status, deleted, chunks = []}
getChunks :: RcvFileId -> UserId -> FilePath -> IO [RcvFileChunk]
getChunks rcvFileEntityId userId fileTmpPath = do
chunks <-
@@ -2394,6 +2429,14 @@ updateRcvFileComplete db rcvFileId = do
updatedAt <- getCurrentTime
DB.execute db "UPDATE rcv_files SET tmp_path = NULL, status = ?, updated_at = ? WHERE rcv_file_id = ?" (RFSComplete, updatedAt, rcvFileId)
updateRcvFileRedirect :: DB.Connection -> DBRcvFileId -> FileDescription 'FRecipient -> IO (Either StoreError ())
updateRcvFileRedirect db rcvFileId FileDescription {key, nonce, chunkSize, chunks} = runExceptT $ do
updatedAt <- liftIO getCurrentTime
liftIO $ DB.execute db "UPDATE rcv_files SET key = ?, nonce = ?, chunk_size = ?, updated_at = ? WHERE rcv_file_id = ?" (key, nonce, chunkSize, updatedAt, rcvFileId)
liftIO $ forM_ chunks $ \fc@FileChunk {replicas} -> do
chunkId <- insertRcvFileChunk db fc rcvFileId
forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertRcvFileChunkReplica db rno replica chunkId
updateRcvFileNoTmpPath :: DB.Connection -> DBRcvFileId -> IO ()
updateRcvFileNoTmpPath db rcvFileId = do
updatedAt <- getCurrentTime
@@ -2541,13 +2584,18 @@ getRcvFilesExpired db ttl = do
|]
(Only cutoffTs)
createSndFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> CryptoFile -> Int -> FilePath -> C.SbKey -> C.CbNonce -> IO (Either StoreError SndFileId)
createSndFile db gVar userId (CryptoFile path cfArgs) numRecipients prefixPath key nonce =
createSndFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> CryptoFile -> Int -> FilePath -> C.SbKey -> C.CbNonce -> Maybe RedirectFileInfo -> IO (Either StoreError SndFileId)
createSndFile db gVar userId (CryptoFile path cfArgs) numRecipients prefixPath key nonce redirect_ =
createWithRandomId gVar $ \sndFileEntityId ->
DB.execute
db
"INSERT INTO snd_files (snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, prefix_path, key, nonce, status) VALUES (?,?,?,?,?,?,?,?,?,?)"
(sndFileEntityId, userId, path, fileKey <$> cfArgs, fileNonce <$> cfArgs, numRecipients, prefixPath, key, nonce, SFSNew)
"INSERT INTO snd_files (snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, prefix_path, key, nonce, status, redirect_size, redirect_digest) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)"
((sndFileEntityId, userId, path, fileKey <$> cfArgs, fileNonce <$> cfArgs, numRecipients) :. (prefixPath, key, nonce, SFSNew, redirectSize_, redirectDigest_))
where
(redirectSize_, redirectDigest_) =
case redirect_ of
Nothing -> (Nothing, Nothing)
Just RedirectFileInfo {size, digest} -> (Just size, Just digest)
getSndFileByEntityId :: DB.Connection -> SndFileId -> IO (Either StoreError SndFile)
getSndFileByEntityId db sndFileEntityId = runExceptT $ do
@@ -2571,17 +2619,18 @@ getSndFile db sndFileId = runExceptT $ do
DB.query
db
[sql|
SELECT snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, digest, prefix_path, key, nonce, status, deleted
SELECT snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, digest, prefix_path, key, nonce, status, deleted, redirect_size, redirect_digest
FROM snd_files
WHERE snd_file_id = ?
|]
(Only sndFileId)
where
toFile :: (SndFileId, UserId, FilePath, Maybe C.SbKey, Maybe C.CbNonce, Int, Maybe FileDigest, Maybe FilePath, C.SbKey, C.CbNonce, SndFileStatus, Bool) -> SndFile
toFile (sndFileEntityId, userId, srcPath, srcKey_, srcNonce_, numRecipients, digest, prefixPath, key, nonce, status, deleted) =
toFile :: (SndFileId, UserId, FilePath, Maybe C.SbKey, Maybe C.CbNonce, Int, Maybe FileDigest, Maybe FilePath, C.SbKey, C.CbNonce) :. (SndFileStatus, Bool, Maybe (FileSize Int64), Maybe FileDigest) -> SndFile
toFile ((sndFileEntityId, userId, srcPath, srcKey_, srcNonce_, numRecipients, digest, prefixPath, key, nonce) :. (status, deleted, redirectSize_, redirectDigest_)) =
let cfArgs = CFArgs <$> srcKey_ <*> srcNonce_
srcFile = CryptoFile srcPath cfArgs
in SndFile {sndFileId, sndFileEntityId, userId, srcFile, numRecipients, digest, prefixPath, key, nonce, status, deleted, chunks = []}
redirect = RedirectFileInfo <$> redirectSize_ <*> redirectDigest_
in SndFile {sndFileId, sndFileEntityId, userId, srcFile, numRecipients, digest, prefixPath, key, nonce, status, deleted, redirect, chunks = []}
getChunks :: SndFileId -> UserId -> Int -> FilePath -> IO [SndFileChunk]
getChunks sndFileEntityId userId numRecipients filePrefixPath = do
chunks <-
@@ -68,6 +68,7 @@ import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20230829_crypto_files
import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20231222_command_created_at
import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20231225_failed_work_items
import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240121_message_delivery_indexes
import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240124_file_redirect
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Parsers (dropPrefix, sumTypeJSON)
import Simplex.Messaging.Transport.Client (TransportHost)
@@ -104,7 +105,8 @@ schemaMigrations =
("m20230829_crypto_files", m20230829_crypto_files, Just down_m20230829_crypto_files),
("m20231222_command_created_at", m20231222_command_created_at, Just down_m20231222_command_created_at),
("m20231225_failed_work_items", m20231225_failed_work_items, Just down_m20231225_failed_work_items),
("m20240121_message_delivery_indexes", m20240121_message_delivery_indexes, Just down_m20240121_message_delivery_indexes)
("m20240121_message_delivery_indexes", m20240121_message_delivery_indexes, Just down_m20240121_message_delivery_indexes),
("m20240124_file_redirect", m20240124_file_redirect, Just down_m20240124_file_redirect)
]
-- | The list of migrations in ascending order by date
@@ -0,0 +1,34 @@
{-# LANGUAGE QuasiQuotes #-}
module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20240124_file_redirect where
import Database.SQLite.Simple (Query)
import Database.SQLite.Simple.QQ (sql)
m20240124_file_redirect :: Query
m20240124_file_redirect =
[sql|
ALTER TABLE snd_files ADD COLUMN redirect_size INTEGER;
ALTER TABLE snd_files ADD COLUMN redirect_digest BLOB;
ALTER TABLE rcv_files ADD COLUMN redirect_id INTEGER REFERENCES rcv_files ON DELETE CASCADE;
ALTER TABLE rcv_files ADD COLUMN redirect_entity_id BLOB;
ALTER TABLE rcv_files ADD COLUMN redirect_size INTEGER;
ALTER TABLE rcv_files ADD COLUMN redirect_digest BLOB;
CREATE INDEX idx_rcv_files_redirect_id on rcv_files(redirect_id);
|]
down_m20240124_file_redirect :: Query
down_m20240124_file_redirect =
[sql|
DROP INDEX idx_rcv_files_redirect_id;
ALTER TABLE snd_files DROP COLUMN redirect_size;
ALTER TABLE snd_files DROP COLUMN redirect_digest;
ALTER TABLE rcv_files DROP COLUMN redirect_id;
ALTER TABLE rcv_files DROP COLUMN redirect_entity_id;
ALTER TABLE rcv_files DROP COLUMN redirect_size;
ALTER TABLE rcv_files DROP COLUMN redirect_digest;
|]
@@ -279,6 +279,10 @@ CREATE TABLE rcv_files(
save_file_key BLOB,
save_file_nonce BLOB,
failed INTEGER DEFAULT 0,
redirect_id INTEGER REFERENCES rcv_files ON DELETE CASCADE,
redirect_entity_id BLOB,
redirect_size INTEGER,
redirect_digest BLOB,
UNIQUE(rcv_file_entity_id)
);
CREATE TABLE rcv_file_chunks(
@@ -322,7 +326,9 @@ CREATE TABLE snd_files(
,
src_file_key BLOB,
src_file_nonce BLOB,
failed INTEGER DEFAULT 0
failed INTEGER DEFAULT 0,
redirect_size INTEGER,
redirect_digest BLOB
);
CREATE TABLE snd_file_chunks(
snd_file_chunk_id INTEGER PRIMARY KEY,
@@ -508,3 +514,4 @@ CREATE INDEX idx_snd_message_deliveries_expired ON snd_message_deliveries(
failed,
internal_id
);
CREATE INDEX idx_rcv_files_redirect_id on rcv_files(redirect_id);
@@ -30,7 +30,7 @@ import System.IO (BufferMode (..), hSetBuffering, stderr, stdout)
import Text.Read (readMaybe)
ntfServerVersion :: String
ntfServerVersion = "1.7.2.0"
ntfServerVersion = "1.7.3.0"
defaultSMPBatchDelay :: Int
defaultSMPBatchDelay = 10000
@@ -42,6 +42,10 @@ ntfServerCLI cfgPath logPath =
doesFileExist iniFile >>= \case
True -> exitError $ "Error: server is already initialized (" <> iniFile <> " exists).\nRun `" <> executableName <> " start`."
_ -> initializeServer opts
OnlineCert certOpts ->
doesFileExist iniFile >>= \case
True -> genOnline cfgPath certOpts
_ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`."
Start ->
doesFileExist iniFile >>= \case
True -> readIniFile iniFile >>= either exitError runServer
@@ -143,6 +147,7 @@ ntfServerCLI cfgPath logPath =
data CliCommand
= Init InitOptions
| OnlineCert CertOptions
| Start
| Delete
@@ -158,6 +163,7 @@ cliCommandP :: FilePath -> FilePath -> FilePath -> Parser CliCommand
cliCommandP cfgPath logPath iniFile =
hsubparser
( command "init" (info (Init <$> initP) (progDesc $ "Initialize server - creates " <> cfgPath <> " and " <> logPath <> " directories and configuration files"))
<> command "cert" (info (OnlineCert <$> certOptionsP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")"))
<> command "start" (info (pure Start) (progDesc $ "Start server (configuration: " <> iniFile <> ")"))
<> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files"))
)
+2 -11
View File
@@ -173,11 +173,12 @@ import Data.String
import Data.Time.Clock.System (SystemTime (..))
import Data.Type.Equality
import GHC.TypeLits (ErrorMessage (..), TypeError, type (+))
import Network.Socket (HostName, ServiceName)
import Network.Socket (ServiceName)
import qualified Simplex.Messaging.Crypto as C
import Simplex.Messaging.Encoding
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Parsers
import Simplex.Messaging.ServiceScheme
import Simplex.Messaging.Transport
import Simplex.Messaging.Transport.Client (TransportHost, TransportHosts (..))
import Simplex.Messaging.Util (bshow, eitherToMaybe, (<$?>))
@@ -915,16 +916,6 @@ serverStrP = do
where
portP = show <$> (A.char ':' *> (A.decimal :: Parser Int))
data SrvLoc = SrvLoc HostName ServiceName
deriving (Eq, Ord, Show)
instance StrEncoding SrvLoc where
strEncode (SrvLoc host port) = B.pack $ host <> if null port then "" else ':' : port
strP = SrvLoc <$> host <*> (port <|> pure "")
where
host = B.unpack <$> A.takeWhile1 (A.notInClass ":#,;/ ")
port = show <$> (A.char ':' *> (A.decimal :: Parser Int))
-- | Transmission correlation ID.
newtype CorrId = CorrId {bs :: ByteString} deriving (Eq, Ord, Show)
+72 -6
View File
@@ -1,3 +1,4 @@
{-# LANGUAGE ApplicativeDo #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE DuplicateRecordFields #-}
{-# LANGUAGE FlexibleContexts #-}
@@ -10,6 +11,7 @@
module Simplex.Messaging.Server.CLI where
import Control.Monad
import Data.ASN1.Types (asn1CharacterToString)
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import Data.Either (fromRight)
@@ -17,6 +19,8 @@ import Data.Ini (Ini, lookupValue)
import Data.Text (Text)
import qualified Data.Text as T
import Data.Text.Encoding (encodeUtf8)
import qualified Data.X509 as X
import qualified Data.X509.File as XF
import Data.X509.Validation (Fingerprint (..))
import Network.Socket (HostName, ServiceName)
import Options.Applicative
@@ -27,12 +31,14 @@ import Simplex.Messaging.Transport.Server (loadFingerprint)
import Simplex.Messaging.Transport.WebSockets (WS)
import Simplex.Messaging.Util (eitherToMaybe, whenM)
import System.Directory (doesDirectoryExist, listDirectory, removeDirectoryRecursive, removePathForcibly)
import System.Environment (lookupEnv)
import System.Exit (exitFailure)
import System.FilePath (combine)
import System.IO (IOMode (..), hFlush, hGetLine, stdout, withFile)
import System.Process (readCreateProcess, shell)
import Text.Read (readMaybe)
exitError :: String -> IO ()
exitError :: String -> IO a
exitError msg = putStrLn msg >> exitFailure
confirmOrExit :: String -> IO ()
@@ -84,14 +90,18 @@ getCliCommand' cmdP version =
versionOption = infoOption version (long "version" <> short 'v' <> help "Show version")
createServerX509 :: FilePath -> X509Config -> IO ByteString
createServerX509 cfgPath x509cfg = do
createOpensslCaConf
createOpensslServerConf
createServerX509 = createServerX509_ True
createServerX509_ :: Bool -> FilePath -> X509Config -> IO ByteString
createServerX509_ createCA cfgPath x509cfg = do
let alg = show $ signAlgorithm (x509cfg :: X509Config)
-- CA certificate (identity/offline)
run $ "openssl genpkey -algorithm " <> alg <> " -out " <> c caKeyFile
run $ "openssl req -new -x509 -days 999999 -config " <> c opensslCaConfFile <> " -extensions v3 -key " <> c caKeyFile <> " -out " <> c caCrtFile
when createCA $ do
createOpensslCaConf
run $ "openssl genpkey -algorithm " <> alg <> " -out " <> c caKeyFile
run $ "openssl req -new -x509 -days 999999 -config " <> c opensslCaConfFile <> " -extensions v3 -key " <> c caKeyFile <> " -out " <> c caCrtFile
-- server certificate (online)
createOpensslServerConf
run $ "openssl genpkey -algorithm " <> alg <> " -out " <> c serverKeyFile
run $ "openssl req -new -config " <> c opensslServerConfFile <> " -reqexts v3 -key " <> c serverKeyFile <> " -out " <> c serverCsrFile
run $ "openssl x509 -req -days 999999 -extfile " <> c opensslServerConfFile <> " -extensions v3 -in " <> c serverCsrFile <> " -CA " <> c caCrtFile <> " -CAkey " <> c caKeyFile <> " -CAcreateserial -out " <> c serverCrtFile
@@ -131,6 +141,59 @@ createServerX509 cfgPath x509cfg = do
withFile (c fingerprintFile) WriteMode (`B.hPutStrLn` strEncode fp)
pure fp
data CertOptions = CertOptions
{ signAlgorithm_ :: Maybe SignAlgorithm,
commonName_ :: Maybe HostName
}
deriving (Show)
certOptionsP :: Parser CertOptions
certOptionsP = do
signAlgorithm_ <-
optional $
option
(maybeReader readMaybe)
( long "sign-algorithm"
<> short 'a'
<> help "Set new signature algorithm used for TLS certificates: ED25519, ED448"
<> metavar "ALG"
)
commonName_ <-
optional $
strOption
( long "cn"
<> help
"Set new Common Name for TLS online certificate"
<> metavar "FQDN"
)
pure CertOptions {signAlgorithm_, commonName_}
genOnline :: FilePath -> CertOptions -> IO ()
genOnline cfgPath CertOptions {signAlgorithm_, commonName_} = do
(signAlgorithm, commonName) <-
case (signAlgorithm_, commonName_) of
(Just alg, Just cn) -> pure (alg, cn)
_ ->
XF.readSignedObject certPath >>= \case
[old] -> either exitError pure . fromX509 . X.signedObject $ X.getSigned old
[] -> exitError $ "No certificate found at " <> certPath
_ -> exitError $ "Too many certificates at " <> certPath
let x509cfg = defaultX509Config {signAlgorithm, commonName}
void $ createServerX509_ False cfgPath x509cfg
putStrLn "Generated new server credentials"
warnCAPrivateKeyFile cfgPath x509cfg
where
certPath = combine cfgPath $ serverCrtFile defaultX509Config
fromX509 X.Certificate {certSignatureAlg, certSubjectDN} = (,) <$> maybe oldAlg Right signAlgorithm_ <*> maybe oldCN Right commonName_
where
oldAlg = case certSignatureAlg of
X.SignatureALG_IntrinsicHash X.PubKeyALG_Ed448 -> Right ED448
X.SignatureALG_IntrinsicHash X.PubKeyALG_Ed25519 -> Right ED25519
alg -> Left $ "Unexpected signature algorithm " <> show alg
oldCN = case X.getDnElement X.DnCommonName certSubjectDN of
Nothing -> Left "Certificate subject has no CN element"
Just cn -> maybe (Left "Certificate subject CN decoding failed") Right $ asn1CharacterToString cn
warnCAPrivateKeyFile :: FilePath -> X509Config -> IO ()
warnCAPrivateKeyFile cfgPath X509Config {caKeyFile} =
putStrLn $
@@ -235,3 +298,6 @@ printServiceInfo serverVersion srv@(ProtoServerWithAuth ProtocolServer {keyHash}
clearDirIfExists :: FilePath -> IO ()
clearDirIfExists path = whenM (doesDirectoryExist path) $ listDirectory path >>= mapM_ (removePathForcibly . combine path)
getEnvPath :: String -> FilePath -> IO FilePath
getEnvPath name def = maybe def (\case "" -> def; f -> f) <$> lookupEnv name
+13 -6
View File
@@ -41,6 +41,10 @@ smpServerCLI cfgPath logPath =
doesFileExist iniFile >>= \case
True -> exitError $ "Error: server is already initialized (" <> iniFile <> " exists).\nRun `" <> executableName <> " start`."
_ -> initializeServer opts
OnlineCert certOpts ->
doesFileExist iniFile >>= \case
True -> genOnline cfgPath certOpts
_ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`."
Start ->
doesFileExist iniFile >>= \case
True -> readIniFile iniFile >>= either exitError runServer
@@ -56,8 +60,8 @@ smpServerCLI cfgPath logPath =
defaultServerPort = "5223"
executableName = "smp-server"
storeLogFilePath = combine logPath "smp-server-store.log"
initializeServer opts
| scripted opts = initialize opts
initializeServer opts@InitOptions {ip, fqdn, scripted}
| scripted = initialize opts
| otherwise = do
putStrLn "Use `smp-server init -h` for available options."
void $ withPrompt "SMP server will be initialized (press Enter)" getLine
@@ -65,9 +69,9 @@ smpServerCLI cfgPath logPath =
logStats <- onOffPrompt "Enable logging daily statistics" False
putStrLn "Require a password to create new messaging queues?"
password <- withPrompt "'r' for random (default), 'n' - no password, or enter password: " serverPassword
let host = fromMaybe (ip opts) (fqdn opts)
let host = fromMaybe ip fqdn
host' <- withPrompt ("Enter server FQDN or IP address for certificate (" <> host <> "): ") getLine
initialize opts {enableStoreLog, logStats, fqdn = if null host' then fqdn opts else Just host', password}
initialize opts {enableStoreLog, logStats, fqdn = if null host' then fqdn else Just host', password}
where
serverPassword =
getLine >>= \case
@@ -78,7 +82,7 @@ smpServerCLI cfgPath logPath =
case strDecode $ encodeUtf8 $ T.pack s of
Right auth -> pure . Just $ ServerPassword auth
_ -> putStrLn "Invalid password. Only latin letters, digits and symbols other than '@' and ':' are allowed" >> serverPassword
initialize InitOptions {enableStoreLog, logStats, signAlgorithm, ip, fqdn, password} = do
initialize InitOptions {enableStoreLog, logStats, signAlgorithm, password} = do
clearDirIfExists cfgPath
clearDirIfExists logPath
createDirectoryIfMissing True cfgPath
@@ -210,6 +214,7 @@ smpServerCLI cfgPath logPath =
data CliCommand
= Init InitOptions
| OnlineCert CertOptions
| Start
| Delete
@@ -231,6 +236,7 @@ cliCommandP :: FilePath -> FilePath -> FilePath -> Parser CliCommand
cliCommandP cfgPath logPath iniFile =
hsubparser
( command "init" (info (Init <$> initP) (progDesc $ "Initialize server - creates " <> cfgPath <> " and " <> logPath <> " directories and configuration files"))
<> command "cert" (info (OnlineCert <$> certOptionsP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")"))
<> command "start" (info (pure Start) (progDesc $ "Start server (configuration: " <> iniFile <> ")"))
<> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files"))
)
@@ -255,7 +261,7 @@ cliCommandP cfgPath logPath iniFile =
( long "sign-algorithm"
<> short 'a'
<> help "Signature algorithm used for TLS certificates: ED25519, ED448"
<> value ED448
<> value ED25519
<> showDefault
<> metavar "ALG"
)
@@ -295,3 +301,4 @@ cliCommandP cfgPath logPath iniFile =
pure InitOptions {enableStoreLog, logStats, signAlgorithm, ip, fqdn, password, scripted}
parseBasicAuth :: ReadM ServerPassword
parseBasicAuth = eitherReader $ fmap ServerPassword . strDecode . B.pack
+35
View File
@@ -0,0 +1,35 @@
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE OverloadedStrings #-}
module Simplex.Messaging.ServiceScheme where
import Control.Applicative ((<|>))
import qualified Data.Attoparsec.ByteString.Char8 as A
import qualified Data.ByteString.Char8 as B
import Data.Functor (($>))
import Network.Socket (HostName, ServiceName)
import Simplex.Messaging.Encoding.String (StrEncoding (..))
data ServiceScheme = SSSimplex | SSAppServer SrvLoc
deriving (Eq, Show)
instance StrEncoding ServiceScheme where
strEncode = \case
SSSimplex -> "simplex:"
SSAppServer srv -> "https://" <> strEncode srv
strP =
"simplex:" $> SSSimplex
<|> "https://" *> (SSAppServer <$> strP)
data SrvLoc = SrvLoc HostName ServiceName
deriving (Eq, Ord, Show)
instance StrEncoding SrvLoc where
strEncode (SrvLoc host port) = B.pack $ host <> if null port then "" else ':' : port
strP = SrvLoc <$> host <*> (port <|> pure "")
where
host = B.unpack <$> A.takeWhile1 (A.notInClass ":#,;/ ")
port = show <$> (A.char ':' *> (A.decimal :: A.Parser Int))
simplexChat :: ServiceScheme
simplexChat = SSAppServer $ SrvLoc "simplex.chat" ""