diff --git a/protocol/xftp.md b/protocol/xftp.md index 4180e652b..4fcebea90 100644 --- a/protocol/xftp.md +++ b/protocol/xftp.md @@ -1,4 +1,4 @@ -Version 3, 2025-01-24 +Version 4, 2026-08-08 # SimpleX File Transfer Protocol @@ -50,11 +50,12 @@ The objective of SimpleX File Transfer Protocol (XFTP) is to facilitate the secu XFTP is implemented as an application level protocol on top of HTTP2 and TLS. -This document describes XFTP protocol version 3. The version history: +This document describes XFTP protocol version 4. The version history: - v1: initial version - v2: authenticated commands - added basic auth support for commands - v3: blocked files - added BLOCKED error type for policy violations +- v4: server public information in handshake The protocol describes the set of commands that senders and recipients can send to XFTP routers to create, upload, download and delete data packets of several pre-defined sizes. XFTP routers SHOULD support packets of 4 sizes: 64KB, 256KB, 1MB and 4MB (1KB = 1024 bytes, 1MB = 1024KB). diff --git a/src/Simplex/FileTransfer/Client.hs b/src/Simplex/FileTransfer/Client.hs index 270f210f7..8ff212d57 100644 --- a/src/Simplex/FileTransfer/Client.hs +++ b/src/Simplex/FileTransfer/Client.hs @@ -39,6 +39,7 @@ import Control.Monad.Except import Control.Monad.IO.Class (liftIO) import Control.Monad.Trans.Except import Crypto.Random (ChaChaDRG) +import qualified Data.Aeson as J import Data.Bifunctor (first) import Data.ByteString.Builder (Builder, byteString) import Data.ByteString.Char8 (ByteString) @@ -157,13 +158,14 @@ getXFTPClient transportSession@(_, srv, _) config@XFTPClientConfig {clientALPN, xftpClientHandshakeV1 :: VersionRangeXFTP -> C.KeyHash -> HTTP2Client -> (SessionId -> IO (Maybe EntitlementProof)) -> THandleParamsXFTP 'TClient -> ExceptT XFTPClientError IO (THandleParamsXFTP 'TClient) xftpClientHandshakeV1 serverVRange keyHash@(C.KeyHash kh) c@HTTP2Client {sessionId, serverKey} mkEntitlementProof thParams0 = do - shs@XFTPServerHandshake {authPubKey = ck} <- getServerHandshake + shs@XFTPServerHandshake {authPubKey = ck, serverInfoBytes} <- getServerHandshake (vr, sk) <- processServerHandshake shs let v = maxVersion vr ep <- if v >= fileStorageTimeXFTPVersion then liftIO (mkEntitlementProof sessionId) else pure Nothing sendClientHandshake XFTPClientHandshake {xftpVersion = v, keyHash, entitlementProof = ep} let thAuth = Just THAuthClient {peerServerPubKey = sk, peerServerCertKey = ck, clientService = Nothing, sessSecret = Nothing} - pure thParams0 {thAuth, thVersion = v, thServerVRange = vr} + serverInfo = J.eitherDecodeStrict' <$> serverInfoBytes + pure thParams0 {thAuth, thVersion = v, thServerVRange = vr, serverInfo} where getServerHandshake :: ExceptT XFTPClientError IO XFTPServerHandshake getServerHandshake = do diff --git a/src/Simplex/FileTransfer/Server.hs b/src/Simplex/FileTransfer/Server.hs index 0f6ac5cfa..74a6f5ed5 100644 --- a/src/Simplex/FileTransfer/Server.hs +++ b/src/Simplex/FileTransfer/Server.hs @@ -23,11 +23,13 @@ import Control.Monad import Control.Monad.Except import Control.Monad.Reader import Control.Monad.Trans.Except +import qualified Data.Aeson as J import Data.Bifunctor (first) import qualified Data.ByteString.Base64.URL as B64 import Data.ByteString.Builder (Builder, byteString) import Data.ByteString.Char8 (ByteString) import qualified Data.ByteString.Char8 as B +import qualified Data.ByteString.Lazy.Char8 as LB import Data.Int (Int64) import Data.List.NonEmpty (NonEmpty) import qualified Data.List.NonEmpty as L @@ -131,7 +133,7 @@ data Handshake data EntitlementChecked = EntNotChecked | EntChecked (Maybe SessionEntitlement) xftpServer :: forall s. FileStoreClass s => XFTPServerConfig s -> TMVar Bool -> M s () -xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpiration, fileExpiration, xftpServerVRange} started = do +xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpiration, fileExpiration, xftpServerVRange, information} started = do expireServerFiles Nothing fileExpiration restoreServerStats raceAny_ @@ -212,7 +214,8 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira fst kp <$ TM.insert sessionId (HandshakeSent (snd kp) ent_) sessions let authPubKey = CertChainPubKey chain (C.signX509 serverSignKey $ C.publicToX509 k) webIdentityProof = C.sign serverSignKey . (<> sessionId) <$> challenge_ - let hs = XFTPServerHandshake {xftpVersionRange = xftpServerVRange, sessionId, authPubKey, webIdentityProof} + serverInfoBytes = LB.toStrict . J.encode <$> information + let hs = XFTPServerHandshake {xftpVersionRange = xftpServerVRange, sessionId, authPubKey, webIdentityProof, serverInfoBytes} shs <- encodeXftp hs #ifdef slow_servers lift randomDelay diff --git a/src/Simplex/FileTransfer/Server/Env.hs b/src/Simplex/FileTransfer/Server/Env.hs index d5bbb1ac3..852036af3 100644 --- a/src/Simplex/FileTransfer/Server/Env.hs +++ b/src/Simplex/FileTransfer/Server/Env.hs @@ -68,6 +68,7 @@ import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Crypto.BBS (BBSPublicKey) import Simplex.Messaging.Protocol (BasicAuth, RcvPublicAuthKey) import Simplex.Messaging.Server.Expiration +import Simplex.Messaging.Server.Information (ServerPublicInfo) import Simplex.Messaging.Transport (EntitlementConfig (..)) import Simplex.Messaging.Transport.Server (ServerCredentials (..), TransportServerConfig (..), loadFingerprint, loadServerCredential) import Simplex.Messaging.Util (tshow) @@ -105,6 +106,8 @@ data XFTPServerConfig s = XFTPServerConfig httpCredentials :: Maybe ServerCredentials, -- | XFTP client-server protocol version range xftpServerVRange :: VersionRangeXFTP, + -- | server public information sent in handshake and used to generate static mini-site + information :: Maybe ServerPublicInfo, -- stats config - see SMP server config logStatsInterval :: Maybe Int64, logStatsStartTime :: Int64, diff --git a/src/Simplex/FileTransfer/Server/Main.hs b/src/Simplex/FileTransfer/Server/Main.hs index 30ec1027f..b3515b492 100644 --- a/src/Simplex/FileTransfer/Server/Main.hs +++ b/src/Simplex/FileTransfer/Server/Main.hs @@ -318,6 +318,7 @@ xftpServerCLI_ generateSite serveStaticFiles cfgPath logPath = do }, httpCredentials = httpCredentials_, xftpServerVRange = supportedFileServerVRange, + information = serverPublicInfo ini, logStatsInterval = logStats $> 86400, -- seconds logStatsStartTime = 0, -- seconds from 00:00 UTC serverStatsLogFile = combine logPath "file-server-stats.daily.log", diff --git a/src/Simplex/FileTransfer/Transport.hs b/src/Simplex/FileTransfer/Transport.hs index eb9677bdf..b51d1e483 100644 --- a/src/Simplex/FileTransfer/Transport.hs +++ b/src/Simplex/FileTransfer/Transport.hs @@ -37,7 +37,6 @@ module Simplex.FileTransfer.Transport ) where -import Control.Applicative (optional) import qualified Control.Exception as E import Control.Logger.Simple import Control.Monad @@ -64,7 +63,7 @@ import Simplex.Messaging.Parsers import Simplex.Messaging.Protocol (BlockingInfo, CommandError) import Simplex.Messaging.Transport (ALPN, CertChainPubKey, ServiceCredentials, SessionId, THandle (..), THandleParams (..), TransportError (..), TransportPeer (..)) import Simplex.Messaging.Transport.HTTP2.File -import Simplex.Messaging.Util (bshow, tshow, (<$?>)) +import Simplex.Messaging.Util (bshow, tshow, (<$?>), (<$$>)) import Simplex.Messaging.Version import Simplex.Messaging.Version.Internal import System.IO (Handle, IOMode (..), withFile) @@ -129,7 +128,9 @@ data XFTPServerHandshake = XFTPServerHandshake -- | pub key to agree shared secrets for command authorization and entity ID encryption. authPubKey :: CertChainPubKey, -- | signed identity challenge from XFTPClientHello - webIdentityProof :: Maybe C.ASignature + webIdentityProof :: Maybe C.ASignature, + -- | optional server public information (JSON-encoded ServerPublicInfo), sent when version >= serverInfoXFTPVersion + serverInfoBytes :: Maybe ByteString } data XFTPClientHandshake = XFTPClientHandshake @@ -162,13 +163,21 @@ ifHasEntitlement :: VersionXFTP -> a -> a -> a ifHasEntitlement v a b = if v >= fileStorageTimeXFTPVersion then a else b instance Encoding XFTPServerHandshake where - smpEncode XFTPServerHandshake {xftpVersionRange, sessionId, authPubKey, webIdentityProof} = - smpEncode (xftpVersionRange, sessionId, authPubKey, C.signatureBytes webIdentityProof) + smpEncode XFTPServerHandshake {xftpVersionRange, sessionId, authPubKey, webIdentityProof, serverInfoBytes} = + smpEncode (xftpVersionRange, sessionId, authPubKey, C.signatureBytes webIdentityProof) <> info + where + info = ifHasServerInfo (maxVersion xftpVersionRange) (smpEncode (Large <$> serverInfoBytes)) "" smpP = do (xftpVersionRange, sessionId, authPubKey) <- smpP - webIdentityProof <- optional $ C.decodeSignature <$?> smpP + -- decode the (length-prefixed) signature bytes deterministically: empty bytes decode to Nothing. + -- (Must not use `optional`, which would backtrack and leave the bytes for the parsers that follow.) + webIdentityProof <- C.decodeSignature <$?> smpP + serverInfoBytes <- ifHasServerInfo (maxVersion xftpVersionRange) (unLarge <$$> smpP) (pure Nothing) Tail _compat <- smpP - pure XFTPServerHandshake {xftpVersionRange, sessionId, authPubKey, webIdentityProof} + pure XFTPServerHandshake {xftpVersionRange, sessionId, authPubKey, webIdentityProof, serverInfoBytes} + +ifHasServerInfo :: VersionXFTP -> a -> a -> a +ifHasServerInfo v a b = if v >= fileStorageTimeXFTPVersion then a else b sendEncFile :: Handle -> (Builder -> IO ()) -> LC.SbState -> Word32 -> IO () sendEncFile h send = go diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs index b7aa37ddd..5caaa5be3 100644 --- a/src/Simplex/Messaging/Agent.hs +++ b/src/Simplex/Messaging/Agent.hs @@ -681,7 +681,7 @@ getConnectionRatchetAdHash c = withAgentEnv c . getConnectionRatchetAdHash' c testProtocolServer :: forall p. ProtocolTypeI p => AgentClient -> NetworkRequestMode -> UserId -> ProtoServerWithAuth p -> IO (Either ProtocolTestFailure (Maybe (Either String ServerPublicInfo))) testProtocolServer c nm userId srv = withAgentEnv' c $ case protocolTypeI @p of SPSMP -> runSMPServerTest c nm userId srv - SPXFTP -> maybe (Right Nothing) Left <$> runXFTPServerTest c nm userId srv + SPXFTP -> runXFTPServerTest c nm userId srv SPNTF -> maybe (Right Nothing) Left <$> runNTFServerTest c nm userId srv -- | set SOCKS5 proxy on/off and optionally set TCP timeouts for fast network diff --git a/src/Simplex/Messaging/Agent/Client.hs b/src/Simplex/Messaging/Agent/Client.hs index 56a9dc5ea..488a5d0e5 100644 --- a/src/Simplex/Messaging/Agent/Client.hs +++ b/src/Simplex/Messaging/Agent/Client.hs @@ -1355,7 +1355,7 @@ runSMPServerTest c@AgentClient {presetDomains} nm userId (ProtoServerWithAuth sr testErr :: ProtocolTestStep -> SMPClientError -> ProtocolTestFailure testErr step = ProtocolTestFailure step . protocolClientError SMP addr -runXFTPServerTest :: AgentClient -> NetworkRequestMode -> UserId -> XFTPServerWithAuth -> AM' (Maybe ProtocolTestFailure) +runXFTPServerTest :: AgentClient -> NetworkRequestMode -> UserId -> XFTPServerWithAuth -> AM' (Either ProtocolTestFailure (Maybe (Either String ServerPublicInfo))) runXFTPServerTest c@AgentClient {presetDomains} nm userId (ProtoServerWithAuth srv auth) = do cfg <- asks $ xftpCfg . config g <- asks random @@ -1381,8 +1381,8 @@ runXFTPServerTest c@AgentClient {presetDomains} nm userId (ProtoServerWithAuth s unless (digest == rcvDigest) $ throwE $ ProtocolTestFailure TSCompareFile $ XFTP (B.unpack $ strEncode srv) DIGEST liftError (testErr TSDeleteFile) $ X.deleteXFTPChunk xftp spKey sId ok <- netTimeoutInt (tcpTimeout xftpNetworkConfig) nm `timeout` X.closeXFTPClient xftp - pure $ either Just (const Nothing) r <|> maybe (Just (ProtocolTestFailure TSDisconnect $ BROKER addr TIMEOUT)) (const Nothing) ok - Left e -> pure (Just $ testErr TSConnect e) + pure $ r >> maybe (Left (ProtocolTestFailure TSDisconnect $ BROKER addr TIMEOUT)) (const $ Right $ serverInfo (X.thParams xftp)) ok + Left e -> pure $ Left (testErr TSConnect e) where addr = B.unpack $ strEncode srv testErr :: ProtocolTestStep -> XFTPClientError -> ProtocolTestFailure diff --git a/tests/AgentTests/FunctionalAPITests.hs b/tests/AgentTests/FunctionalAPITests.hs index d831cdea7..cf6c3c8f6 100644 --- a/tests/AgentTests/FunctionalAPITests.hs +++ b/tests/AgentTests/FunctionalAPITests.hs @@ -54,6 +54,7 @@ module AgentTests.FunctionalAPITests pattern SENT, agentCfgVPrevPQ, agentCfgV7, + testServerInformation, ) where diff --git a/tests/XFTPAgent.hs b/tests/XFTPAgent.hs index 961b2a862..8237db2ab 100644 --- a/tests/XFTPAgent.hs +++ b/tests/XFTPAgent.hs @@ -11,7 +11,7 @@ module XFTPAgent where -import AgentTests.FunctionalAPITests (get, rfGet, runRight, runRight_, sfGet, withAgent) +import AgentTests.FunctionalAPITests (get, rfGet, runRight, runRight_, sfGet, withAgent, testServerInformation) import Control.Logger.Simple import Control.Monad @@ -97,7 +97,7 @@ xftpAgentTests = it "if file is expired on server, should report error and continue receiving next file" testXFTPAgentExpiredOnServer it "should request additional recipient IDs when number of recipients exceeds maximum per request" testXFTPAgentRequestAdditionalRecipientIDs describe "XFTP server test via agent API" $ do - it "should pass without basic auth" $ \_ -> testXFTPServerTest Nothing (noAuthSrv testXFTPServer2) `shouldReturn` Right Nothing + it "should pass without basic auth" $ \_ -> testXFTPServerTest Nothing (noAuthSrv testXFTPServer2) `shouldReturn` Right (Just (Right testServerInformation)) let srv1 = testXFTPServer2 {keyHash = "1234"} it "should fail with incorrect fingerprint" $ \_ -> do testXFTPServerTest Nothing (noAuthSrv srv1) `shouldReturn` Left (ProtocolTestFailure TSConnect $ BROKER (B.unpack $ strEncode srv1) $ NETWORK NEUnknownCAError) @@ -105,13 +105,13 @@ xftpAgentTests = let auth = Just "abcd" srv = ProtoServerWithAuth testXFTPServer2 authErr = ProtocolTestFailure TSCreateFile $ XFTP (B.unpack $ strEncode testXFTPServer2) AUTH - it "should pass with correct password" $ \_ -> testXFTPServerTest auth (srv auth) `shouldReturn` Right Nothing + it "should pass with correct password" $ \_ -> testXFTPServerTest auth (srv auth) `shouldReturn` Right (Just (Right testServerInformation)) it "should fail without password" $ \_ -> testXFTPServerTest auth (srv Nothing) `shouldReturn` Left authErr it "should fail with incorrect password" $ \_ -> testXFTPServerTest auth (srv $ Just "wrong") `shouldReturn` Left authErr testXFTPServerTest :: HasCallStack => Maybe BasicAuth -> XFTPServerWithAuth -> IO (Either ProtocolTestFailure (Maybe (Either String ServerPublicInfo))) testXFTPServerTest newFileBasicAuth srv = - withXFTPServerCfg testXFTPServerConfig {newFileBasicAuth, xftpPort = xftpTestPort2} $ \_ -> + withXFTPServerCfg testXFTPServerConfig {newFileBasicAuth, xftpPort = xftpTestPort2, information = Just testServerInformation} $ \_ -> -- initially passed server is not running withAgent 1 agentCfg initAgentServers testDB $ \a -> testProtocolServer a NRMInteractive 1 srv diff --git a/tests/XFTPClient.hs b/tests/XFTPClient.hs index 1c5b58100..772b1b692 100644 --- a/tests/XFTPClient.hs +++ b/tests/XFTPClient.hs @@ -201,6 +201,7 @@ testXFTPServerConfig = }, httpCredentials = Nothing, xftpServerVRange = supportedFileServerVRange, + information = Nothing, logStatsInterval = Nothing, logStatsStartTime = 0, serverStatsLogFile = "tests/tmp/xftp-server-stats.daily.log",