diff --git a/cbits/sntrup761.c b/cbits/sntrup761.c index 7d64eeb08..487e84b48 100644 --- a/cbits/sntrup761.c +++ b/cbits/sntrup761.c @@ -719,23 +719,32 @@ Small_random (small * out, void *random_ctx, sntrup761_random_func * random) /* ----- Streamlined NTRU Prime Core */ +/* x^p-x-1 has a degree-19 factor mod 3, so a random g is not invertible in R3 + with probability about 3^-19; KeyGen_attempts failures in a row mean a broken RNG */ +#define KeyGen_attempts 10 + /* h,(f,ginv) = KeyGen() */ -static void +/* returns 0 if KeyGen succeeded; else -1 */ +static int KeyGen (Fq * h, small * f, small * ginv, void *random_ctx, sntrup761_random_func * random) { small g[p]; Fq finv[p]; + int i; - for (;;) + for (i = 0; i < KeyGen_attempts; ++i) { Small_random (g, random_ctx, random); if (R3_recip (ginv, g) == 0) break; } + if (i == KeyGen_attempts) + return -1; Short_random (f, random_ctx, random); Rq_recip3 (finv, f); /* always works */ Rq_mult_small (h, finv, g); + return 0; } /* c = Encrypt(r,h) */ @@ -884,18 +893,21 @@ typedef small Inputs[p]; /* passed by reference */ #define PublicKeys_bytes Rq_bytes /* pk,sk = ZKeyGen() */ -static void +/* returns 0 if KeyGen succeeded; else -1 */ +static int ZKeyGen (unsigned char *pk, unsigned char *sk, void *random_ctx, sntrup761_random_func * random) { Fq h[p]; small f[p], v[p]; - KeyGen (h, f, v, random_ctx, random); + if (KeyGen (h, f, v, random_ctx, random) != 0) + return -1; Rq_encode (pk, h); Small_encode (sk, f); sk += Small_bytes; Small_encode (sk, v); + return 0; } /* C = ZEncrypt(r,pk) */ @@ -960,19 +972,21 @@ HashSession (unsigned char *k, int b, const unsigned char *y, /* ----- Streamlined NTRU Prime */ /* pk,sk = KEM_KeyGen() */ -void +int sntrup761_keypair (unsigned char *pk, unsigned char *sk, void *random_ctx, sntrup761_random_func * random) { int i; - ZKeyGen (pk, sk, random_ctx, random); + if (ZKeyGen (pk, sk, random_ctx, random) != 0) + return -1; sk += SecretKeys_bytes; for (i = 0; i < PublicKeys_bytes; ++i) *sk++ = pk[i]; random (random_ctx, Inputs_bytes, sk); sk += Inputs_bytes; Hash_prefix (sk, 4, pk, PublicKeys_bytes); + return 0; } /* c,r_enc = Hide(r,pk,cache); cache is Hash4(pk) */ diff --git a/cbits/sntrup761.h b/cbits/sntrup761.h index 4b1a23bd9..acb221611 100644 --- a/cbits/sntrup761.h +++ b/cbits/sntrup761.h @@ -19,7 +19,8 @@ typedef void sntrup761_random_func (void *ctx, size_t length, uint8_t *dst); -void +/* returns 0 on success, -1 if the RNG never produced an invertible polynomial */ +int sntrup761_keypair (uint8_t *pk, uint8_t *sk, void *random_ctx, sntrup761_random_func *random); diff --git a/contributing/CODE.md b/contributing/CODE.md index ab5d7efcc..b6ced3089 100644 --- a/contributing/CODE.md +++ b/contributing/CODE.md @@ -7,6 +7,7 @@ This file provides guidance on coding style and approaches and on building the c When designing code and planning implementations: - Apply adversarial thinking, and consider what may happen if one of the communicating parties is malicious. - Formulate an explicit threat model for each change - who can do which undesirable things and under which circumstances. +- Use the default cryptographic primitive for each purpose from the [primitive registry](../protocol/crypto-registry.md); any exception requires review, and the registry must be updated with every new primitive use or domain-separation string. ## Code Quality Standards diff --git a/protocol/agent-protocol.md b/protocol/agent-protocol.md index 1a047ee13..f30b33b92 100644 --- a/protocol/agent-protocol.md +++ b/protocol/agent-protocol.md @@ -85,7 +85,7 @@ SMP agent protocol has 2 main parts: ![Duplex connection procedure](./diagrams/duplex-messaging/duplex-creating.svg) -The procedure of establishing a duplex connection is explained on the example of Alice and Bob creating a bi-directional connection consisting of two unidirectional (simplex) queues, using SMP agents (A and B) to facilitate it, and two different SMP routers (which could be the same router). It is shown on the diagram above and has these steps: +The procedure of establishing a duplex connection is explained on the example of Alice (the initiating party) and Bob (the joining party) creating a bi-directional connection consisting of two unidirectional (simplex) queues, using SMP agents (A and B) to facilitate it, and two different SMP routers (which could be the same router). It is shown on the diagram above and has these steps: 1. Alice requests the new connection from the SMP agent A using agent `createConnection` api function. 2. Agent A creates an SMP queue on the router (using [SMP protocol](./simplex-messaging.md) `NEW` command) and responds to Alice with the invitation that contains queue information and the encryption keys Bob's agent B should use. The invitation format is described in [Connection link](connection-link-1-time-invitation-and-contact-address). @@ -208,7 +208,7 @@ This syntax of decrypted SMP client message body is defined by `decryptedAgentMe Decrypted SMP message client body can be one of 4 types: - `agentConnInfo` - used by the initiating party when confirming reply queue - sent in `agentConfirmation` envelope. - `agentConnInfoReply` - used by accepting party, includes reply queue(s) in the initial confirmation - sent in `agentConfirmation` envelope. -- `agentRatchetInfo` - used to pass additional information when renegotiating double ratchet encryption - sent in `agentRatchetKey` envelope. +- `agentRatchetInfo` - used to pass additional information when renegotiating double ratchet encryption - sent in `agentRatchetKey` envelope. A key sent in reply to another key includes the hash of that key; agents do not reply to such keys. - `agentMessage` - all other agent messages. `agentMessage` contains these parts: @@ -233,7 +233,8 @@ connInfo = *OCTET agentConnInfoReply = %s"D" smpQueues connInfo smpQueues = length 1*newQueueInfo ; NonEmpty list of reply queues agentRatchetInfo = %s"R" ratchetInfo -ratchetInfo = *OCTET +ratchetInfo = [answeredKeyHash *OCTET] ; bytes after answeredKeyHash are ignored +answeredKeyHash = %s"0" / (%s"1" shortString) ; "0" in a key that starts renegotiation, otherwise SHA-256 of the two raw public keys of the answered key agentMessage = %s"M" agentMsgHeader aMessage agentMsgHeader = agentMsgId prevMsgHash diff --git a/protocol/crypto-registry.md b/protocol/crypto-registry.md new file mode 100644 index 000000000..2154e65a2 --- /dev/null +++ b/protocol/crypto-registry.md @@ -0,0 +1,189 @@ +Revision 1, 2026-10-01 + +# SimpleX Network: cryptographic primitive registry + +The cryptographic primitives, constructions and domain-separation strings used by this repository, and which of them new code may use. For the threat model see [Security](./security.md). + +Module aliases follow the codebase: `C` = [Simplex.Messaging.Crypto](../src/Simplex/Messaging/Crypto.hs), `LC` = [Crypto.Lazy](../src/Simplex/Messaging/Crypto/Lazy.hs), `CR` = [Crypto.Ratchet](../src/Simplex/Messaging/Crypto/Ratchet.hs), `SL` = [Crypto.ShortLink](../src/Simplex/Messaging/Crypto/ShortLink.hs), `KEM` = [Crypto.SNTRUP761](../src/Simplex/Messaging/Crypto/SNTRUP761.hs) and [its bindings](../src/Simplex/Messaging/Crypto/SNTRUP761/Bindings.hs), `BBS` = [Crypto.BBS](../src/Simplex/Messaging/Crypto/BBS.hs). Lengths are in bytes. + +## Table of contents + +- [Policy](#policy) +- [Defaults for new code](#defaults-for-new-code) +- [Primitives](#primitives) +- [SimpleX box constructions](#simplex-box-constructions) +- [Domain separation and KDF inputs](#domain-separation-and-kdf-inputs) +- [Nonces and IVs](#nonces-and-ivs) +- [Implementation sources](#implementation-sources) +- [xftp-web (TypeScript)](#xftp-web-typescript) +- [Tests](#tests) + +## Policy + +- New code uses the default primitive for its purpose from [Defaults for new code](#defaults-for-new-code). +- Using any other primitive, or a primitive marked *restricted* outside its listed purpose, requires a written justification in the pull request and review by a maintainer responsible for cryptography. +- *Legacy-only* primitives stay only for the existing wire formats and stored data listed below. Do not add call sites. +- Every new HKDF info string or hash prefix must be unique, start with `SimpleX`, and be added to [Domain separation and KDF inputs](#domain-separation-and-kdf-inputs). +- Changing a primitive, key length, KDF input, info string or nonce construction changes the wire format. It requires a protocol version and an update of this registry and of the affected `protocol/` specification. + +Status values used below: + +| Status | Meaning | +|---|---| +| Default | Approved and preferred for new code for this purpose | +| Approved | Approved for new code for this purpose when the default does not fit | +| Restricted | Approved only for the listed purpose (external interoperability or a single protocol) | +| Legacy-only | Kept for existing wire formats or stored data; no new call sites | + +## Defaults for new code + +| Purpose | Default | Functions | +|---|---|---| +| Signature | Ed25519 | `C.sign'`, `C.verify'` with `C.SEd25519` | +| Key agreement | X25519 | `C.generateKeyPair`, `C.dh'` | +| Post-quantum KEM | sntrup761 | `KEM.sntrup761Keypair`, `sntrup761Enc`, `sntrup761Dec` | +| Hybrid DH + KEM secret | HKDF-SHA512 over `dh \|\| kemSecret` with a new info string, as in `CR.rootKdf` | `C.hkdf` | +| Authenticated encryption | XSalsa20-Poly1305 [SimpleX secretbox](#simplex-box-constructions) | `C.sbEncrypt`/`C.sbDecrypt`, `LC.sbEncryptTailTag` for streams | +| Authenticated encryption with a DH secret | XSalsa20-Poly1305 [SimpleX crypto_box](#simplex-box-constructions) | `C.cbEncrypt`/`C.cbDecrypt` | +| Forward-secret symmetric session | HKDF-SHA512 key chain | `C.sbcInit`, `C.sbcHkdf` | +| KDF | HKDF-SHA512 | `C.hkdf` | +| Hash commitment, derived identifier | SHA3-256 | `C.sha3_256` | +| Certificate fingerprint | SHA-256 X.509 fingerprint | `C.signedFingerprint`, `C.certificateFingerprint` | +| Randomness | ChaChaDRG seeded from system entropy | `C.newRandom`, `C.randomBytes` and the `random*` helpers | + +## Primitives + +| Primitive | Status | Purpose and protocol | Implementation | Lengths | +|---|---|---|---|---| +| Ed25519 | Default | SMP/NTF/XFTP queue and file keys (agent default `rcvAuthAlg`, `sndAuthAlg` in [Agent/Env/SQLite.hs](../src/Simplex/Messaging/Agent/Env/SQLite.hs)), short-link signatures and owner auth (`SL.encodeSign`, `SL.newOwnerAuth`), XRCP identity and session keys ([RemoteControl/Invitation.hs](../src/Simplex/RemoteControl/Invitation.hs)), agent service request signatures, client/service TLS certificates ([Transport/Credentials.hs](../src/Simplex/Messaging/Transport/Credentials.hs)) | crypton `Crypto.PubKey.Ed25519` via `C.sign'`/`C.verify'` | pub 32, priv 32, sig 64; X.509 SPKI 44 | +| Ed448 | Approved | Server CA and online certificates (default `signAlgorithm = ED448` in [Server/CLI.hs](../src/Simplex/Messaging/Server/CLI.hs)); accepted for SMP command signatures and TLS | crypton `Crypto.PubKey.Ed448`; server key generation by the `openssl genpkey` CLI | pub 57, priv 57, sig 114; SPKI 69 | +| X25519 | Default | Per-queue e2e and server-to-recipient `crypto_box` keys, SMP session DH, command authenticator (SMP v7+), proxy (PRXY/PFWD), notification tokens, XFTP chunk transport, XRCP hello and announcements | crypton `Crypto.PubKey.Curve25519` via `C.dh'` | pub 32, priv 32, secret 32; SPKI 44 | +| X448 | Restricted | E2E double ratchet only: X3DH and DH ratchet (`CR.RatchetX448`, E2E v3) | crypton `Crypto.PubKey.Curve448` | pub 56, priv 56, secret 56 | +| sntrup761 | Default | PQ KEM in double ratchet (E2E v3, agent v5+) and XRCP v1 hello | Vendored C [cbits/sntrup761.c](../cbits/sntrup761.c) via FFI; randomness from ChaChaDRG through `haskell_rng_func` ([Bindings/RNG.hs](../src/Simplex/Messaging/Crypto/SNTRUP761/Bindings/RNG.hs)) | pk 1158, sk 1763, ct 1039, shared 32 | +| XSalsa20-Poly1305, SimpleX crypto_box | Default | SMP message bodies (server to recipient), per-queue e2e envelope, confirmations, proxy forwarding, notifications, XRCP hello, XFTP chunk transport | crypton `Crypto.Cipher.XSalsa`, `Crypto.MAC.Poly1305`; `C.cryptoBox`, `LC.cbInit` | key 32 (DH secret), nonce 24, tag 16 | +| XSalsa20-Poly1305, SimpleX secretbox | Default | SMP transport block encryption (SMP v11+), short-link data (SMP v15+), XFTP file encryption, local file encryption ([Crypto/File.hs](../src/Simplex/Messaging/Crypto/File.hs)), XRCP session | same; `C.sbEncrypt`, `LC.sbEncryptTailTag`, `LC.sbInit` | key 32, nonce 24, tag 16 | +| crypto_box authenticator | Approved | Deniable sender command authorization with X25519 queue keys (SMP v7+); agent currently creates Ed25519 keys | `C.cbAuthenticate`, `C.cbVerify`: `crypto_box(sha512(msg))` | 80 = 64 + 16 | +| AES-256-GCM, 16-byte IV | Legacy-only | Double ratchet header and body (E2E v3, [pqdr.md](./pqdr.md)) | crypton `Crypto.Cipher.AES`, `Crypto.Cipher.Types`; `C.encryptAEAD`, `C.decryptAEAD`, `C.initAEAD`; J0 = GHASH(IV) as NIST SP 800-38D defines for non-96-bit IVs | key 32, IV 16, tag 16; padded header 88 (PQ off) or 2310 (PQ on), `CR.paddedHeaderLen` | +| AES-256-GCM, 12-byte IV | Restricted | WebRTC frame encryption in simplex-chat (`Simplex.Chat.Mobile.WebRTC`), for WebCrypto interoperability | `C.encryptAESNoPad`, `C.decryptAESNoPad`, `C.initAEADGCM`, `C.GCMIV` | key 32, IV 12, tag 16 | +| HKDF-SHA512 | Default | All KDFs listed in [Domain separation](#domain-separation-and-kdf-inputs) | crypton `Crypto.KDF.HKDF` with `SHA512`; `C.hkdf` (extract + expand) | output per use, at most 255 * 64 | +| SHA-256 | Default for X.509 fingerprints, otherwise Restricted | X.509 fingerprints (`C.KeyHash`, server identity, XRCP CA, service certificate hash, SMP v16+), XFTP chunk digests, agent message hashes, `requestCode`, ratchet key dedup hash, security code `codeAD = sha256(rcAD)` | crypton; `C.sha256Hash`, `LC.sha256Hash`, `getFingerprint ... HashSHA256` | 32 | +| SHA-512 | Restricted | XFTP file digests and file description hash, input of the crypto_box authenticator | crypton; `C.sha512Hash`, `LC.sha512Hash` | 64 | +| SHA-512 (inside sntrup761) | Restricted | sntrup761 internal hash | [cbits/sha512.c](../cbits/sha512.c) calls OpenSSL `SHA512()` from the system `libcrypto` (`extra-libraries: crypto`) | 64 | +| SHA3-256 | Default | Short-link key `sha3_256(fixedData)` (SMP v15+), XRCP hybrid secret, service request binding (agent v8) | crypton; `C.sha3_256`, `KEM.kemHybridSecret` | 32 | +| SHA3-384 | Restricted | Client-supplied sender ID: first 24 bytes of `sha3_384(corrId)`, computed by the agent (`prepareConnectionLink'`, `newRcvConnSrv`) and the server (`createQueue`) | crypton; `C.sha3_384` | 48, truncated to 24 | +| Keccak-256 | Restricted | Label hash for SMP name queries (`NameQuery NQHash`), must match the on-chain registry | crypton `Keccak_256`; `labelHash` in [SimplexName.hs](../src/Simplex/Messaging/SimplexName.hs) | 32 | +| MD5 | Legacy-only, non-security | XOR-aggregated queue ID hash `IdsHash` for service subscription reconciliation (SUBS, NSUBS, SOKS, ENDS) | crypton (`C.md5Hash`, `Protocol.queueIdHash`); SQLite UDF `simplex_xor_md5_combine` ([Agent/Store/SQLite.hs](../src/Simplex/Messaging/Agent/Store/SQLite.hs)); PostgreSQL pgcrypto `digest(..., 'md5')` in server and NTF schemas | 16 | +| ChaChaDRG | Default | Keys, nonces, correlation IDs, random IDs, sntrup761 randomness | crypton `Crypto.Random`; `C.newRandom` seeds with `drgNew` from system entropy | n/a | +| BBS+ over BLS12-381, SHA-256 suite | Restricted | Badge entitlement credentials and proofs ([Crypto/Entitlement.hs](../src/Simplex/Messaging/Crypto/Entitlement.hs)), XFTP storage-time extension | Vendored submodules libbbs and blst (C and assembly) via FFI; randomness from libbbs `getentropy`, `SecRandomCopyBytes` with the `commoncrypto` flag, `BCryptGenRandom` on Windows ([cbits/getentropy_win.c](../cbits/getentropy_win.c)) | sk 32, pk 96, sig 80, proof 272 + 32 per undisclosed message | +| ECDSA with SHA-256 (ES256) | Restricted | APNS provider JWT ([Push/APNS.hs](../src/Simplex/Messaging/Notifications/Server/Push/APNS.hs)) | crypton `Crypto.PubKey.ECC.ECDSA.sign`; key read by cryptostore `Crypto.Store.PKCS8`; curve taken from the key file | signature DER `SEQUENCE {r, s}`, base64url | +| TLS 1.3 / 1.2 | Restricted | SMP, XFTP, NTF, XRCP transport: `TLS_CHACHA20_POLY1305_SHA256` (1.3), `ECDHE_ECDSA_CHACHA20POLY1305_SHA256` (1.2), groups X448 and X25519, Ed448 and Ed25519 signatures (`defaultSupportedParams`) | `tls` 1.9 | n/a | +| TLS for browsers | Restricted | XFTP server with HTTPS credentials (`defaultSupportedParamsHTTPS`: `ciphersuite_strong`, FFDHE, P-521, ECDSA and RSA signatures); SMP server HTTPS requires RSA-4096 (`checkHTTPSCredentials`) | `tls`, `warp-tls` | n/a | +| X.509 | Restricted | Certificate chains, fingerprints, signed session keys (`C.signX509`, `C.verifyX509`) | `crypton-x509`, `crypton-x509-validation`, `cryptostore` | n/a | +| SQLCipher | Restricted | Agent SQLite database at rest (`PRAGMA key`); cipher parameters are SQLCipher defaults, not set in this repository | `direct-sqlcipher` (git dependency, [cabal.project](../cabal.project)) | n/a | + +## SimpleX box constructions + +`C.cryptoBox` and `LC.sbInit_` compute, for a 32-byte key `k` and 24-byte nonce `n`: + +``` +k1 = HSalsa20(k, 0^16) +subkey = HSalsa20(k1, n[0..16]) +stream = Salsa20(subkey, n[16..24]) +polyKey = stream[0..32] +ct = msg XOR stream[32..] +tag = Poly1305(polyKey, ct) +``` + +| Construction | Key `k` | Equivalent libsodium call | Layout | +|---|---|---|---| +| SimpleX crypto_box (`C.cbEncrypt`, `LC.cbInit`) | X25519 shared secret | `crypto_box_easy` (`crypto_box_beforenm` is `HSalsa20(dh, 0^16)`) | strict: `tag \|\| ct`; lazy tail-tag: `ct \|\| tag` | +| SimpleX secretbox (`C.sbEncrypt`, `LC.sbInit`, `KEM.kcb*`) | 32-byte symmetric key | `crypto_secretbox_easy` with key `crypto_core_hsalsa20(0^16, k)`, not with `k` | same | + +Padding before encryption: `C.pad` prefixes a 2-byte big-endian length and fills with `#` (message at most 65533 bytes); `LC.pad` prefixes an 8-byte length. `*NoPad` variants skip padding. + +## Domain separation and KDF inputs + +HKDF is `C.hkdf salt ikm info len` (HKDF-SHA512). + +| Info string | Salt | IKM | Output (split) | Function | Use | +|---|---|---|---|---|---| +| `"SimpleXX3DH"` | 64 zero bytes | `dh1 \|\| dh2 \|\| dh3 [\|\| kemShared]` | 96: `hk`, `nhk`, root key (32 each) | `CR.pqX3dh` | Ratchet initialization, added in E2E v2; KEM secret from E2E v3 (current minimum) | +| `"SimpleXVerifyCode"` | 64 zero bytes | same as `SimpleXX3DH` | 32: `rcVCPQ` | `CR.pqX3dh` | Verification code covering all handshake keys, new ratchets | +| `"SimpleXRootRatchet"` | root key | `dh [\|\| kemShared]` | 96: root key, chain key, next header key | `CR.rootKdf` | DH/PQ ratchet step | +| `"SimpleXChainRatchet"` | empty | chain key | 96: chain key, message key (32 each), message IV (16), header IV (16) | `CR.chainKdf` | Per-message keys | +| `"SimpleXSbChainInit"` | SMP session ID | X25519 session secret | 64: two 32-byte chain keys | `C.sbcInit` from `Transport.blockEncryption` | SMP transport block encryption, SMP v11+ | +| `"SimpleXSbChainInit"` | empty | XRCP hybrid secret (below) | 64: two 32-byte chain keys | `C.sbcInit` in [RemoteControl/Client.hs](../src/Simplex/RemoteControl/Client.hs) | XRCP v1 session | +| `"SimpleXSbChain"` | empty | chain key | 88: chain key (32), secretbox key (32), nonce (24) | `C.sbcHkdf` | Each SMP block and XRCP message | +| `"SimpleXContactLink"` | empty | link key (32) | 56: link ID (24), secretbox key (32) | `SL.contactShortLinkKdf` | Contact short links, SMP v15+ | +| `"SimpleXInvLink"` | empty | link key (32) | 32: secretbox key | `SL.invShortLinkKdf` | Invitation short links, SMP v15+ | + +Other derivations: + +| Value | Construction | Function | Use | +|---|---|---|---| +| Service request binding | `sha3_256("SimpleXService" \|\| rcAD)` | `serviceReqBinding` in [Agent.hs](../src/Simplex/Messaging/Agent.hs) | Agent RPC, agent v8 | +| BBS header | `"SimpleX badges v1"` | `entitlementBBSHeader` | Badge credentials | +| XRCP hybrid secret | `sha3_256(x25519Dh \|\| kemShared)` | `KEM.kemHybridSecret` | XRCP v1 | +| Short-link key | `sha3_256(smpEncode FixedLinkData)` | `SL.encodeSignFixedData`, checked in `SL.decryptLinkData` | SMP v15+ | +| Sender ID | `take 24 (sha3_384 corrId)` | `prepareConnectionLink'`, `newRcvConnSrv` in Agent.hs; `createQueue` in [Server.hs](../src/Simplex/Messaging/Server.hs) | Client-supplied sender ID with link data, SMP v15+; the server rejects a mismatch to prevent an ID oracle | +| Ratchet associated data | `pubKeyBytes sk1 \|\| pubKeyBytes rk1` (raw X448, 112) | `CR.pqX3dh` | AD of every ratchet AEAD | +| Security code | `sha256(rcAD)` | `ratchetVerifyCodes` in [AgentStore.hs](../src/Simplex/Messaging/Agent/Store/AgentStore.hs) | Connection verification | +| Contact request code | `sha256(smpEncode (k1, k2, kem, sndId))` | `requestCode` in Agent.hs | Contact request binding | +| Command authenticator | `crypto_box(sha512(authorized))` | `C.cbAuthenticate` | SMP v7+ | +| Queue IDs hash | `xor` of `md5(queueId)` | `Protocol.queueIdsHash` | Service subscriptions | + +`"SimpleXSbChainInit"` and `"SimpleXSbChain"` are shared by SMP block encryption and XRCP. Their outputs are separated only by the IKM (and the salt for `sbcInit`). + +## Nonces and IVs + +| Use | Construction | +|---|---| +| SMP command authenticator, proxied command | `C.cbNonce corrId`, where `corrId` is a random 24-byte nonce (`C.randomCbNonce`) | +| SMP proxied responses | `C.reverseNonce` of the request nonce | +| Server-to-recipient message body | `C.cbNonce msgId`; `msgIdBytes = 24` in [Server/Main.hs](../src/Simplex/Messaging/Server/Main.hs) | +| Per-queue e2e envelope, short-link data, notifications, XRCP hello, XFTP chunk download | Random 24-byte nonce, sent with the ciphertext | +| XFTP file, local encrypted file | Random key and nonce per file (`C.randomSbKey`, `C.randomCbNonce`) | +| SMP block, XRCP session messages | Key and nonce from `C.sbcHkdf`, one per block | +| Ratchet header, body | 16-byte IVs from `CR.chainKdf`; header IV is sent, body IV is not | +| WebRTC frames | 12-byte `C.GCMIV` supplied by the caller in simplex-chat | + +## Implementation sources + +| Source | Version or pin | Provides | +|---|---|---| +| crypton | 0.34 | Ed25519, Ed448, X25519, X448, XSalsa20, Poly1305, AES-GCM, HKDF, SHA-2, SHA-3, Keccak, MD5, ChaChaDRG, ECDSA | +| tls, crypton-x509, crypton-x509-validation, cryptostore | 1.9.0, 1.7.6, 1.6.12, 0.3.0.1 | TLS, X.509, PKCS#8 | +| [cbits/sntrup761.c](../cbits/sntrup761.c) | Copy of draft-josefsson-ntruprime-streamlined-00 ([cbits/README.md](../cbits/README.md)) | sntrup761 | +| [cbits/sha512.c](../cbits/sha512.c) and system OpenSSL `libcrypto` | system | SHA-512 for sntrup761 | +| `cbits/libbbs` submodule ([simplex-chat/libbbs](https://github.com/simplex-chat/libbbs)) | `59a0f4bf` | BBS+ (`bbs_sha256_ciphersuite`), SHA-256, SHAKE256 | +| `cbits/blst` submodule ([supranational/blst](https://github.com/supranational/blst)) | `db3defd0` | BLS12-381 (C and `build/assembly.S`, `-D__BLST_PORTABLE__`) | +| direct-sqlcipher | git `f814ee68` | SQLCipher | +| `openssl` CLI | system | Server CA and certificate key generation (`createServerX509_`) | + +## xftp-web (TypeScript) + +[xftp-web](../xftp-web/) reimplements a subset for the browser XFTP client. It has no AES-GCM, HKDF, SHA-3, MD5, X448 or sntrup761 code. + +| Primitive | Implementation | Haskell counterpart | +|---|---|---| +| Ed25519 sign, verify, keys | libsodium-wrappers-sumo (`crypto_sign_*`), [src/crypto/keys.ts](../xftp-web/src/crypto/keys.ts) | `C.sign'`, `C.verify'` | +| Ed448 verify | `@noble/curves/ed448`, keys.ts `verifyEd448` | `C.verify'` | +| X25519 | libsodium `crypto_scalarmult`, `crypto_box_keypair` | `C.dh'` | +| SimpleX crypto_box, secretbox, tail-tag streaming | Salsa20 block function written in TypeScript, libsodium `crypto_core_hsalsa20` and `crypto_onetimeauth_*`, [src/crypto/secretbox.ts](../xftp-web/src/crypto/secretbox.ts) | `C.cryptoBox`, `LC.sbInit_` | +| crypto_box authenticator | [src/protocol/client.ts](../xftp-web/src/protocol/client.ts) `cbAuthenticate`, `cbVerify` | `C.cbAuthenticate`, `C.cbVerify` | +| SHA-256, SHA-512 | libsodium `crypto_hash_sha256`, `crypto_hash_sha512*`, [src/crypto/digest.ts](../xftp-web/src/crypto/digest.ts) | `C.sha256Hash`, `C.sha512Hash` | +| Random | WebCrypto `crypto.getRandomValues`; libsodium for key generation | `C.randomBytes` | + +## Tests + +None of the tests below compares against published reference vectors (NIST, RFC 8032, RFC 7748, RFC 5869, NaCl, the sntrup761 draft or the BBS draft). Interoperability is tested between this repository's own implementations. + +| Area | Test module (hspec group) | Kind | +|---|---|---| +| Ed25519, Ed448, X25519 crypto_box, secretbox, lazy and tail-tag secretbox, AES-GCM 12-byte IV, X.509 key encoding, X.509 chains, sntrup761, BBS+, entitlements, padding | [tests/CoreTests/CryptoTests.hs](../tests/CoreTests/CryptoTests.hs) | Round-trip; fixed lengths for BBS+; fixed bytes for padding | +| Double ratchet (X25519 and X448), PQ KEM agreement, AES-GCM 16-byte IV | [tests/AgentTests/DoubleRatchetTests.hs](../tests/AgentTests/DoubleRatchetTests.hs) | Round-trip; decoding of a stored v2 ratchet JSON | +| Short links (HKDF info strings, SHA3-256 link key) | [tests/AgentTests/ShortLinkTests.hs](../tests/AgentTests/ShortLinkTests.hs) | Round-trip, tamper rejection | +| File encryption | [tests/CoreTests/CryptoFileTests.hs](../tests/CoreTests/CryptoFileTests.hs) | Round-trip | +| XRCP session (SHA3-256 hybrid, sb chain) | [tests/RemoteControl.hs](../tests/RemoteControl.hs) | End-to-end | +| SMP authenticators, block encryption, `IdsHash` (MD5) | [tests/ServerTests.hs](../tests/ServerTests.hs) | End-to-end; with the PostgreSQL queue store this checks Haskell MD5 against pgcrypto | +| Haskell and xftp-web: SHA-256/512, Ed25519, Ed448 identity proof, X25519, DER keys, crypto_box, secretbox, tail-tag, authenticator, file encryption | [tests/XFTPWebTests.hs](../tests/XFTPWebTests.hs) (`XFTP Web Client`) | Byte-identical cross-language output | diff --git a/protocol/pqdr.md b/protocol/pqdr.md index d3d3e2b48..1519a59c9 100644 --- a/protocol/pqdr.md +++ b/protocol/pqdr.md @@ -62,14 +62,16 @@ It is possible to reduce size overhead by using only one KEM agreement and makin ## Double ratchet with encrypted headers augmented with double PQ KEM -Algorithm below assumes that in addition to shared secret from the initial key agreement, there will be an encapsulation key available from the party that published its keys (Bob). +Algorithm below assumes that in addition to shared secret from the initial key agreement, there will be an encapsulation key available from the initiating party, that sent its keys in the connection invitation to the joining party (see [agent protocol](./agent-protocol.md)). Following the double ratchet specification, the pseudo-code below names the joining party Alice and the initiating party Bob. + +Unlike X3DH prekey bundles, these keys are not reusable keys published for any party to use. The initiating party generates two X448 key pairs and, optionally, a sntrup761 KEM key pair for each connection, sends the public keys in the invitation, and deletes the stored private keys once the ratchet is initialized; only the second X448 private key and the agreed KEM key pair remain in the ratchet state as its initial ratchet keys. The joining party generates its keys when joining and keeps only its KEM key pair, in the ratchet state. The exception is a contact address that publishes ratchet keys in its link data: all requesters use these keys until the address owner rotates them, and the owner keeps the private keys of a few recent generations. ### Initialization The double ratchet initialization is defined in pseudo-code. This pseudo-code is identical to Signal algorithm specification except for that parts that add post-quantum key agreement. ``` -// Alice obtained Bob's keys and initializes ratchet first +// Alice (joining party) received Bob's keys in the invitation and initializes ratchet first def RatchetInitAlicePQ2HE(state, SK, bob_dh_public_key, shared_hka, shared_nhkb, bob_pq_kem_encapsulation_key): state.DHRs = GENERATE_DH() state.DHRr = bob_dh_public_key @@ -89,7 +91,7 @@ def RatchetInitAlicePQ2HE(state, SK, bob_dh_public_key, shared_hka, shared_nhkb, state.HKr = None state.NHKr = shared_nhkb -// Bob initializes ratchet second, having received Alice's connection request +// Bob (initiating party) initializes ratchet second, having received Alice's first message def RatchetInitBobPQ2HE(state, SK, bob_dh_key_pair, shared_hka, shared_nhkb, bob_pq_kem_key_pair): state.DHRs = bob_dh_key_pair state.DHRr = None @@ -210,6 +212,8 @@ The outer envelope contains the encrypted header (used as associated data for bo The message body is encrypted with AES-256-GCM using the message key derived from the sending chain key (`KDF_CK`). The associated data for body encryption is the concatenation of the ratchet associated data and the encoded encrypted header. +`KDF_CK(CK)` is HKDF-SHA512 with empty salt, `CK` as input key material and info `"SimpleXChainRatchet"`, producing 96 bytes split into the next chain key (32 bytes), the message key (32 bytes), the message body IV (16 bytes, not transmitted) and `headerIV` (16 bytes). Both IVs are used as 16-byte AES-256-GCM IVs, not the 12-byte IVs recommended by NIST SP 800-38D, so the initial counter block is J0 = GHASH(IV || 0^64 || [128]_64) as defined there for non-96-bit IVs. WebCrypto and other conforming implementations compute it when given the full 16-byte IV; truncating the IV to 12 bytes produces different ciphertext. + ```abnf encRatchetMessage = versionedLength encMessageHeader msgAuthTag encMsgBody ; encMessageHeader is used as associated data for body decryption: AD = rcAD || encMessageHeader @@ -274,7 +278,7 @@ As SimpleX Messaging Protocol pads messages to a fixed size, using 16kb transpor Sharing the initial keys in case of SimpleX Chat it is equivalent to sharing the invitation link. As encapsulation key is large, it may be inconvenient to share it in the link in some contexts, e.g. when QR codes are used. -It is possible to postpone sharing the encapsulation key until the first message from Alice (confirmation message in SMP protocol), the party sending connection request. The upside here is that the invitation link size would not increase. The downside is that the user profile shared in this confirmation will not be encrypted with PQ-resistant algorithm. +It is possible to postpone sharing the encapsulation key until the first message from the joining party (confirmation message in SMP protocol). The upside here is that the invitation link size would not increase. The downside is that the user profile shared in this confirmation will not be encrypted with PQ-resistant algorithm. Another consideration is pairwise ratchets in groups. Key generation in sntrup761 is quite slow - on slow devices it can be as slow as 10-20 keys per second, so using this primitive in groups larger than 10-20 members would result in slow performance. diff --git a/protocol/security.md b/protocol/security.md index dabe78139..cf4db5dcc 100644 --- a/protocol/security.md +++ b/protocol/security.md @@ -15,7 +15,7 @@ This document describes the cryptographic primitives and threat model for the Si - [SimpleX Messaging Protocol router that proxies the messages to another SMP router](#simplex-messaging-protocol-router-that-proxies-the-messages-to-another-smp-router) - [An attacker who obtained Alice's (decrypted) chat database](#an-attacker-who-obtained-alices-decrypted-chat-database) - [A user's contact](#a-users-contact) - - [An attacker who observes Alice showing an introduction message to Bob](#an-attacker-who-observes-alice-showing-an-introduction-message-to-bob) + - [An attacker who observes the initiating party showing an introduction message to the joining party](#an-attacker-who-observes-the-initiating-party-showing-an-introduction-message-to-the-joining-party) - [An attacker with Internet access](#an-attacker-with-internet-access) @@ -37,6 +37,8 @@ This document describes the cryptographic primitives and threat model for the Si - AES-GCM AEAD cipher, - SHA512-based HKDF for key derivation. +All primitives in use, their lengths, domain-separation strings and the policy for new code are listed in the [cryptographic primitive registry](./crypto-registry.md). + ## Threat Model @@ -190,15 +192,15 @@ This document describes the cryptographic primitives and threat model for the Si - cannot collaborate with another of the user's contacts to confirm they are communicating with the same user. -### An attacker who observes Alice showing an introduction message to Bob +### An attacker who observes the initiating party showing an introduction message to the joining party *can:* -- Impersonate Bob to Alice. +- Impersonate the joining party to the initiating party. *cannot:* -- Impersonate Alice to Bob. +- Impersonate the initiating party to the joining party. ### An attacker with Internet access diff --git a/protocol/simplex-messaging.md b/protocol/simplex-messaging.md index c5e5d8489..684874ee5 100644 --- a/protocol/simplex-messaging.md +++ b/protocol/simplex-messaging.md @@ -86,7 +86,7 @@ It's designed with the focus on communication security and integrity, under the It is designed as a low level protocol for other application protocols to solve the problem of secure and private message transmission, making [MITM attack][1] very difficult at any part of the message transmission system. -This document describes SMP protocol version 22. Versions 1-5 are discontinued. The version history: +This document describes SMP protocol version 23. Versions 1-5 are discontinued. The version history: - v1: binary protocol encoding - v2: message flags (used to control notifications) @@ -109,6 +109,7 @@ This document describes SMP protocol version 22. Versions 1-5 are discontinued. - v20: public namespaces resolver (RSLV command, RNAME response) — direct or forwarded via PFWD - v21: server public information in handshake - v22: `RNAME` says whether a name can be registered, not only what it resolves to +- v23: version in nonces of forwarded commands, random nonces for forwarded responses ## Introduction @@ -1128,7 +1129,7 @@ The proxy router may respond with error response in case the destination router Sender can send `SKEY` and `SEND` commands via proxy after obtaining the session ID with `PRXY` command (see [Request proxied session](#request-proxied-session)). -Transmission sent to proxy router should use session ID as entity ID and use a random correlation ID of 24 bytes as a nonce for crypto_box encryption of transmission to the destination router. The random ephemeral X25519 key to encrypt transmission should be unique per command, and it should be combined with the key sent by the router in the handshake header to proxy and to the client in `PKEY` command. +Transmission sent to proxy router should use session ID as entity ID and use a random correlation ID of 24 bytes as a nonce for crypto_box encryption of transmission to the destination router. When `smpVersion` in `PFWD` is 23 or higher, the first 2 bytes of this nonce are XOR-ed with `smpVersion`. The random ephemeral X25519 key to encrypt transmission should be unique per command, and it should be combined with the key sent by the router in the handshake header to proxy and to the client in `PKEY` command. Encrypted transmission should use the received session ID from the connection between proxy router and destination router in the authorized body. @@ -1140,10 +1141,11 @@ commandKey = length x509encoded The proxy router will forward the encrypted transmission in `RFWD` command (see below). -Having received the `RRES` response from the destination router, proxy router will forward `PRES` response to the client. `PRES` response should use the same correlation ID as `PFWD` command. The destination router will use this correlation ID increased by 1 as a nonce for encryption of the response. +Having received the `RRES` response from the destination router, proxy router will forward `PRES` response to the client. `PRES` response should use the same correlation ID as `PFWD` command. The destination router will use this correlation ID increased by 1 as a nonce for encryption of the response. When `smpVersion` in `PFWD` is 23 or higher, the destination router uses a random nonce instead, and sends it before the encrypted response. ```abnf -proxyResponse = %s"PRES" SP +proxyResponse = %s"PRES" SP [responseNonce] +responseNonce = %s"0" / (%s"1" 24*24 OCTET) ; from v23 forwardedResponse = *OCTET ; client-encrypted SMP response, decrypted by client using per-command DH secret ``` @@ -1170,7 +1172,7 @@ The shared secret for encrypting transmission bodies between proxy router and de ```abnf -relayResponse = %s"RRES" SP +relayResponse = %s"RRES" SP [responseNonce] responseTransmission = fwdCorrId forwardedResponse ; fwdCorrId and forwardedResponse defined above in RFWD section ``` diff --git a/protocol/xrcp.md b/protocol/xrcp.md index a9df02f5a..9a2a532c3 100644 --- a/protocol/xrcp.md +++ b/protocol/xrcp.md @@ -71,7 +71,7 @@ The session invitation contains this data: Host application decrypts (except the first session) and validates the invitation: - Session signature is valid. -- Timestamp is within some window from the current time. +- Timestamp of a multicast announcement is not earlier than 3660 seconds before and not later than 3600 seconds after the current time of the host. The host ignores announcements outside of this interval and continues listening. - Long-term key signature is valid. - Long-term CA and signature key are the same as in the first session. - Some version in the offered range is supported. diff --git a/simplexmq.cabal b/simplexmq.cabal index f15bccff3..41d1b10da 100644 --- a/simplexmq.cabal +++ b/simplexmq.cabal @@ -195,6 +195,7 @@ library Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260712_address_dr_rpc Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260823_snd_files_entitlement Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes + Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260929_ratchet_indexes else exposed-modules: Simplex.Messaging.Agent.Store.SQLite @@ -250,6 +251,7 @@ library Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260712_address_dr_rpc Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260823_snd_files_entitlement Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes + Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260929_ratchet_indexes Simplex.Messaging.Agent.Store.SQLite.Util if flag(client_postgres) || flag(server_postgres) exposed-modules: diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs index be34deafb..067031899 100644 --- a/src/Simplex/Messaging/Agent.hs +++ b/src/Simplex/Messaging/Agent.hs @@ -2435,62 +2435,66 @@ enqueueMessageB c reqs = do cfg <- asks config (_, reqMids) <- unsafeWithStore c $ \db -> do mapAccumLM (\ids r -> storeSentMsg db cfg ids r `E.catchAny` \e -> (ids,) <$> handleInternal e) IM.empty reqs - forME reqMids $ \((csqs_, _, _, _), InternalId msgId, pqSecr) -> forM csqs_ $ \(cData, sq :| sqs) -> do - submitPendingMsg c sq - let sqs' = filter (isActiveSndQ cData) sqs - pure ((msgId, pqSecr), if null sqs' then Nothing else Just (sqs', msgId)) + forME reqMids $ submitSentMsg c where - storeSentMsg :: - DB.Connection -> - AgentConfig -> - IntMap (Maybe Int64, AMessage) -> - Either AgentErrorType (Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage) -> - IO (IntMap (Maybe Int64, AMessage), Either AgentErrorType ((Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage), InternalId, PQEncryption)) - storeSentMsg db cfg aMessageIds = \case - Left e -> pure (aMessageIds, Left e) - Right req@(csqs_, pqEnc_, msgFlags, mbr) -> case mbr of - VRValue i_ aMessage -> case i_ >>= (`IM.lookup` aMessageIds) of - Just _ -> pure (aMessageIds, Left $ INTERNAL "enqueueMessageB: storeSentMsg duplicate saved message body") - Nothing -> do - (mbId_, r) <- case csqs_ of - Left e -> pure (Nothing, Left e) - Right (cData, sq :| _) -> do - mbId <- createSndMsgBody db aMessage - (Just mbId,) <$> storeSentMsg_ cData sq mbId aMessage - let aMessageIds' = maybe id (`IM.insert` (mbId_, aMessage)) i_ aMessageIds - pure (aMessageIds', r) - VRRef i -> case csqs_ of - Left e -> pure $ (aMessageIds, Left e) - Right (cData, sq :| _) -> case IM.lookup i aMessageIds of - Just (Just mbId, aMessage) -> (aMessageIds,) <$> storeSentMsg_ cData sq mbId aMessage - Just (Nothing, aMessage) -> do - mbId <- createSndMsgBody db aMessage - let aMessageIds' = IM.insert i (Just mbId, aMessage) aMessageIds - (aMessageIds',) <$> storeSentMsg_ cData sq mbId aMessage - Nothing -> pure (aMessageIds, Left $ INTERNAL "enqueueMessageB: storeSentMsg missing saved message body id") - where - storeSentMsg_ cData@ConnData {connId} sq sndMsgBodyId aMessage = fmap (first storeError) $ runExceptT $ do - let AgentConfig {e2eEncryptVRange} = cfg - internalTs <- liftIO getCurrentTime - (internalId, internalSndId, prevMsgHash) <- ExceptT $ updateSndIds db connId - -- We need to do pre-flight encoding that is not stored in database - -- to calculate its hash and remember it on connection (createSndMsg -> updateSndMsgHash) - -- to enable next enqueue. - -- (As encoding is different per connection, we can't store shared body, so it's repeated on delivery) - let agentMsgStr = encodeAgentMsgStr aMessage internalSndId prevMsgHash - internalHash = C.sha256Hash agentMsgStr - currentE2EVersion = maxVersion e2eEncryptVRange - (mek, paddedLen, pqEnc) <- agentRatchetEncryptHeader db cData e2eEncAgentMsgLength pqEnc_ currentE2EVersion - withExceptT (SEAgentError . cryptoError) $ CR.rcCheckCanPad paddedLen agentMsgStr - let msgType = aMessageType aMessage - -- msgBody is empty, because snd_messages record is linked to snd_message_bodies - msgData = SndMsgData {internalId, internalSndId, internalTs, msgType, msgFlags, msgBody = "", pqEncryption = pqEnc, internalHash, prevMsgHash, sndMsgPrepData_ = Just SndMsgPrepData {encryptKey = mek, paddedLen, sndMsgBodyId}} - liftIO $ createSndMsg db connId msgData - liftIO $ createSndMsgDelivery db sq internalId - pure (req, internalId, pqEnc) handleInternal :: E.SomeException -> IO (Either AgentErrorType b) handleInternal = pure . Left . INTERNAL . show +submitSentMsg :: AgentClient -> ((Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage), InternalId, PQEncryption) -> AM' (Either AgentErrorType ((AgentMsgId, PQEncryption), Maybe ([SndQueue], AgentMsgId))) +submitSentMsg c ((csqs_, _, _, _), InternalId msgId, pqSecr) = forM csqs_ $ \(cData, sq :| sqs) -> do + submitPendingMsg c sq + let sqs' = filter (isActiveSndQ cData) sqs + pure ((msgId, pqSecr), if null sqs' then Nothing else Just (sqs', msgId)) + +storeSentMsg :: + DB.Connection -> + AgentConfig -> + IntMap (Maybe Int64, AMessage) -> + Either AgentErrorType (Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage) -> + IO (IntMap (Maybe Int64, AMessage), Either AgentErrorType ((Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage), InternalId, PQEncryption)) +storeSentMsg db cfg aMessageIds = \case + Left e -> pure (aMessageIds, Left e) + Right req@(csqs_, pqEnc_, msgFlags, mbr) -> case mbr of + VRValue i_ aMessage -> case i_ >>= (`IM.lookup` aMessageIds) of + Just _ -> pure (aMessageIds, Left $ INTERNAL "enqueueMessageB: storeSentMsg duplicate saved message body") + Nothing -> do + (mbId_, r) <- case csqs_ of + Left e -> pure (Nothing, Left e) + Right (cData, sq :| _) -> do + mbId <- createSndMsgBody db aMessage + (Just mbId,) <$> storeSentMsg_ cData sq mbId aMessage + let aMessageIds' = maybe id (`IM.insert` (mbId_, aMessage)) i_ aMessageIds + pure (aMessageIds', r) + VRRef i -> case csqs_ of + Left e -> pure $ (aMessageIds, Left e) + Right (cData, sq :| _) -> case IM.lookup i aMessageIds of + Just (Just mbId, aMessage) -> (aMessageIds,) <$> storeSentMsg_ cData sq mbId aMessage + Just (Nothing, aMessage) -> do + mbId <- createSndMsgBody db aMessage + let aMessageIds' = IM.insert i (Just mbId, aMessage) aMessageIds + (aMessageIds',) <$> storeSentMsg_ cData sq mbId aMessage + Nothing -> pure (aMessageIds, Left $ INTERNAL "enqueueMessageB: storeSentMsg missing saved message body id") + where + storeSentMsg_ cData@ConnData {connId} sq sndMsgBodyId aMessage = fmap (first storeError) $ runExceptT $ do + let AgentConfig {e2eEncryptVRange} = cfg + internalTs <- liftIO getCurrentTime + (internalId, internalSndId, prevMsgHash) <- ExceptT $ updateSndIds db connId + -- We need to do pre-flight encoding that is not stored in database + -- to calculate its hash and remember it on connection (createSndMsg -> updateSndMsgHash) + -- to enable next enqueue. + -- (As encoding is different per connection, we can't store shared body, so it's repeated on delivery) + let agentMsgStr = encodeAgentMsgStr aMessage internalSndId prevMsgHash + internalHash = C.sha256Hash agentMsgStr + currentE2EVersion = maxVersion e2eEncryptVRange + (mek, paddedLen, pqEnc) <- agentRatchetEncryptHeader db cData e2eEncAgentMsgLength pqEnc_ currentE2EVersion + withExceptT (SEAgentError . cryptoError) $ CR.rcCheckCanPad paddedLen agentMsgStr + let msgType = aMessageType aMessage + -- msgBody is empty, because snd_messages record is linked to snd_message_bodies + msgData = SndMsgData {internalId, internalSndId, internalTs, msgType, msgFlags, msgBody = "", pqEncryption = pqEnc, internalHash, prevMsgHash, sndMsgPrepData_ = Just SndMsgPrepData {encryptKey = mek, paddedLen, sndMsgBodyId}} + liftIO $ createSndMsg db connId msgData + liftIO $ createSndMsgDelivery db sq internalId + pure (req, internalId, pqEnc) + encodeAgentMsgStr :: AMessage -> InternalSndId -> PrevSndMsgHash -> ByteString encodeAgentMsgStr aMessage internalSndId prevMsgHash = do let privHeader = APrivHeader (unSndId internalSndId) prevMsgHash @@ -2865,15 +2869,18 @@ synchronizeRatchet' c connId pqSupport' force = withConnLock c connId "synchroni SomeConn _ (DuplexConnection cData@ConnData {pqSupport} rqs sqs) | ratchetSyncAllowed cData || force -> do -- check queues are not switching? - when (pqSupport' /= pqSupport) $ withStore' c $ \db -> setConnPQSupport db connId pqSupport' let cData' = cData {pqSupport = pqSupport'} :: ConnData - AgentConfig {e2eEncryptVRange} <- asks config + AgentConfig {e2eEncryptVRange, smpAgentVRange} <- asks config g <- asks random (pks, e2eParams) <- liftIO $ CR.generateRcvE2EParams g (maxVersion e2eEncryptVRange) pqSupport' - enqueueRatchetKeyMsgs c cData' sqs e2eParams - withStore' c $ \db -> do - setConnRatchetSync db connId RSStarted - setRatchetX3dhKeys db connId pks + msgId <- withStore c $ \db -> runExceptT $ do + msgId <- storeRatchetKey db (L.head sqs) (maxVersion smpAgentVRange) e2eParams Nothing + liftIO $ do + when (pqSupport' /= pqSupport) $ setConnPQSupport db connId pqSupport' + setConnRatchetSync db connId RSStarted + setRatchetX3dhKeys db connId pks + pure msgId + lift $ enqueueRatchetKeyMsgs c cData' sqs msgId let cData'' = cData' {ratchetSyncState = RSStarted} :: ConnData conn' = DuplexConnection cData'' rqs sqs connectionStats c conn' @@ -3404,6 +3411,9 @@ subscriber c@AgentClient {msgQ, subQ} = run $ forever $ do run a = a `catchOwn` \e -> notify $ CRITICAL True $ "Agent subscriber stopped: " <> show e notify err = atomically $ writeTBQueue subQ ("", "", AEvt SAEConn $ ERR err) +maxRatchetKeyHashes :: Int +maxRatchetKeyHashes = 100 + cleanupManager :: AgentClient -> AM' () cleanupManager c@AgentClient {subQ} = do AgentConfig {initialCleanupDelay, cleanupInterval = int, storedMsgDataTTL = ttl, cleanupBatchSize = limit} <- @@ -3413,7 +3423,7 @@ cleanupManager c@AgentClient {subQ} = do run ERR deleteConns run ERR $ withStore' c $ \db -> deleteRcvMsgHashesExpired db ttl limit run ERR $ withStore' c $ \db -> deleteSndMsgsExpired db ttl limit - run ERR $ withStore' c $ \db -> deleteRatchetKeyHashesExpired db ttl limit + run ERR $ withStore' c $ \db -> deleteRatchetKeyHashesExpired db ttl maxRatchetKeyHashes run ERR $ withStore' c (`deleteExpiredNtfTokensToDelete` ttl) run RFERR deleteRcvFilesExpired run RFERR deleteRcvFilesDeleted @@ -3610,9 +3620,9 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar _ -> prohibited "handshake: incorrect state" >> ack (Just e2eDh, Nothing) -> do decryptClientMessage e2eDh clientMsg >>= \case - (SMP.PHEmpty, AgentRatchetKey {agentVersion, e2eEncryption}) -> do + (SMP.PHEmpty, AgentRatchetKey {agentVersion, e2eEncryption, info}) -> do conn' <- updateConnVersion conn cData agentVersion - qDuplex conn' "AgentRatchetKey" $ \a -> newRatchetKey e2eEncryption a >> ack + qDuplex conn' "AgentRatchetKey" $ \a -> newRatchetKey e2eEncryption info a >> ack (SMP.PHEmpty, AgentMsgEnvelope {agentVersion, encAgentMessage}) -> do conn' <- updateConnVersion conn cData agentVersion -- primary queue is set as Active in helloMsg, below is to set additional queues Active @@ -3655,7 +3665,7 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar qDuplexAckDel conn'' name a = qDuplex conn'' name a >> ackDel msgId resetRatchetSync :: AM (Connection c) resetRatchetSync - | rss `notElem` ([RSOk, RSStarted] :: [RatchetSyncState]) = do + | rss `notElem` ([RSOk, RSStarted] :: [RatchetSyncState]) = ifM ((RSStarted ==) <$> getRatchetSyncState) (pure conn') $ do let cData'' = (toConnData conn') {ratchetSyncState = RSOk} :: ConnData conn'' = updateConnection cData'' conn' cStats <- connectionStats c conn'' @@ -3696,7 +3706,7 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar notifySync :: AM () notifySync = qDuplex conn' "AGENT A_CRYPTO error" $ \connDuplex -> do let rss' = cryptoErrToSyncState e - when (rss `elem` ([RSOk, RSAllowed, RSRequired] :: [RatchetSyncState])) $ do + whenM ((`elem` ([RSOk, RSAllowed, RSRequired] :: [RatchetSyncState])) <$> getRatchetSyncState) $ do let cData'' = (toConnData conn') {ratchetSyncState = rss'} :: ConnData conn'' = updateConnection cData'' connDuplex cStats <- connectionStats c conn'' @@ -3792,6 +3802,9 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar enqueueCmd :: InternalCommand -> AM () enqueueCmd = enqueueCommand c "" connId (Just srv) . AInternalCommand + getRatchetSyncState :: AM RatchetSyncState + getRatchetSyncState = withStore c (`getConnRatchetSync` connId) + unexpected :: BrokerMsg -> AM () unexpected r = do logServer "<--" c srv rId $ "unexpected: " <> bshow r @@ -4160,42 +4173,47 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar DuplexConnection {} -> action conn' _ -> qError $ name <> ": message must be sent to duplex connection" - newRatchetKey :: CR.RcvE2ERatchetParams 'C.X448 -> Connection 'CDuplex -> AM () - newRatchetKey e2eOtherPartyParams@(CR.E2ERatchetParams e2eVersion k1Rcv k2Rcv _) conn'@(DuplexConnection cData'@ConnData {lastExternalSndId, pqSupport} _ sqs) = + newRatchetKey :: CR.RcvE2ERatchetParams 'C.X448 -> ByteString -> Connection 'CDuplex -> AM () + newRatchetKey e2eOtherPartyParams@(CR.E2ERatchetParams e2eVersion k1Rcv k2Rcv _) info conn'@(DuplexConnection cData'@ConnData {lastExternalSndId, pqSupport} _ sqs) = unlessM ratchetExists $ do AgentConfig {e2eEncryptVRange} <- asks config unless (e2eVersion `isCompatible` e2eEncryptVRange) (throwE $ AGENT A_VERSION) - keys <- getSendRatchetKeys + keys_ <- getSendRatchetKeys let rcVs = CR.RatchetVersions {current = e2eVersion, maxSupported = maxVersion e2eEncryptVRange} - initRatchet rcVs keys - notifyAgreed + case keys_ of + Just (keys, replyKey_) -> initRatchet rcVs keys replyKey_ >> notifyAgreed + Nothing -> withStore' c $ \db -> addProcessedRatchetKeyHash db connId rkHashRcv where rkHashRcv = rkHash k1Rcv k2Rcv rkHash k1 k2 = C.sha256Hash $ C.pubKeyBytes k1 <> C.pubKeyBytes k2 + answeredKeyHash_ = case smpDecode info of + Right (AgentRatchetInfo RatchetInfo {answeredKeyHash}) -> answeredKeyHash + _ -> Nothing ratchetExists :: AM Bool ratchetExists = withStore' c $ \db -> do exists <- checkRatchetKeyHashExists db connId rkHashRcv - unless exists $ addProcessedRatchetKeyHash db connId rkHashRcv pure exists - getSendRatchetKeys :: AM (CR.RcvE2EPrivRatchetParams 'C.X448) - getSendRatchetKeys = case rss of - RSOk -> sendReplyKey -- receiving client - RSAllowed -> sendReplyKey - RSRequired -> sendReplyKey - RSStarted -> withStore c (`getRatchetX3dhKeys` connId) -- initiating client - RSAgreed -> do - withStore' c $ \db -> setConnRatchetSync db connId RSRequired + getSendRatchetKeys :: AM (Maybe (CR.RcvE2EPrivRatchetParams 'C.X448, Maybe (CR.RcvE2ERatchetParams 'C.X448))) + getSendRatchetKeys = getRatchetSyncState >>= \rss' -> case (rss', answeredKeyHash_) of + (RSStarted, Nothing) -> Just . (,Nothing) <$> withStore c (`getRatchetX3dhKeys` connId) -- initiating client + (RSStarted, Just h) -> fmap (,Nothing) . answeredKeys h <$> withStore c (`getRatchetX3dhKeys` connId) + (_, Nothing) -> Just <$> sendReplyKey -- receiving client + (RSAgreed, Just _) -> do + withStore' c $ \db -> addProcessedRatchetKeyHash db connId rkHashRcv >> setConnRatchetSync db connId RSRequired notifyRatchetSyncError -- can communicate for other client to reset to RSRequired -- - need to add new AgentMsgEnvelope, AgentMessage, AgentMessageType -- - need to deduplicate on receiving side throwE $ AGENT (A_CRYPTO RATCHET_SYNC) + (_, Just _) -> pure Nothing where + answeredKeys h keys@(pk1, pk2, _) + | rkHash (C.publicKey pk1) (C.publicKey pk2) == h = Just keys + | otherwise = Nothing sendReplyKey = do g <- asks random (pks, e2eParams) <- liftIO $ CR.generateRcvE2EParams g e2eVersion pqSupport - enqueueRatchetKeyMsgs c cData' sqs e2eParams - pure pks + pure (pks, Just e2eParams) notifyRatchetSyncError = do let cData'' = cData' {ratchetSyncState = RSRequired} :: ConnData conn'' = updateConnection cData'' conn' @@ -4207,23 +4225,33 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar conn'' = updateConnection cData'' conn' cStats <- connectionStats c conn'' notify $ RSYNC RSAgreed Nothing cStats - recreateRatchet :: CR.Ratchet 'C.X448 -> AM () - recreateRatchet rc = withStore' c $ \db -> do - setConnRatchetSync db connId RSAgreed - deleteRatchet db connId - createRatchet db connId rc + recreateRatchet :: Maybe (CR.RcvE2ERatchetParams 'C.X448) -> Maybe AMessage -> CR.Ratchet 'C.X448 -> AM () + recreateRatchet replyKey_ eready_ rc = do + cfg@AgentConfig {smpAgentVRange} <- asks config + (msgId_, r_) <- withStore c $ \db -> runExceptT $ do + msgId_ <- forM replyKey_ $ \e2eParams -> storeRatchetKey db (L.head sqs) (maxVersion smpAgentVRange) e2eParams (Just rkHashRcv) + liftIO $ do + addProcessedRatchetKeyHash db connId rkHashRcv + setConnRatchetSync db connId RSAgreed + deleteRatchet db connId + createRatchet db connId rc + r_ <- forM eready_ $ \msg -> liftIO $ storeSentMsg db cfg IM.empty (Right (Right (cData', sqs), Nothing, SMP.MsgFlags {notification = True}, vrValue msg)) >>= either E.throwIO pure . snd + pure (msgId_, r_) + forM_ msgId_ $ lift . enqueueRatchetKeyMsgs c cData' sqs + forM_ r_ $ \r -> lift $ do + r' <- submitSentMsg c r + enqueueSavedMessageB c $ mapMaybe snd $ rights [r'] -- compare public keys `k1` in AgentRatchetKey messages sent by self and other party -- to determine ratchet initilization ordering - initRatchet :: CR.RatchetVersions -> CR.RcvE2EPrivRatchetParams 'C.X448 -> AM () - initRatchet rcVs (pk1, pk2, pKem) + initRatchet :: CR.RatchetVersions -> CR.RcvE2EPrivRatchetParams 'C.X448 -> Maybe (CR.RcvE2ERatchetParams 'C.X448) -> AM () + initRatchet rcVs (pk1, pk2, pKem) replyKey_ | rkHash (C.publicKey pk1) (C.publicKey pk2) <= rkHashRcv = do rcParams <- liftError cryptoError $ CR.pqX3dhRcv (pk1, pk2, pKem) e2eOtherPartyParams - recreateRatchet $ CR.initRcvRatchet rcVs pk2 rcParams pqSupport + recreateRatchet replyKey_ Nothing $ CR.initRcvRatchet rcVs pk2 rcParams pqSupport | otherwise = do (_, rcDHRs) <- atomically . C.generateKeyPair =<< asks random rcParams <- liftEitherWith cryptoError $ CR.pqX3dhSnd (pk1, pk2, CR.APRKP CR.SRKSProposed <$> pKem) e2eOtherPartyParams - recreateRatchet $ CR.initSndRatchet rcVs k2Rcv rcDHRs rcParams - void . enqueueMessages' c cData' sqs SMP.MsgFlags {notification = True} $ EREADY lastExternalSndId + recreateRatchet replyKey_ (Just $ EREADY lastExternalSndId) $ CR.initSndRatchet rcVs k2Rcv rcDHRs rcParams checkMsgIntegrity :: PrevExternalSndId -> ExternalSndId -> PrevRcvMsgHash -> ByteString -> MsgIntegrity checkMsgIntegrity prevExtSndId extSndId internalPrevMsgHash receivedPrevMsgHash @@ -4336,32 +4364,25 @@ storeConfirmation c cData@ConnData {connId, pqSupport, connAgentVersion = v} sq liftIO $ createSndMsg db connId msgData liftIO $ createSndMsgDelivery db sq internalId -enqueueRatchetKeyMsgs :: AgentClient -> ConnData -> NonEmpty SndQueue -> CR.RcvE2ERatchetParams 'C.X448 -> AM () -enqueueRatchetKeyMsgs c cData (sq :| sqs) e2eEncryption = do - msgId <- enqueueRatchetKey c sq e2eEncryption - mapM_ (lift . enqueueSavedMessage c msgId) $ filter (isActiveSndQ cData) sqs +enqueueRatchetKeyMsgs :: AgentClient -> ConnData -> NonEmpty SndQueue -> InternalId -> AM' () +enqueueRatchetKeyMsgs c cData (sq :| sqs) (InternalId msgId) = do + submitPendingMsg c sq + mapM_ (enqueueSavedMessage c msgId) $ filter (isActiveSndQ cData) sqs -enqueueRatchetKey :: AgentClient -> SndQueue -> CR.RcvE2ERatchetParams 'C.X448 -> AM AgentMsgId -enqueueRatchetKey c sq@SndQueue {connId} e2eEncryption = do - aVRange <- asks $ smpAgentVRange . config - msgId <- storeRatchetKey $ maxVersion aVRange - lift $ submitPendingMsg c sq - pure $ unId msgId - where - storeRatchetKey :: VersionSMPA -> AM InternalId - storeRatchetKey agentVersion = withStore c $ \db -> runExceptT $ do - internalTs <- liftIO getCurrentTime - (internalId, internalSndId, prevMsgHash) <- ExceptT $ updateSndIds db connId - let agentMsg = AgentRatchetInfo "" - agentMsgStr = smpEncode agentMsg - internalHash = C.sha256Hash agentMsgStr - let msgBody = smpEncode $ AgentRatchetKey {agentVersion, e2eEncryption, info = agentMsgStr} - msgType = agentMessageType agentMsg - -- this message is e2e encrypted with queue key, not with double ratchet - msgData = SndMsgData {internalId, internalSndId, internalTs, msgType, msgBody, pqEncryption = PQEncOff, msgFlags = SMP.MsgFlags {notification = True}, internalHash, prevMsgHash, sndMsgPrepData_ = Nothing} - liftIO $ createSndMsg db connId msgData - liftIO $ createSndMsgDelivery db sq internalId - pure internalId +storeRatchetKey :: DB.Connection -> SndQueue -> VersionSMPA -> CR.RcvE2ERatchetParams 'C.X448 -> Maybe ByteString -> ExceptT StoreError IO InternalId +storeRatchetKey db sq@SndQueue {connId} agentVersion e2eEncryption answeredKeyHash_ = do + internalTs <- liftIO getCurrentTime + (internalId, internalSndId, prevMsgHash) <- ExceptT $ updateSndIds db connId + let agentMsg = AgentRatchetInfo RatchetInfo {answeredKeyHash = answeredKeyHash_} + agentMsgStr = smpEncode agentMsg + internalHash = C.sha256Hash agentMsgStr + let msgBody = smpEncode $ AgentRatchetKey {agentVersion, e2eEncryption, info = agentMsgStr} + msgType = agentMessageType agentMsg + -- this message is e2e encrypted with queue key, not with double ratchet + msgData = SndMsgData {internalId, internalSndId, internalTs, msgType, msgBody, pqEncryption = PQEncOff, msgFlags = SMP.MsgFlags {notification = True}, internalHash, prevMsgHash, sndMsgPrepData_ = Nothing} + liftIO $ createSndMsg db connId msgData + liftIO $ createSndMsgDelivery db sq internalId + pure internalId -- encoded AgentMessage -> encoded EncAgentMessage agentRatchetEncrypt :: DB.Connection -> ConnData -> ByteString -> (PQSupport -> Int) -> Maybe PQEncryption -> CR.VersionE2E -> ExceptT StoreError IO (ByteString, PQEncryption) @@ -4386,7 +4407,7 @@ agentRatchetDecrypt g db connId encAgentMsg = do agentRatchetDecrypt' :: TVar ChaChaDRG -> DB.Connection -> ConnId -> CR.RatchetX448 -> ByteString -> ExceptT StoreError IO (ByteString, PQEncryption) agentRatchetDecrypt' g db connId rc encAgentMsg = do - skipped <- liftIO $ getSkippedMsgKeys db connId + skipped <- liftIO $ getSkippedMsgKeys db connId CR.maxSkippedMsgKeys (agentMsgBody_, rc', skippedDiff) <- withExceptT (SEAgentError . cryptoError) $ CR.rcDecrypt g rc skipped encAgentMsg agentMsgBody <- liftEither $ first (SEAgentError . cryptoError) agentMsgBody_ liftIO $ updateRatchet db connId rc' skippedDiff diff --git a/src/Simplex/Messaging/Agent/Client.hs b/src/Simplex/Messaging/Agent/Client.hs index ed1935e87..655470385 100644 --- a/src/Simplex/Messaging/Agent/Client.hs +++ b/src/Simplex/Messaging/Agent/Client.hs @@ -1260,9 +1260,12 @@ sendOrProxySMPCommand c nm userId destSrv@ProtocolServer {host = destHosts} conn Left e -> throwE e ipAddressProtected :: NetworkConfig -> ProtocolServer p -> Bool -ipAddressProtected NetworkConfig {socksProxy, hostMode} (ProtocolServer _ hosts _ _) = do - isJust socksProxy || (hostMode == HMOnion && any isOnionHost hosts) +ipAddressProtected NetworkConfig {socksProxy, socksMode, hostMode} (ProtocolServer _ hosts _ _) + | isJust socksProxy = socksMode == SMAlways || if hostMode == HMPublic then allOnion else anyOnion + | otherwise = hostMode == HMOnion && anyOnion where + anyOnion = any isOnionHost hosts + allOnion = all isOnionHost hosts isOnionHost = \case THOnionHost _ -> True; _ -> False withNtfClient :: AgentClient -> NetworkRequestMode -> NtfServer -> EntityId -> ByteString -> (NtfClient -> ExceptT NtfClientError IO a) -> AM a diff --git a/src/Simplex/Messaging/Agent/Protocol.hs b/src/Simplex/Messaging/Agent/Protocol.hs index afe698769..5f7977e45 100644 --- a/src/Simplex/Messaging/Agent/Protocol.hs +++ b/src/Simplex/Messaging/Agent/Protocol.hs @@ -79,6 +79,7 @@ module Simplex.Messaging.Agent.Protocol SMPConfirmation (..), AgentMsgEnvelope (..), AgentMessage (..), + RatchetInfo (..), RequestSignature (..), AgentMessageType (..), APrivHeader (..), @@ -928,7 +929,7 @@ data AgentMessage | -- AgentConnInfoReply is used by accepting party in duplexHandshake mode (v2), allowing to include reply queue(s) in the initial confirmation. -- It made removed REPLY message unnecessary. AgentConnInfoReply (NonEmpty SMPQueueInfo) ConnInfo - | AgentRatchetInfo ByteString + | AgentRatchetInfo RatchetInfo | AgentMessage APrivHeader AMessage | AgentServiceRequest (NonEmpty SMPQueueInfo) (Maybe RequestSignature) MsgBody | AgentServiceResponse MsgBody @@ -939,7 +940,7 @@ instance Encoding AgentMessage where smpEncode = \case AgentConnInfo cInfo -> smpEncode ('I', Tail cInfo) AgentConnInfoReply smpQueues cInfo -> smpEncode ('D', smpQueues, Tail cInfo) -- 'D' stands for "duplex" - AgentRatchetInfo info -> smpEncode ('R', Tail info) + AgentRatchetInfo info -> smpEncode ('R', info) AgentMessage hdr aMsg -> smpEncode ('M', hdr, aMsg) AgentServiceRequest qs sig_ body -> smpEncode ('A', qs, sig_, Tail body) AgentServiceResponse body -> smpEncode ('P', Tail body) @@ -948,13 +949,23 @@ instance Encoding AgentMessage where smpP >>= \case 'I' -> AgentConnInfo . unTail <$> smpP 'D' -> AgentConnInfoReply <$> smpP <*> (unTail <$> smpP) - 'R' -> AgentRatchetInfo . unTail <$> smpP + 'R' -> AgentRatchetInfo <$> smpP 'M' -> AgentMessage <$> smpP <*> smpP 'A' -> AgentServiceRequest <$> smpP <*> smpP <*> (unTail <$> smpP) 'P' -> AgentServiceResponse . unTail <$> smpP 'J' -> AgentRejection . unTail <$> smpP _ -> fail "bad AgentMessage" +data RatchetInfo = RatchetInfo {answeredKeyHash :: Maybe ByteString} + deriving (Eq, Show) + +instance Encoding RatchetInfo where + smpEncode RatchetInfo {answeredKeyHash} = smpEncode answeredKeyHash + smpP = do + answeredKeyHash <- smpP <|> pure Nothing + _ <- A.takeByteString + pure RatchetInfo {answeredKeyHash} + -- internal type for storing message type in the database data AgentMessageType = AM_CONN_INFO diff --git a/src/Simplex/Messaging/Agent/Store/AgentStore.hs b/src/Simplex/Messaging/Agent/Store/AgentStore.hs index 311cdc14e..21c087d46 100644 --- a/src/Simplex/Messaging/Agent/Store/AgentStore.hs +++ b/src/Simplex/Messaging/Agent/Store/AgentStore.hs @@ -76,6 +76,7 @@ module Simplex.Messaging.Agent.Store.AgentStore getExpiredServiceConns, deleteExpiredServiceRequests, getDeletedWaitingDeliveryConnIds, + getConnRatchetSync, setConnRatchetSync, addProcessedRatchetKeyHash, checkRatchetKeyHashExists, @@ -1594,12 +1595,16 @@ getRatchet_ q db connId = where ratchet = maybe (Left SERatchetNotFound) Right . fromOnly -getSkippedMsgKeys :: DB.Connection -> ConnId -> IO SkippedMsgKeys -getSkippedMsgKeys db connId = - skipped <$> DB.query db "SELECT header_key, msg_n, msg_key FROM skipped_messages WHERE conn_id = ?" (Only connId) +getSkippedMsgKeys :: DB.Connection -> ConnId -> Int -> IO SkippedMsgKeys +getSkippedMsgKeys db connId maxKeys = do + (keys, oldKeys) <- splitAt maxKeys <$> DB.query db "SELECT skipped_message_id, header_key, msg_n, msg_key FROM skipped_messages WHERE conn_id = ? ORDER BY skipped_message_id DESC LIMIT ?" (connId, maxKeys + 1) + case oldKeys of + (skippedMsgId :: Int64, _, _, _) : _ -> DB.execute db "DELETE FROM skipped_messages WHERE conn_id = ? AND skipped_message_id <= ?" (connId, skippedMsgId) + [] -> pure () + pure $ skipped keys where skipped = foldl' addSkippedKey M.empty - addSkippedKey smks (hk, msgN, mk) = M.alter (Just . addMsgKey) hk smks + addSkippedKey smks (_, hk, msgN, mk) = M.alter (Just . addMsgKey) hk smks where addMsgKey = maybe (M.singleton msgN mk) (M.insert msgN mk) @@ -2793,6 +2798,11 @@ getDeletedWaitingDeliveryConnIds :: DB.Connection -> IO [ConnId] getDeletedWaitingDeliveryConnIds db = map fromOnly <$> DB.query_ db "SELECT conn_id FROM connections WHERE deleted_at_wait_delivery IS NOT NULL" +getConnRatchetSync :: DB.Connection -> ConnId -> IO (Either StoreError RatchetSyncState) +getConnRatchetSync db connId = + firstRow fromOnly SEConnNotFound $ + DB.query db "SELECT ratchet_sync_state FROM connections WHERE conn_id = ?" (Only connId) + setConnRatchetSync :: DB.Connection -> ConnId -> RatchetSyncState -> IO () setConnRatchetSync db connId ratchetSyncState = DB.execute db "UPDATE connections SET ratchet_sync_state = ? WHERE conn_id = ?" (ratchetSyncState, connId) @@ -2814,21 +2824,35 @@ checkRatchetKeyHashExists db connId hash = (connId, Binary hash) deleteRatchetKeyHashesExpired :: DB.Connection -> NominalDiffTime -> Int -> IO () -deleteRatchetKeyHashesExpired db ttl limit = do +deleteRatchetKeyHashesExpired db ttl maxConnHashes = do cutoffTs <- addUTCTime (-ttl) <$> getCurrentTime +#if defined(dbPostgres) DB.execute db - [sql| - DELETE FROM processed_ratchet_key_hashes - WHERE processed_ratchet_key_hash_id IN ( - SELECT processed_ratchet_key_hash_id - FROM processed_ratchet_key_hashes - WHERE created_at < ? - ORDER BY created_at ASC - LIMIT ? - ) - |] - (cutoffTs, limit) + ("DELETE FROM processed_ratchet_key_hashes h USING (" <> maxExcessIdsQuery <> ") e WHERE h.conn_id = e.conn_id AND h.processed_ratchet_key_hash_id <= e.max_excess_id AND h.created_at < ?") + (maxConnHashes, maxConnHashes, cutoffTs) +#else + maxExcessIds <- DB.query db maxExcessIdsQuery (maxConnHashes, maxConnHashes) + DB.executeMany + db + "DELETE FROM processed_ratchet_key_hashes WHERE conn_id = ? AND processed_ratchet_key_hash_id <= ? AND created_at < ?" + (map (\(connId :: ConnId, maxExcessId :: Int64) -> (connId, maxExcessId, cutoffTs)) maxExcessIds) +#endif + where + maxExcessIdsQuery :: Query + maxExcessIdsQuery = + [sql| + SELECT conn_id, ( + SELECT processed_ratchet_key_hash_id + FROM processed_ratchet_key_hashes + WHERE conn_id = c.conn_id + ORDER BY processed_ratchet_key_hash_id DESC + LIMIT 1 OFFSET ? + ) AS max_excess_id + FROM processed_ratchet_key_hashes c + GROUP BY conn_id + HAVING COUNT(*) > ? + |] -- | returns all connection queues, the first queue is the primary one getRcvQueuesByConnId_ :: DB.Connection -> ConnId -> IO (Maybe (NonEmpty RcvQueue)) diff --git a/src/Simplex/Messaging/Agent/Store/Postgres/Migrations/App.hs b/src/Simplex/Messaging/Agent/Store/Postgres/Migrations/App.hs index f34b120e2..26c7a80da 100644 --- a/src/Simplex/Messaging/Agent/Store/Postgres/Migrations/App.hs +++ b/src/Simplex/Messaging/Agent/Store/Postgres/Migrations/App.hs @@ -16,6 +16,7 @@ import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260411_service_certs import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260712_address_dr_rpc import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260823_snd_files_entitlement import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes +import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260929_ratchet_indexes import Simplex.Messaging.Agent.Store.Shared (Migration (..)) schemaMigrations :: [(String, Text, Maybe Text)] @@ -31,7 +32,8 @@ schemaMigrations = ("20260411_service_certs", m20260411_service_certs, Just down_m20260411_service_certs), ("20260712_address_dr_rpc", m20260712_address_dr_rpc, Just down_m20260712_address_dr_rpc), ("20260823_snd_files_entitlement", m20260823_snd_files_entitlement, Just down_m20260823_snd_files_entitlement), - ("20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes) + ("20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes), + ("20260929_ratchet_indexes", m20260929_ratchet_indexes, Just down_m20260929_ratchet_indexes) ] -- | The list of migrations in ascending order by date diff --git a/src/Simplex/Messaging/Agent/Store/Postgres/Migrations/M20260929_ratchet_indexes.hs b/src/Simplex/Messaging/Agent/Store/Postgres/Migrations/M20260929_ratchet_indexes.hs new file mode 100644 index 000000000..511064cf5 --- /dev/null +++ b/src/Simplex/Messaging/Agent/Store/Postgres/Migrations/M20260929_ratchet_indexes.hs @@ -0,0 +1,34 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE QuasiQuotes #-} + +module Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260929_ratchet_indexes where + +import Data.Text (Text) +import Text.RawString.QQ (r) + +m20260929_ratchet_indexes :: Text +m20260929_ratchet_indexes = + [r| +DROP INDEX idx_skipped_messages_conn_id; +CREATE INDEX idx_skipped_messages_conn_id ON skipped_messages(conn_id, skipped_message_id); + +DELETE FROM processed_ratchet_key_hashes AS h +WHERE EXISTS ( + SELECT 1 FROM processed_ratchet_key_hashes d + WHERE d.conn_id = h.conn_id AND d.hash = h.hash AND d.processed_ratchet_key_hash_id < h.processed_ratchet_key_hash_id +); + +DROP INDEX idx_processed_ratchet_key_hashes_hash; +CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash); +CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON processed_ratchet_key_hashes(conn_id, processed_ratchet_key_hash_id); +|] + +down_m20260929_ratchet_indexes :: Text +down_m20260929_ratchet_indexes = + [r| +DROP INDEX idx_processed_ratchet_key_hashes_conn_id; +DROP INDEX idx_processed_ratchet_key_hashes_hash; +CREATE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash); +DROP INDEX idx_skipped_messages_conn_id; +CREATE INDEX idx_skipped_messages_conn_id ON skipped_messages(conn_id); +|] diff --git a/src/Simplex/Messaging/Agent/Store/Postgres/Migrations/agent_postgres_schema.sql b/src/Simplex/Messaging/Agent/Store/Postgres/Migrations/agent_postgres_schema.sql index a0d408c36..55a9ab78d 100644 --- a/src/Simplex/Messaging/Agent/Store/Postgres/Migrations/agent_postgres_schema.sql +++ b/src/Simplex/Messaging/Agent/Store/Postgres/Migrations/agent_postgres_schema.sql @@ -1169,11 +1169,15 @@ CREATE INDEX idx_ntf_tokens_ntf_host_ntf_port ON smp_agent_test_protocol_schema. +CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (conn_id, processed_ratchet_key_hash_id); + + + CREATE INDEX idx_processed_ratchet_key_hashes_created_at ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (created_at); -CREATE INDEX idx_processed_ratchet_key_hashes_hash ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (conn_id, hash); +CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (conn_id, hash); @@ -1241,7 +1245,7 @@ CREATE UNIQUE INDEX idx_server_certs_user_id_host_port ON smp_agent_test_protoco -CREATE INDEX idx_skipped_messages_conn_id ON smp_agent_test_protocol_schema.skipped_messages USING btree (conn_id); +CREATE INDEX idx_skipped_messages_conn_id ON smp_agent_test_protocol_schema.skipped_messages USING btree (conn_id, skipped_message_id); diff --git a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/App.hs b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/App.hs index e50d8bd43..bae3b6afc 100644 --- a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/App.hs +++ b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/App.hs @@ -52,6 +52,7 @@ import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260411_service_certs import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260712_address_dr_rpc import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260823_snd_files_entitlement import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes +import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260929_ratchet_indexes import Simplex.Messaging.Agent.Store.Shared (Migration (..)) schemaMigrations :: [(String, Query, Maybe Query)] @@ -103,7 +104,8 @@ schemaMigrations = ("m20260411_service_certs", m20260411_service_certs, Just down_m20260411_service_certs), ("m20260712_address_dr_rpc", m20260712_address_dr_rpc, Just down_m20260712_address_dr_rpc), ("m20260823_snd_files_entitlement", m20260823_snd_files_entitlement, Just down_m20260823_snd_files_entitlement), - ("m20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes) + ("m20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes), + ("m20260929_ratchet_indexes", m20260929_ratchet_indexes, Just down_m20260929_ratchet_indexes) ] -- | The list of migrations in ascending order by date diff --git a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20260929_ratchet_indexes.hs b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20260929_ratchet_indexes.hs new file mode 100644 index 000000000..a1e70a2e2 --- /dev/null +++ b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20260929_ratchet_indexes.hs @@ -0,0 +1,29 @@ +{-# LANGUAGE QuasiQuotes #-} + +module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260929_ratchet_indexes where + +import Database.SQLite.Simple (Query) +import Database.SQLite.Simple.QQ (sql) + +-- idx_skipped_messages_conn_id is not changed: SQLite index entries include rowid (skipped_message_id) +m20260929_ratchet_indexes :: Query +m20260929_ratchet_indexes = + [sql| +DELETE FROM processed_ratchet_key_hashes AS h +WHERE EXISTS ( + SELECT 1 FROM processed_ratchet_key_hashes d + WHERE d.conn_id = h.conn_id AND d.hash = h.hash AND d.processed_ratchet_key_hash_id < h.processed_ratchet_key_hash_id +); + +DROP INDEX idx_processed_ratchet_key_hashes_hash; +CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash); +CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON processed_ratchet_key_hashes(conn_id, processed_ratchet_key_hash_id); + |] + +down_m20260929_ratchet_indexes :: Query +down_m20260929_ratchet_indexes = + [sql| +DROP INDEX idx_processed_ratchet_key_hashes_conn_id; +DROP INDEX idx_processed_ratchet_key_hashes_hash; +CREATE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash); + |] diff --git a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql index de0c1494b..e118a17eb 100644 --- a/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql +++ b/src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql @@ -573,10 +573,6 @@ CREATE INDEX idx_encrypted_rcv_message_hashes_hash ON encrypted_rcv_message_hash conn_id, hash ); -CREATE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes( - conn_id, - hash -); CREATE INDEX idx_snd_messages_rcpt_internal_id ON snd_messages( conn_id, rcpt_internal_id @@ -641,6 +637,14 @@ CREATE INDEX idx_connections_deleted ON connections(deleted); CREATE INDEX idx_connections_service_request_expires_at ON connections( service_request_expires_at ); +CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes( + conn_id, + hash +); +CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON processed_ratchet_key_hashes( + conn_id, + processed_ratchet_key_hash_id +); CREATE TRIGGER tr_rcv_queue_insert AFTER INSERT ON rcv_queues FOR EACH ROW diff --git a/src/Simplex/Messaging/Client.hs b/src/Simplex/Messaging/Client.hs index f8f1a4cb9..7a2eedeb9 100644 --- a/src/Simplex/Messaging/Client.hs +++ b/src/Simplex/Messaging/Client.hs @@ -195,6 +195,7 @@ data PClient v err msg = PClient transportHost :: TransportHost, tcpConnectTimeout :: NetworkTimeout, tcpTimeout :: NetworkTimeout, + proxiedRelayVRange :: VersionRange v, sendPings :: TVar Bool, lastReceived :: TVar UTCTime, timeoutErrorCount :: TVar Int, @@ -240,6 +241,7 @@ smpClientStub g sessionId thVersion thAuth = do transportHost = "localhost", tcpConnectTimeout, tcpTimeout, + proxiedRelayVRange = supportedClientSMPRelayVRange, sendPings, lastReceived, timeoutErrorCount, @@ -477,6 +479,7 @@ data ProtocolClientConfig v = ProtocolClientConfig serviceCredentials :: Maybe ServiceCredentials, -- | client-server protocol version range serverVRange :: VersionRange v, + proxiedRelayVRange :: VersionRange v, -- | agree shared session secret (used in SMP proxy for additional encryption layer) agreeSecret :: Bool, -- | Whether connecting client is a proxy server. See comment in ClientHandshake @@ -495,6 +498,7 @@ defaultClientConfig clientALPN useSNI serverVRange = clientALPN, serviceCredentials = Nothing, serverVRange, + proxiedRelayVRange = serverVRange, agreeSecret = False, proxyServer = False, useSNI @@ -505,6 +509,7 @@ defaultSMPClientConfig :: ProtocolClientConfig SMPVersion defaultSMPClientConfig = (defaultClientConfig (Just alpnSupportedSMPHandshakes) False supportedClientSMPRelayVRange) { defaultTransport = (show defaultSMPPort, transport @TLS), + proxiedRelayVRange = supportedClientSMPRelayVRange, agreeSecret = True } {-# INLINE defaultSMPClientConfig #-} @@ -568,7 +573,7 @@ type SMPTransportSession = TransportSession BrokerMsg -- A single queue can be used for multiple 'SMPClient' instances, -- as 'SMPServerTransmission' includes server information. getProtocolClient :: forall v err msg. Protocol v err msg => TVar ChaChaDRG -> NetworkRequestMode -> TransportSession msg -> ProtocolClientConfig v -> [HostName] -> Maybe (TBQueue (ServerTransmissionBatch v err msg)) -> UTCTime -> (ProtocolClient v err msg -> IO ()) -> IO (Either (ProtocolClientError err) (ProtocolClient v err msg)) -getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qSize, networkConfig, clientALPN, serviceCredentials, serverVRange, agreeSecret, proxyServer, useSNI} presetDomains msgQ proxySessTs disconnected = do +getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qSize, networkConfig, clientALPN, serviceCredentials, serverVRange, proxiedRelayVRange, agreeSecret, proxyServer, useSNI} presetDomains msgQ proxySessTs disconnected = do case chooseTransportHost networkConfig (host srv) of Right useHost -> (getCurrentTime >>= mkProtocolClient useHost >>= runClient useTransport useHost) @@ -593,6 +598,7 @@ getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qS transportHost, tcpConnectTimeout, tcpTimeout, + proxiedRelayVRange, sendPings, lastReceived, timeoutErrorCount, @@ -1115,10 +1121,10 @@ deleteSMPQueues = okSMPCommands DEL -- send PRXY :: SMPServer -> Maybe BasicAuth -> Command Sender -- receives PKEY :: SessionId -> X.CertificateChain -> X.SignedExact X.PubKey -> BrokerMsg connectSMPProxiedRelay :: SMPClient -> NetworkRequestMode -> SMPServer -> Maybe BasicAuth -> ExceptT SMPClientError IO ProxiedRelay -connectSMPProxiedRelay c@ProtocolClient {client_ = PClient {tcpConnectTimeout, tcpTimeout}} nm relayServ@ProtocolServer {port = relayPort, keyHash = C.KeyHash kh} proxyAuth = +connectSMPProxiedRelay c@ProtocolClient {client_ = PClient {tcpConnectTimeout, tcpTimeout, proxiedRelayVRange}} nm relayServ@ProtocolServer {port = relayPort, keyHash = C.KeyHash kh} proxyAuth = sendProtocolCommand_ c nm Nothing tOut Nothing NoEntity (Cmd SProxiedClient (PRXY relayServ proxyAuth)) >>= \case PKEY sId vr (CertChainPubKey chain key) -> - case supportedClientSMPRelayVRange `compatibleVersion` vr of + case proxiedRelayVRange `compatibleVersion` vr of Nothing -> throwE $ transportErr TEVersion Just (Compatible v) -> do relayKey <- liftEitherWith (const $ transportErr $ TEHandshake IDENTITY) =<< liftIO (runExceptT $ validateRelay chain key) @@ -1169,7 +1175,7 @@ instance StrEncoding ProxyClientError where -- consider how to process slow responses - is it handled somehow locally or delegated to the caller -- this method is used in the client -- sends PFWD :: C.PublicKeyX25519 -> EncTransmission -> Command Sender --- receives PRES :: EncResponse -> BrokerMsg -- proxy to client +-- receives PRES :: Maybe C.CbNonce -> EncResponse -> BrokerMsg -- proxy to client -- When client sends message via proxy, there may be one successful scenario and 9 error scenarios -- as shown below (WTF stands for unexpected response, ??? for response that failed to parse). @@ -1228,14 +1234,14 @@ proxySMPCommand c@ProtocolClient {thParams = proxyThParams, client_ = PClient {c TBError e _ : _ -> throwE $ PCETransportError e TBTransmission s _ : _ -> pure s TBTransmissions s _ _ : _ -> pure s - et <- liftEitherWith PCECryptoError $ EncTransmission <$> C.cbEncrypt cmdSecret nonce b paddedProxiedTLength + et <- liftEitherWith PCECryptoError $ EncTransmission <$> C.cbEncrypt cmdSecret (encTransmissionNonce v nonce) b paddedProxiedTLength -- proxy interaction errors are wrapped let tOut = Just $ 2 * netTimeoutInt tcpTimeout nm tryE (sendProtocolCommand_ c nm (Just nonce) tOut Nothing (EntityId sessionId) (Cmd SProxiedClient (PFWD v cmdPubKey et))) >>= \case Right r -> case r of - PRES (EncResponse er) -> do + PRES nonce_ (EncResponse er) -> do -- server interaction errors are thrown directly - t' <- liftEitherWith PCECryptoError $ C.cbDecrypt cmdSecret (C.reverseNonce nonce) er + t' <- liftEitherWith PCECryptoError $ C.cbDecrypt cmdSecret (fromMaybe (C.reverseNonce nonce) nonce_) er case tParse serverThParams t' of t'' :| [] -> case tDecodeClient serverThParams t'' of (_, _, cmd) -> case cmd of @@ -1253,10 +1259,10 @@ proxySMPCommand c@ProtocolClient {thParams = proxyThParams, client_ = PClient {c -- this method is used in the proxy -- sends RFWD :: EncFwdTransmission -> Command Sender --- receives RRES :: EncFwdResponse -> BrokerMsg +-- receives RRES :: Maybe C.CbNonce -> EncFwdResponse -> BrokerMsg -- proxy should send PRES to the client with EncResponse -- Always uses background timeout mode -forwardSMPTransmission :: SMPClient -> CorrId -> VersionSMP -> C.PublicKeyX25519 -> EncTransmission -> ExceptT SMPClientError IO EncResponse +forwardSMPTransmission :: SMPClient -> CorrId -> VersionSMP -> C.PublicKeyX25519 -> EncTransmission -> ExceptT SMPClientError IO (Maybe C.CbNonce, EncResponse) forwardSMPTransmission c@ProtocolClient {thParams, client_ = PClient {clientCorrId = g}} fwdCorrId fwdVersion fwdKey fwdTransmission = do -- prepare params sessSecret <- case thAuth thParams of @@ -1268,11 +1274,11 @@ forwardSMPTransmission c@ProtocolClient {thParams, client_ = PClient {clientCorr eft = EncFwdTransmission $ C.cbEncryptNoPad sessSecret nonce (smpEncode fwdT) -- send sendProtocolCommand_ c NRMBackground (Just nonce) Nothing Nothing NoEntity (Cmd SProxyService (RFWD eft)) >>= \case - RRES (EncFwdResponse efr) -> do + RRES nonce_ (EncFwdResponse efr) -> do -- unwrap r' <- liftEitherWith PCECryptoError $ C.cbDecryptNoPad sessSecret (C.reverseNonce nonce) efr FwdResponse {fwdCorrId = _, fwdResponse} <- liftEitherWith (const $ PCEResponseError BLOCK) $ smpDecode r' - pure fwdResponse + pure (nonce_, fwdResponse) r -> throwE $ unexpectedResponse r -- get queue information - always sent interactively diff --git a/src/Simplex/Messaging/Crypto.hs b/src/Simplex/Messaging/Crypto.hs index 0bc5238c4..a22833976 100644 --- a/src/Simplex/Messaging/Crypto.hs +++ b/src/Simplex/Messaging/Crypto.hs @@ -931,6 +931,8 @@ data CryptoError CERatchetEarlierMessage Word32 | -- | duplicate message number CERatchetDuplicateMessage + | -- | KEM key generation failed, indicating a broken RNG + CryptoKEMKeyGenError deriving (Eq, Show, Exception) aesKeySize :: Int @@ -1045,9 +1047,7 @@ md5Hash = BA.convert . (hash :: ByteString -> Digest MD5) -- | AEAD-GCM encryption with associated data. -- --- Used as part of double ratchet encryption. --- This function requires 16 bytes IV, it transforms IV in cryptonite_aes_gcm_init here: --- https://github.com/haskell-crypto/cryptonite/blob/master/cbits/cryptonite_aes.c +-- Used as part of double ratchet encryption, with a 16-byte IV (see @initAEAD@). encryptAEAD :: Key -> IV -> Int -> ByteString -> ByteString -> ExceptT CryptoError IO (AuthTag, ByteString) encryptAEAD aesKey ivBytes paddedLen ad msg = do aead <- initAEAD @AES256 aesKey ivBytes @@ -1067,10 +1067,7 @@ encryptAEADNoPad aesKey ivBytes ad msg = do -- | AEAD-GCM decryption with associated data. -- --- Used as part of double ratchet encryption. --- This function requires 16 bytes IV, it transforms IV in cryptonite_aes_gcm_init here: --- https://github.com/haskell-crypto/cryptonite/blob/master/cbits/cryptonite_aes.c --- To make it compatible with WebCrypto we will need to start using initAEADGCM. +-- Used as part of double ratchet encryption, with a 16-byte IV (see @initAEAD@). decryptAEAD :: Key -> IV -> ByteString -> ByteString -> AuthTag -> ExceptT CryptoError IO ByteString decryptAEAD aesKey ivBytes ad msg (AuthTag authTag) = do aead <- initAEAD @AES256 aesKey ivBytes @@ -1148,9 +1145,9 @@ maxLength :: forall i. KnownNat i => Int maxLength = fromIntegral (natVal $ Proxy @i) {-# INLINE maxLength #-} --- this function requires 16 bytes IV, it transforms IV in cryptonite_aes_gcm_init here: --- https://github.com/haskell-crypto/cryptonite/blob/master/cbits/cryptonite_aes.c --- This is used for double ratchet encryption, so to make it compatible with WebCrypto we will need to deprecate it and start using initAEADGCM +-- The 16-byte double ratchet IV is intentionally not the 96-bit IV recommended by NIST SP 800-38D, so GCM derives J0 = GHASH(IV || 0^64 || [128]_64), +-- as in crypton_aes_gcm_init: https://hackage.haskell.org/package/crypton-0.34/src/cbits/crypton_aes.c +-- WebCrypto and other SP 800-38D implementations interoperate only when given all 16 IV bytes. initAEAD :: forall c. AES.BlockCipher c => Key -> IV -> ExceptT CryptoError IO (AES.AEAD c) initAEAD (Key aesKey) (IV ivBytes) = do iv <- makeIV @c ivBytes diff --git a/src/Simplex/Messaging/Crypto/Ratchet.hs b/src/Simplex/Messaging/Crypto/Ratchet.hs index b38f2b5d1..067fedd89 100644 --- a/src/Simplex/Messaging/Crypto/Ratchet.hs +++ b/src/Simplex/Messaging/Crypto/Ratchet.hs @@ -72,6 +72,7 @@ module Simplex.Messaging.Crypto.Ratchet rcEncryptHeader, rcEncryptMsg, rcDecrypt, + maxSkippedMsgKeys, -- used in tests MsgHeader (..), RatchetInitParams (..), @@ -119,7 +120,7 @@ import Simplex.Messaging.Crypto.SNTRUP761.Bindings import Simplex.Messaging.Encoding import Simplex.Messaging.Encoding.String import Simplex.Messaging.Parsers (defaultJSON, parseE, parseE') -import Simplex.Messaging.Util (($>>=), (<$?>)) +import Simplex.Messaging.Util ((<$?>)) import Simplex.Messaging.Version import Simplex.Messaging.Version.Internal import UnliftIO.STM @@ -327,9 +328,13 @@ instance StrEncoding AnyE2ERatchetParamsUri where Nothing -> pure $ AnyE2ERatchetParamsUri SRKSProposed a $ E2ERatchetParamsUri vr k1 k2 Nothing _ -> fail "bad e2e params" where - kemP query = - queryParam_ "kem_key" query - $>>= \k -> Just . kemParams k <$> queryParam_ "kem_ct" query + kemP query = do + k_ <- queryParam_ "kem_key" query + ct_ <- queryParam_ "kem_ct" query + case (k_, ct_) of + (Just k, _) -> pure $ Just $ kemParams k ct_ + (Nothing, Nothing) -> pure Nothing + (Nothing, Just _) -> fail "bad e2e params: kem_ct without kem_key" kemParams k = \case Nothing -> ARKP SRKSProposed $ RKParamsProposed k Just ct -> ARKP SRKSAccepted $ RKParamsAccepted ct k @@ -958,6 +963,9 @@ type DecryptResult a = (Either CryptoError ByteString, Ratchet a, SkippedMsgDiff maxSkip :: Word32 maxSkip = 512 +maxSkippedMsgKeys :: Int +maxSkippedMsgKeys = 2000 + rcDecrypt :: forall a. (AlgorithmI a, DhAlgorithm a) => diff --git a/src/Simplex/Messaging/Crypto/SNTRUP761/Bindings.hs b/src/Simplex/Messaging/Crypto/SNTRUP761/Bindings.hs index 861abf69e..a99567e9a 100644 --- a/src/Simplex/Messaging/Crypto/SNTRUP761/Bindings.hs +++ b/src/Simplex/Messaging/Crypto/SNTRUP761/Bindings.hs @@ -16,6 +16,8 @@ module Simplex.Messaging.Crypto.SNTRUP761.Bindings ) where import Control.Concurrent.STM +import Control.Exception (throwIO) +import Control.Monad (when) import Crypto.Random (ChaChaDRG) import Data.Aeson (FromJSON (..), ToJSON (..)) import Data.Bifunctor (bimap) @@ -23,6 +25,7 @@ import Data.ByteArray (ScrubbedBytes) import qualified Data.ByteArray as BA import Data.ByteString (ByteString) import Simplex.Messaging.Agent.Store.DB (FromField (..), ToField (..)) +import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Crypto.SNTRUP761.Bindings.Defines import Simplex.Messaging.Crypto.SNTRUP761.Bindings.FFI import Simplex.Messaging.Crypto.SNTRUP761.Bindings.RNG (rngFuncPtr, withDRG) @@ -55,14 +58,13 @@ pattern KEMSharedKey s <- KEMSharedKey_ s type KEMKeyPair = (KEMPublicKey, KEMSecretKey) sntrup761Keypair :: TVar ChaChaDRG -> IO KEMKeyPair -sntrup761Keypair drg = - bimap KEMPublicKey_ KEMSecretKey - <$> BA.allocRet - c_SNTRUP761_SECRETKEY_SIZE - ( \skPtr -> - BA.alloc c_SNTRUP761_PUBLICKEY_SIZE $ \pkPtr -> - withDRG drg $ \cxtPtr -> c_sntrup761_keypair pkPtr skPtr cxtPtr rngFuncPtr - ) +sntrup761Keypair drg = do + ((r, pk), sk) <- + BA.allocRet c_SNTRUP761_SECRETKEY_SIZE $ \skPtr -> + BA.allocRet c_SNTRUP761_PUBLICKEY_SIZE $ \pkPtr -> + withDRG drg $ \cxtPtr -> c_sntrup761_keypair pkPtr skPtr cxtPtr rngFuncPtr + when (r /= 0) $ throwIO C.CryptoKEMKeyGenError + pure (KEMPublicKey_ pk, KEMSecretKey sk) sntrup761Enc :: TVar ChaChaDRG -> KEMPublicKey -> IO (KEMCiphertext, KEMSharedKey) sntrup761Enc drg (KEMPublicKey pk) = diff --git a/src/Simplex/Messaging/Crypto/SNTRUP761/Bindings/FFI.hs b/src/Simplex/Messaging/Crypto/SNTRUP761/Bindings/FFI.hs index 4983e9210..fc0093144 100644 --- a/src/Simplex/Messaging/Crypto/SNTRUP761/Bindings/FFI.hs +++ b/src/Simplex/Messaging/Crypto/SNTRUP761/Bindings/FFI.hs @@ -10,9 +10,9 @@ import Foreign import Foreign.C import Simplex.Messaging.Crypto.SNTRUP761.Bindings.RNG (RNGContext, RNGFunc) --- void sntrup761_keypair (uint8_t *pk, uint8_t *sk, void *random_ctx, sntrup761_random_func *random); +-- int sntrup761_keypair (uint8_t *pk, uint8_t *sk, void *random_ctx, sntrup761_random_func *random); foreign import ccall "sntrup761_keypair" - c_sntrup761_keypair :: Ptr Word8 -> Ptr Word8 -> Ptr RNGContext -> FunPtr RNGFunc -> IO () + c_sntrup761_keypair :: Ptr Word8 -> Ptr Word8 -> Ptr RNGContext -> FunPtr RNGFunc -> IO CInt -- void sntrup761_enc (uint8_t *c, uint8_t *k, const uint8_t *pk, void *random_ctx, sntrup761_random_func *random); foreign import ccall "sntrup761_enc" diff --git a/src/Simplex/Messaging/Protocol.hs b/src/Simplex/Messaging/Protocol.hs index 14f2a967f..a62e84224 100644 --- a/src/Simplex/Messaging/Protocol.hs +++ b/src/Simplex/Messaging/Protocol.hs @@ -168,6 +168,7 @@ module Simplex.Messaging.Protocol EncFwdTransmission (..), EncResponse (..), EncTransmission (..), + encTransmissionNonce, FwdResponse (..), FwdTransmission (..), NameRecord (..), @@ -280,7 +281,7 @@ import Simplex.Messaging.ServiceScheme import Simplex.Messaging.SimplexName (LabelHash, SimplexDomain (..), SimplexTLD (..), fullDomainName, labelHash) import Simplex.Messaging.Transport import Simplex.Messaging.Transport.Client (TransportHost, TransportHosts (..)) -import Simplex.Messaging.Util (bshow, eitherToMaybe, safeDecodeUtf8, (<$?>)) +import Simplex.Messaging.Util (bshow, eitherToMaybe, packZipWith, safeDecodeUtf8, (<$?>)) import Simplex.Messaging.Version import Simplex.Messaging.Version.Internal @@ -702,6 +703,11 @@ instance Encoding NewNtfCreds where newtype EncTransmission = EncTransmission ByteString deriving (Show) +encTransmissionNonce :: VersionSMP -> C.CbNonce -> C.CbNonce +encTransmissionNonce v nonce@(C.CbNonce s) + | v >= fwdNoncesSMPVersion = C.cbNonce $ packZipWith xor (smpEncode v) s <> BS.drop 2 s + | otherwise = nonce + data FwdTransmission = FwdTransmission { fwdCorrId :: CorrId, fwdVersion :: VersionSMP, @@ -737,8 +743,8 @@ data BrokerMsg where NMSG :: C.CbNonce -> EncNMsgMeta -> BrokerMsg -- Should include certificate chain PKEY :: SessionId -> VersionRangeSMP -> CertChainPubKey -> BrokerMsg -- TLS-signed server key for proxy shared secret and initial sender key - RRES :: EncFwdResponse -> BrokerMsg -- relay to proxy - PRES :: EncResponse -> BrokerMsg -- proxy to client + RRES :: Maybe C.CbNonce -> EncFwdResponse -> BrokerMsg -- relay to proxy + PRES :: Maybe C.CbNonce -> EncResponse -> BrokerMsg -- proxy to client END :: BrokerMsg ENDS :: Int64 -> IdsHash -> BrokerMsg DELD :: BrokerMsg @@ -1985,8 +1991,8 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where NID nId srvNtfDh -> e (NID_, ' ', nId, srvNtfDh) NMSG nmsgNonce encNMsgMeta -> e (NMSG_, ' ', nmsgNonce, encNMsgMeta) PKEY sid vr certKey -> e (PKEY_, ' ', sid, vr, certKey) - RRES (EncFwdResponse encBlock) -> e (RRES_, ' ', Tail encBlock) - PRES (EncResponse encBlock) -> e (PRES_, ' ', Tail encBlock) + RRES nonce_ (EncFwdResponse encBlock) -> fwdResp RRES_ nonce_ encBlock + PRES nonce_ (EncResponse encBlock) -> fwdResp PRES_ nonce_ encBlock END -> e END_ ENDS n idsHash -> serviceResp ENDS_ n idsHash DELD -> e DELD_ @@ -2010,6 +2016,9 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where serviceResp tag n idsHash | v >= rcvServiceSMPVersion = e (tag, ' ', n, idsHash) | otherwise = e (tag, ' ', n) + fwdResp tag nonce_ encBlock + | v >= fwdNoncesSMPVersion = e (tag, ' ', nonce_, Tail encBlock) + | otherwise = e (tag, ' ', Tail encBlock) protocolP v = \case MSG_ -> do @@ -2041,8 +2050,8 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where NID_ -> NID <$> _smpP <*> smpP NMSG_ -> NMSG <$> _smpP <*> smpP PKEY_ -> PKEY <$> _smpP <*> smpP <*> smpP - RRES_ -> RRES <$> (EncFwdResponse . unTail <$> _smpP) - PRES_ -> PRES <$> (EncResponse . unTail <$> _smpP) + RRES_ -> fwdRespP RRES EncFwdResponse + PRES_ -> fwdRespP PRES EncResponse END_ -> pure END ENDS_ -> serviceRespP ENDS DELD_ -> pure DELD @@ -2059,6 +2068,9 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where serviceRespP resp | v >= rcvServiceSMPVersion = resp <$> _smpP <*> smpP | otherwise = resp <$> _smpP <*> pure mempty + fwdRespP :: (Maybe C.CbNonce -> a -> BrokerMsg) -> (ByteString -> a) -> Parser BrokerMsg + fwdRespP resp enc = resp <$> (A.space *> nonceP) <*> (enc <$> A.takeByteString) + nonceP = if v >= fwdNoncesSMPVersion then smpP else pure Nothing fromProtocolError = \case PECmdSyntax -> CMD SYNTAX @@ -2075,7 +2087,7 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where -- PONG response must not have queue ID PONG -> noEntityMsg PKEY {} -> noEntityMsg - RRES _ -> noEntityMsg + RRES {} -> noEntityMsg ALLS -> noEntityMsg RNAME {} -> noEntityMsg -- other broker responses must have queue ID diff --git a/src/Simplex/Messaging/Server.hs b/src/Simplex/Messaging/Server.hs index a50f260a4..d97aacb49 100644 --- a/src/Simplex/Messaging/Server.hs +++ b/src/Simplex/Messaging/Server.hs @@ -1320,11 +1320,6 @@ isContactQueue QueueRec {queueMode, senderKey} = case queueMode of Just QMContact -> True Nothing -> isNothing senderKey -- for backward compatibility with pre-SKEY contact addresses -isSecuredMsgQueue :: QueueRec -> Bool -isSecuredMsgQueue QueueRec {queueMode, senderKey} = case queueMode of - Just QMContact -> False - _ -> isJust senderKey - -- Random correlation ID is used as a nonce in case crypto_box authenticator is used to authorize transmission verifyCmdAuthorization :: Maybe (THandleAuth 'TServer) -> Maybe TAuthorizations -> ByteString -> CorrId -> C.APublicAuthKey -> Bool verifyCmdAuthorization thAuth tAuth authorized corrId key = maybe False (verify key) tAuth @@ -1463,7 +1458,7 @@ client inc own pRequests forkProxiedCmd $ do liftIO (runExceptT (forwardSMPTransmission smp corrId fwdV pubKey encBlock) `E.catches` clientHandlers) >>= \case - Right r -> PRES r <$ inc own pSuccesses + Right (nonce_, r) -> PRES nonce_ r <$ inc own pSuccesses Left e -> ERR (smpProxyError e) <$ case e of PCEProtocolError {} -> inc own pSuccesses _ -> inc own pErrorsOther @@ -2132,7 +2127,7 @@ client unless (fwdVersion `isCompatible` thServerVRange thParams') $ throwE $ transportErr TEVersion let clientSecret = C.dh' fwdKey serverPrivKey clientNonce = C.cbNonce $ bs fwdCorrId - b <- liftEitherWith (const CRYPTO) $ C.cbDecrypt clientSecret clientNonce et + b <- liftEitherWith (const CRYPTO) $ C.cbDecrypt clientSecret (encTransmissionNonce fwdVersion clientNonce) et let clntTHParams = smpTHParamsSetVersion fwdVersion thParams' -- only allowing single forwarded transactions t' <- case tParse clntTHParams b of @@ -2145,9 +2140,13 @@ client TBError _ _ : _ -> throwE BLOCK TBTransmission b' _ : _ -> pure b' TBTransmissions b' _ _ : _ -> pure b' - r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (C.reverseNonce clientNonce) r' paddedProxiedTLength + nonce_ <- + if fwdVersion >= fwdNoncesSMPVersion + then Just <$> (atomically . C.randomCbNonce =<< asks random) + else pure Nothing + r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (fromMaybe (C.reverseNonce clientNonce) nonce_) r' paddedProxiedTLength let fr = FwdResponse {fwdCorrId, fwdResponse = r2} - pure $ RRES $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr) + pure $ RRES nonce_ $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr) -- the inner response, or Nothing if forked (RSLV). r_ <- lift (rejectOrVerify clntThAuth t') >>= \case -- rejectOrVerify filters allowed commands, no need to repeat it here. diff --git a/src/Simplex/Messaging/Server/QueueStore.hs b/src/Simplex/Messaging/Server/QueueStore.hs index 3904871bf..fa86e85d4 100644 --- a/src/Simplex/Messaging/Server/QueueStore.hs +++ b/src/Simplex/Messaging/Server/QueueStore.hs @@ -13,12 +13,14 @@ module Simplex.Messaging.Server.QueueStore ServiceRec (..), CertFingerprint, ServerEntityStatus (..), + isSecuredMsgQueue, ) where import Control.Applicative (optional, (<|>)) import qualified Data.ByteString.Char8 as B import Data.Functor (($>)) import Data.List.NonEmpty (NonEmpty) +import Data.Maybe (isJust) import qualified Data.X509 as X import qualified Data.X509.Validation as XV import Simplex.Messaging.Encoding @@ -48,6 +50,11 @@ data QueueRec = QueueRec } deriving (Show) +isSecuredMsgQueue :: QueueRec -> Bool +isSecuredMsgQueue QueueRec {queueMode, senderKey} = case queueMode of + Just QMContact -> False + _ -> isJust senderKey + data NtfCreds = NtfCreds { notifierId :: NotifierId, notifierKey :: NtfPublicAuthKey, diff --git a/src/Simplex/Messaging/Server/QueueStore/Postgres.hs b/src/Simplex/Messaging/Server/QueueStore/Postgres.hs index 0c7327776..1f464b89a 100644 --- a/src/Simplex/Messaging/Server/QueueStore/Postgres.hs +++ b/src/Simplex/Messaging/Server/QueueStore/Postgres.hs @@ -201,9 +201,9 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where addQueueLinkData st sq lnkId d = withQueueRec sq $ \q -> case queueData q of Nothing -> - addLink q $ \db -> DB.execute db qry (d :. (lnkId, rId)) + addLink q $ \db -> DB.execute db qry (d :. (lnkId, rId, QMContact)) Just (lnkId', _) | lnkId' == lnkId -> - addLink q $ \db -> DB.execute db (qry <> " AND (fixed_data IS NULL OR fixed_data = ?)") (d :. (lnkId, rId, fst d)) + addLink q $ \db -> DB.execute db (qry <> " AND (fixed_data IS NULL OR fixed_data = ?)") (d :. (lnkId, rId, QMContact, fst d)) _ -> throwE AUTH where rId = recipientId sq @@ -211,7 +211,8 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where assertUpdated $ withDB' "addQueueLinkData" st update atomically $ writeTVar (queueRec sq) $ Just q {queueData = Just (lnkId, d)} withLog "addQueueLinkData" st $ \s -> logCreateLink s rId lnkId d - qry = "UPDATE msg_queues SET fixed_data = ?, user_data = ?, link_id = ? WHERE recipient_id = ? AND deleted_at IS NULL" + -- the sender key condition is checked in SQL because without cache each command reads its own copy of the queue record + qry = "UPDATE msg_queues SET fixed_data = ?, user_data = ?, link_id = ? WHERE recipient_id = ? AND deleted_at IS NULL AND (sender_key IS NULL OR queue_mode = ?)" deleteQueueLinkData :: PostgresQueueStore q -> q -> IO (Either ErrorType ()) deleteQueueLinkData st sq = diff --git a/src/Simplex/Messaging/Server/QueueStore/STM.hs b/src/Simplex/Messaging/Server/QueueStore/STM.hs index fa33f7d88..8d56897b5 100644 --- a/src/Simplex/Messaging/Server/QueueStore/STM.hs +++ b/src/Simplex/Messaging/Server/QueueStore/STM.hs @@ -172,6 +172,7 @@ instance StoreQueueClass q => QueueStoreClass q (STMQueueStore q) where rId = recipientId sq qr = queueRec sq add q = case queueData q of + _ | isSecuredMsgQueue q -> pure $ Left AUTH Nothing -> addLink Just (lnkId', d') | lnkId' == lnkId && fst d' == fst d -> addLink _ -> pure $ Left AUTH diff --git a/src/Simplex/Messaging/Transport.hs b/src/Simplex/Messaging/Transport.hs index 6fa307fd1..0a082a388 100644 --- a/src/Simplex/Messaging/Transport.hs +++ b/src/Simplex/Messaging/Transport.hs @@ -54,6 +54,7 @@ module Simplex.Messaging.Transport namesSMPVersion, serverInfoSMPVersion, nameAvailSMPVersion, + fwdNoncesSMPVersion, simplexMQVersion, smpBlockSize, TransportConfig (..), @@ -177,6 +178,7 @@ smpBlockSize = 16384 -- 20 - public namespaces resolver, RSLV command (6/20/2026) -- 21 - server public information in handshake (7/5/2026) -- 22 - RNAME answers name availability as well as the record (7/25/2026) +-- 23 - version in forwarded command nonce, random nonce in forwarded responses (10/2/2026) data SMPVersion @@ -218,6 +220,9 @@ serverInfoSMPVersion = VersionSMP 21 nameAvailSMPVersion :: VersionSMP nameAvailSMPVersion = VersionSMP 22 +fwdNoncesSMPVersion :: VersionSMP +fwdNoncesSMPVersion = VersionSMP 23 + minClientSMPRelayVersion :: VersionSMP minClientSMPRelayVersion = VersionSMP 14 @@ -225,20 +230,20 @@ minServerSMPRelayVersion :: VersionSMP minServerSMPRelayVersion = VersionSMP 14 currentClientSMPRelayVersion :: VersionSMP -currentClientSMPRelayVersion = VersionSMP 22 +currentClientSMPRelayVersion = VersionSMP 23 currentServerSMPRelayVersion :: VersionSMP -currentServerSMPRelayVersion = VersionSMP 22 +currentServerSMPRelayVersion = VersionSMP 23 -- Max SMP protocol version to be used in e2e encrypted connection between -- client and server, as defined by SMP proxy. Normally set below the current -- version to prevent client version fingerprinting by the destination relays --- when clients upgrade at different times. Pinned to the current version (22) --- for this release because a proxied RSLV only carries availability from --- nameAvailSMPVersion (22), so the one-version anti-fingerprinting buffer does --- not apply yet; it reappears once the current version advances past 22. +-- when clients upgrade at different times. Pinned to the current version (23) +-- for this release because forwarded commands use the nonces from +-- fwdNoncesSMPVersion (23), so the one-version anti-fingerprinting buffer does +-- not apply yet; it reappears once the current version advances past 23. proxiedSMPRelayVersion :: VersionSMP -proxiedSMPRelayVersion = VersionSMP 22 +proxiedSMPRelayVersion = VersionSMP 23 -- minimal supported protocol version is 14 supportedClientSMPRelayVRange :: VersionRangeSMP diff --git a/src/Simplex/RemoteControl/Client.hs b/src/Simplex/RemoteControl/Client.hs index a9970c273..1602f1993 100644 --- a/src/Simplex/RemoteControl/Client.hs +++ b/src/Simplex/RemoteControl/Client.hs @@ -26,6 +26,7 @@ module Simplex.RemoteControl.Client -- for tests only sendRCPacket, receiveRCPacket, + findRCCtrlPairing, ) where import Control.Applicative ((<|>)) @@ -46,7 +47,7 @@ import Data.List.NonEmpty (NonEmpty (..)) import qualified Data.List.NonEmpty as L import Data.Maybe (isNothing) import qualified Data.Text as T -import Data.Time.Clock.System (getSystemTime) +import Data.Time.Clock.System (SystemTime (..), getSystemTime) import Data.Tuple (swap) import Data.Word (Word16) import qualified Data.X509 as X @@ -395,8 +396,10 @@ findRCCtrlPairing :: NonEmpty RCCtrlPairing -> RCEncInvitation -> ExceptT RCErro findRCCtrlPairing pairings RCEncInvitation {dhPubKey, nonce, encInvitation} = do (pairing, signedInvStr) <- liftEither $ decrypt (L.toList pairings) signedInv <- liftEitherWith RCESyntax $ strDecode signedInvStr - inv@(RCVerifiedInvitation RCInvitation {dh = invDh}) <- maybe (throwE RCEInvitation) pure $ verifySignedInvitation signedInv + inv@(RCVerifiedInvitation RCInvitation {dh = invDh, ts}) <- maybe (throwE RCEInvitation) pure $ verifySignedInvitation signedInv unless (invDh == dhPubKey) $ throwE RCEInvitation + now <- systemSeconds <$> liftIO getSystemTime + unless (now - 3660 <= systemSeconds ts && systemSeconds ts <= now + 3600) $ throwE RCEInvitation pure (pairing, inv) where decrypt :: [RCCtrlPairing] -> Either RCErrorType (RCCtrlPairing, ByteString) diff --git a/src/Simplex/RemoteControl/Discovery.hs b/src/Simplex/RemoteControl/Discovery.hs index 4a69a57a1..baff4adaa 100644 --- a/src/Simplex/RemoteControl/Discovery.hs +++ b/src/Simplex/RemoteControl/Discovery.hs @@ -122,18 +122,22 @@ closeListener subscribers sock = joinMulticast :: TMVar Int -> N.Socket -> N.HostAddress -> IO () joinMulticast subscribers sock group = do now <- atomically $ takeTMVar subscribers - when (now == 0) $ do - setMembership sock group True >>= \case - Left e -> atomically (putTMVar subscribers now) >> logError ("setMembership failed " <> tshow e) - Right () -> atomically $ putTMVar subscribers (now + 1) + if now == 0 + then + setMembership sock group True >>= \case + Left e -> atomically (putTMVar subscribers now) >> logError ("setMembership failed " <> tshow e) + Right () -> atomically $ putTMVar subscribers (now + 1) + else atomically $ putTMVar subscribers (now + 1) partMulticast :: TMVar Int -> N.Socket -> N.HostAddress -> IO () partMulticast subscribers sock group = do now <- atomically $ takeTMVar subscribers - when (now == 1) $ - setMembership sock group False >>= \case - Left e -> atomically (putTMVar subscribers now) >> logError ("setMembership failed " <> tshow e) - Right () -> atomically $ putTMVar subscribers (now - 1) + if now == 1 + then + setMembership sock group False >>= \case + Left e -> atomically (putTMVar subscribers (now - 1)) >> logError ("setMembership failed " <> tshow e) + Right () -> atomically $ putTMVar subscribers (now - 1) + else atomically $ putTMVar subscribers (max 0 (now - 1)) listenerHostAddr4 :: UDP.ListenSocket -> N.HostAddress listenerHostAddr4 sock = case UDP.mySockAddr sock of diff --git a/tests/AgentTests/ConnectionRequestTests.hs b/tests/AgentTests/ConnectionRequestTests.hs index c3b255139..40ca12cf9 100644 --- a/tests/AgentTests/ConnectionRequestTests.hs +++ b/tests/AgentTests/ConnectionRequestTests.hs @@ -20,6 +20,8 @@ module AgentTests.ConnectionRequestTests import AgentTests.EqInstances () import Data.ByteString (ByteString) +import qualified Data.ByteString.Char8 as B +import Data.Either (isLeft) import Network.HTTP.Types (urlEncode) import Simplex.Messaging.Agent.Protocol import qualified Simplex.Messaging.Crypto as C @@ -285,6 +287,9 @@ connectionRequestTests = contactAddressV6 #== ("https://simplex.chat/contact#/?v=1-2&smp=" <> url queueStr) -- adjusted to v6 contactAddressV6 #== ("https://simplex.chat/contact#/?v=2-2&smp=" <> url queueStr) contactAddressClientData #==# ("simplex:/contact#/?v=6-8&smp=" <> url queueStr <> "&data=" <> url "{\"type\":\"group_link\", \"group_link_id\":\"abc\"}") + it "should reject KEM ciphertext without KEM key in e2e params" $ + strDecode @(RcvE2ERatchetParamsUri 'C.X448) (strEncode testE2ERatchetParams <> "&kem_ct=" <> strEncode (B.replicate 1039 '\0')) + `shouldSatisfy` isLeft it "should serialize / parse queue address, connection invitations and contact addresses as binary" $ do smpEncodingTest queue smpEncodingTest queueNoQM -- this passes, no queue mode patch in SMPQueueUri encoding @@ -357,6 +362,10 @@ connectionRequestTests = smpEncodingTest $ AgentServiceRequest [qInfo] Nothing "service request payload" smpEncodingTest $ AgentServiceResponse "service response payload" smpEncodingTest $ AgentRejection "rejected: not allowed" + it "should serialize and parse ratchet key info" $ do + smpDecode "R" `shouldBe` Right (AgentRatchetInfo RatchetInfo {answeredKeyHash = Nothing}) + smpDecode "R1\3abcdef" `shouldBe` Right (AgentRatchetInfo RatchetInfo {answeredKeyHash = Just "abc"}) + smpEncodingTest $ AgentRatchetInfo RatchetInfo {answeredKeyHash = Just "0123456789abcdef0123456789abcdef"} where smpEncodingTest :: (Encoding a, Eq a, Show a, HasCallStack) => a -> Expectation smpEncodingTest a = smpDecode (smpEncode a) `shouldBe` Right a diff --git a/tests/AgentTests/FunctionalAPITests.hs b/tests/AgentTests/FunctionalAPITests.hs index 57e54c55b..6ab503866 100644 --- a/tests/AgentTests/FunctionalAPITests.hs +++ b/tests/AgentTests/FunctionalAPITests.hs @@ -70,8 +70,8 @@ import Data.ByteString.Char8 (ByteString) import qualified Data.ByteString.Char8 as B import Data.Either (isRight) import Data.Int (Int64) -import Data.List (find, isPrefixOf, isSuffixOf) -import Data.List.NonEmpty (NonEmpty) +import Data.List (find, isInfixOf, isPrefixOf, isSuffixOf) +import Data.List.NonEmpty (NonEmpty (..)) import qualified Data.Map as M import Data.Maybe (isJust, isNothing) import qualified Data.Set as S @@ -87,12 +87,12 @@ import SMPAgentClient import SMPClient import Simplex.Messaging.Agent hiding (acceptContact, createConnection, deleteConnection, deleteConnections, getConnShortLink, joinConnection, sendMessage, setConnShortLink, subscribeConnection, suspendConnection) import qualified Simplex.Messaging.Agent as A -import Simplex.Messaging.Agent.Client (ProtocolTestFailure (..), ProtocolTestStep (..), ServerQueueInfo (..), UserNetworkInfo (..), UserNetworkType (..), waitForUserNetwork) +import Simplex.Messaging.Agent.Client (ProtocolTestFailure (..), ProtocolTestStep (..), ServerQueueInfo (..), UserNetworkInfo (..), UserNetworkType (..), sendAgentMessage, waitForUserNetwork) import Simplex.Messaging.Agent.Env.SQLite (AgentConfig (..), Env (..), InitialAgentServers (..), createAgentStore) import Simplex.Messaging.Agent.Protocol hiding (CON, CONF, INFO, REQ, SENT) import qualified Simplex.Messaging.Agent.Protocol as A import Simplex.Messaging.Agent.Store (Connection' (..), SomeConn' (..), StoredRcvQueue (..)) -import Simplex.Messaging.Agent.Store.AgentStore (getConn) +import Simplex.Messaging.Agent.Store.AgentStore (deleteRatchetKeyHashesExpired, getConn, getRatchetX3dhKeys) import Simplex.Messaging.Agent.Store.Common (DBStore (..), withTransaction) import Simplex.Messaging.Agent.Store.Interface import qualified Simplex.Messaging.Agent.Store.DB as DB @@ -229,7 +229,11 @@ pattern Rcvd' :: AgentMsgId -> AgentMsgId -> AEvent 'AEConn pattern Rcvd' aMsgId rcvdMsgId <- RCVD MsgMeta {integrity = MsgOk, recipient = (aMsgId, _)} [MsgReceipt {agentMsgId = rcvdMsgId, msgRcptStatus = MROk}] smpCfgVPrev :: ProtocolClientConfig SMPVersion -smpCfgVPrev = (smpCfg agentCfg) {serverVRange = prevRange $ serverVRange $ smpCfg agentCfg} +smpCfgVPrev = + (smpCfg agentCfg) + { serverVRange = prevRange $ serverVRange $ smpCfg agentCfg, + proxiedRelayVRange = prevRange $ proxiedRelayVRange $ smpCfg agentCfg + } -- ntfCfgVPrev :: ProtocolClientConfig NTFVersion -- ntfCfgVPrev = (ntfCfg agentCfg) {clientALPN = Nothing, serverVRange = V.mkVersionRange (VersionNTF 1) (VersionNTF 1)} @@ -457,6 +461,16 @@ functionalAPITests ps = do testRatchetSyncSuspendForeground ps it "should synchronize ratchets when clients start synchronization simultaneously" $ testRatchetSyncSimultaneous ps + it "should ignore replayed ratchet key after expired hashes are deleted" $ + testRatchetSyncReplayedKey ps + it "should synchronize ratchets when synchronization is forced again" $ + testRatchetSyncRepeated ps + it "should not mark ratchet key as processed when ratchet recreation fails" $ + testRatchetSyncFailedKeyNotProcessed ps + it "should not store reply ratchet key when ratchet recreation fails" $ + testRatchetSyncFailedRecreationNoReply ps + it "should not store ratchet key when starting synchronization fails" $ + testRatchetSyncStartFailedNoKey ps #endif describe "Subscription mode OnlyCreate" $ do it "messages delivered only when polled" $ @@ -2735,6 +2749,117 @@ testRatchetSyncSimultaneous ps = do disposeAgentClient bob disposeAgentClient bob2 +testRatchetSyncReplayedKey :: HasCallStack => (ASrvTransport, AStoreType) -> IO () +testRatchetSyncReplayedKey ps = withAgentClients2 $ \alice bob -> do + (aliceId, bobId, bob2) <- withSmpServerStoreMsgLogOn ps testPort $ \_ -> + setupDesynchronizedRatchet alice bob + ("", "", DOWN _ _) <- nGet alice + ("", "", DOWN _ _) <- nGet bob2 + _ <- runRight $ synchronizeRatchet bob2 aliceId PQSupportOn False + Right pks <- withTransaction (store $ agentEnv bob2) (`getRatchetX3dhKeys` aliceId) + Right (SomeConn _ (DuplexConnection _ _ (sq :| _))) <- withTransaction (store $ agentEnv bob2) (`getConn` aliceId) + withSmpServerStoreMsgLogOn ps testPort $ \_ -> do + concurrently_ + (getInAnyOrder alice [ratchetSyncP' bobId RSAgreed, serverUpP]) + (getInAnyOrder bob2 [ratchetSyncP' aliceId RSAgreed, serverUpP]) + get alice =##> ratchetSyncP bobId RSOk + get bob2 =##> ratchetSyncP aliceId RSOk + withTransaction (store $ agentEnv alice) $ \db -> deleteRatchetKeyHashesExpired db 0 100 + let keyMsg = AgentRatchetKey {agentVersion = currentSMPAgentVersion, e2eEncryption = CR.mkRcvE2ERatchetParams CR.currentE2EEncryptVersion pks, info = ""} + Right _ <- runReaderT (runExceptT $ sendAgentMessage bob2 sq SMP.noMsgFlags $ smpEncode keyMsg) (agentEnv bob2) + runRight_ $ exchangeGreetingsMsgIds alice bobId 10 bob2 aliceId 7 + disposeAgentClient bob2 + +testRatchetSyncRepeated :: HasCallStack => (ASrvTransport, AStoreType) -> IO () +testRatchetSyncRepeated ps = withAgentClients2 $ \alice bob -> do + (aliceId, bobId, bob2) <- startRatchetSyncOffline ps alice bob + ConnectionStats {ratchetSyncState = rss2} <- runRight $ synchronizeRatchet bob2 aliceId PQSupportOn True + rss2 `shouldBe` RSStarted + + withSmpServerStoreMsgLogOn ps testPort $ \_ -> do + concurrently_ + (getInAnyOrder alice [ratchetSyncP' bobId RSAgreed, serverUpP]) + (getInAnyOrder bob2 [ratchetSyncP' aliceId RSAgreed, serverUpP]) + runRight_ $ do + get alice =##> ratchetSyncP bobId RSAgreed + get alice =##> ratchetSyncP bobId RSOk + get bob2 =##> ratchetSyncP aliceId RSOk + msgId <- sendMessage alice bobId SMP.noMsgFlags "hello" + get alice ##> ("", bobId, SENT msgId) + get bob2 =##> \case ("", c, Msg "hello") -> c == aliceId; _ -> False + ackMessage bob2 aliceId 8 Nothing + map fst <$> processedRatchetKeyHashes bob2 `shouldReturn` [aliceId, aliceId] + disposeAgentClient bob2 + +testRatchetSyncFailedKeyNotProcessed :: HasCallStack => (ASrvTransport, AStoreType) -> IO () +testRatchetSyncFailedKeyNotProcessed ps = withAgentClients2 $ \alice bob -> do + (aliceId, bobId, bob2) <- startRatchetSyncOffline ps alice bob + withTransaction (store $ agentEnv bob2) $ \db -> + DB.execute_ db "UPDATE ratchets SET x3dh_priv_key_1 = NULL" + + withSmpServerStoreMsgLogOn ps testPort $ \_ -> + concurrently_ + (getInAnyOrder alice [ratchetSyncP' bobId RSAgreed, serverUpP]) + (getInAnyOrder bob2 [x3dhKeysNotFoundP aliceId, serverUpP]) + map fst <$> processedRatchetKeyHashes alice `shouldReturn` [bobId] + processedRatchetKeyHashes bob2 `shouldReturn` [] + disposeAgentClient bob2 + where + x3dhKeysNotFoundP :: ConnId -> ATransmission -> Bool + x3dhKeysNotFoundP cId = \case + (_, cId', AEvt SAEConn (ERR (A.INTERNAL e))) -> cId' == cId && "SEX3dhKeysNotFound" `isPrefixOf` e + _ -> False + +testRatchetSyncFailedRecreationNoReply :: HasCallStack => (ASrvTransport, AStoreType) -> IO () +testRatchetSyncFailedRecreationNoReply ps = withAgentClients2 $ \alice bob -> do + (_, bobId, bob2) <- startRatchetSyncOffline ps alice bob + aliceSndMsgs <- sndMessages alice + withTransaction (store $ agentEnv alice) $ \db -> + DB.execute_ db "CREATE TRIGGER fail_ratchet_insert BEFORE INSERT ON ratchets BEGIN SELECT RAISE(ABORT, 'ratchet insert failed'); END" + withSmpServerStoreMsgLogOn ps testPort $ \_ -> + concurrently_ + (getInAnyOrder alice [ratchetInsertFailedP bobId, serverUpP]) + (getInAnyOrder bob2 [serverUpP]) + processedRatchetKeyHashes alice `shouldReturn` [] + sndMessages alice `shouldReturn` aliceSndMsgs + disposeAgentClient bob2 + where + ratchetInsertFailedP :: ConnId -> ATransmission -> Bool + ratchetInsertFailedP cId = \case + (_, cId', AEvt SAEConn (ERR (A.INTERNAL e))) -> cId' == cId && "ratchet insert failed" `isInfixOf` e + _ -> False + +testRatchetSyncStartFailedNoKey :: HasCallStack => (ASrvTransport, AStoreType) -> IO () +testRatchetSyncStartFailedNoKey ps = withAgentClients2 $ \alice bob -> do + (aliceId, _, bob2) <- withSmpServerStoreMsgLogOn ps testPort $ \_ -> + setupDesynchronizedRatchet alice bob + bobSndMsgs <- sndMessages bob2 + withTransaction (store $ agentEnv bob2) $ \db -> + DB.execute_ db "CREATE TRIGGER fail_ratchet_update BEFORE UPDATE ON ratchets BEGIN SELECT RAISE(ABORT, 'ratchet update failed'); END" + Left (A.INTERNAL e) <- runExceptT $ synchronizeRatchet bob2 aliceId PQSupportOff False + e `shouldContain` "ratchet update failed" + ConnectionStats {ratchetSyncState} <- runRight $ getConnectionServers bob2 aliceId + ratchetSyncState `shouldBe` RSRequired + withTransaction (store $ agentEnv bob2) (`DB.query_` "SELECT conn_id, pq_support FROM connections") `shouldReturn` [(aliceId, PQSupportOn)] + sndMessages bob2 `shouldReturn` bobSndMsgs + disposeAgentClient bob2 + +startRatchetSyncOffline :: HasCallStack => (ASrvTransport, AStoreType) -> AgentClient -> AgentClient -> IO (ConnId, ConnId, AgentClient) +startRatchetSyncOffline ps alice bob = do + (aliceId, bobId, bob2) <- withSmpServerStoreMsgLogOn ps testPort $ \_ -> + setupDesynchronizedRatchet alice bob + ("", "", DOWN _ _) <- nGet alice + ("", "", DOWN _ _) <- nGet bob2 + ConnectionStats {ratchetSyncState} <- runRight $ synchronizeRatchet bob2 aliceId PQSupportOn False + ratchetSyncState `shouldBe` RSStarted + pure (aliceId, bobId, bob2) + +processedRatchetKeyHashes :: AgentClient -> IO [(ConnId, ByteString)] +processedRatchetKeyHashes c = withTransaction (store $ agentEnv c) (`DB.query_` "SELECT conn_id, hash FROM processed_ratchet_key_hashes") + +sndMessages :: AgentClient -> IO [(ConnId, Int64)] +sndMessages c = withTransaction (store $ agentEnv c) (`DB.query_` "SELECT conn_id, internal_id FROM snd_messages ORDER BY conn_id, internal_id") + getMsg :: AgentClient -> ConnId -> ExceptT AgentErrorType IO a -> ExceptT AgentErrorType IO a getMsg c cId action = do liftIO $ noMessages c "nothing should be delivered before GET" diff --git a/tests/AgentTests/SQLiteTests.hs b/tests/AgentTests/SQLiteTests.hs index c22eddd7a..8a5fd14d7 100644 --- a/tests/AgentTests/SQLiteTests.hs +++ b/tests/AgentTests/SQLiteTests.hs @@ -20,12 +20,13 @@ import Control.Concurrent.Async (concurrently_) import Control.Concurrent.MVar import Control.Concurrent.STM import Control.Exception (SomeException) -import Control.Monad (replicateM_) +import Control.Monad (forM_, replicateM_) import Control.Monad.Trans.Except import Crypto.Random (ChaChaDRG) import Data.ByteArray (ScrubbedBytes) import Data.ByteString.Char8 (ByteString) import Data.List (isInfixOf) +import qualified Data.Map.Strict as M import qualified Data.Text as T import Data.Text.Encoding (encodeUtf8) import Data.Time @@ -51,6 +52,7 @@ import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Crypto.File (CryptoFile (..)) import Simplex.Messaging.Crypto.Ratchet (pattern IKPQOn) import qualified Simplex.Messaging.Crypto.Ratchet as CR +import Simplex.Messaging.Encoding (Encoding (..)) import Simplex.Messaging.Encoding.String (StrEncoding (..)) import Simplex.Messaging.Protocol (EntityId (..), QueueMode (..), SubscriptionMode (..), pattern VersionSMPC) import qualified Simplex.Messaging.Protocol as SMP @@ -135,6 +137,8 @@ storeTests = do testCreateRcvMsg testCreateSndMsg testCreateRcvAndSndMsgs + describe "deleteRatchetKeyHashesExpired" testDeleteRatchetKeyHashesExpired + it "should keep only the newest skipped message keys" testGetSkippedMsgKeys describe "Work items" $ do it "should getPendingQueueMsg" testGetPendingQueueMsg it "should getPendingServerCommand" testGetPendingServerCommand @@ -599,6 +603,42 @@ testCreateRcvAndSndMsgs = testCreateSndMsg_ db "snd_hash_1" connId sq $ mkSndMsgData (InternalId 5) (InternalSndId 2) "snd_hash_2" testCreateSndMsg_ db "snd_hash_2" connId sq $ mkSndMsgData (InternalId 6) (InternalSndId 3) "snd_hash_3" +testDeleteRatchetKeyHashesExpired :: SpecWith DBStore +testDeleteRatchetKeyHashesExpired = + it "should delete expired ratchet key hashes except the newest in each connection" . withStoreTransaction $ \db -> do + g <- C.newRandom + Right connId <- createNewConn db g cData1 {connId = ""} SCMInvitation + Right connId' <- createNewConn db g cData1 {connId = ""} SCMContact + let hashes = ["h1", "h2", "h3", "h4", "h5", "h6"] + forM_ hashes $ addProcessedRatchetKeyHash db connId + forM_ (take 4 hashes) $ addProcessedRatchetKeyHash db connId' + deleteRatchetKeyHashesExpired db 86400 4 + mapM (checkRatchetKeyHashExists db connId) hashes `shouldReturn` replicate 6 True + deleteRatchetKeyHashesExpired db 0 4 + mapM (checkRatchetKeyHashExists db connId) hashes `shouldReturn` [False, False, True, True, True, True] + mapM (checkRatchetKeyHashExists db connId') (take 4 hashes) `shouldReturn` replicate 4 True + +testGetSkippedMsgKeys :: DBStore -> Expectation +testGetSkippedMsgKeys st = do + g <- C.newRandom + withTransaction st $ \db -> do + Right connId <- createNewConn db g cData1 {connId = ""} SCMInvitation + Right connId' <- createNewConn db g cData1 {connId = ""} SCMInvitation + createSkippedKeys db connId' + createSkippedKeys db connId + M.map M.keys <$> getSkippedMsgKeys db connId 4 + `shouldReturn` M.singleton (C.Key "header_key") [1, 2, 3, 4] + getMsgNs db connId `shouldReturn` [1 .. 4] + getMsgNs db connId' `shouldReturn` [1 .. 10] + where + createSkippedKeys :: DB.Connection -> ConnId -> IO () + createSkippedKeys db connId = do + DB.execute db "INSERT INTO ratchets (conn_id) VALUES (?)" (Only connId) + forM_ ([10, 9 .. 1] :: [Int]) $ \msgN -> + DB.execute db "INSERT INTO skipped_messages (conn_id, header_key, msg_n, msg_key) VALUES (?, ?, ?, ?)" (connId, "header_key" :: ByteString, msgN, smpEncode ("key" :: ByteString, "iv" :: ByteString)) + getMsgNs :: DB.Connection -> ConnId -> IO [Int] + getMsgNs db connId = map fromOnly <$> DB.query db "SELECT msg_n FROM skipped_messages WHERE conn_id = ? ORDER BY msg_n" (Only connId) + testCloseReopenStore :: IO () testCloseReopenStore = do st <- createStore' diff --git a/tests/AgentTests/SchemaDump.hs b/tests/AgentTests/SchemaDump.hs index d9aa79513..ee401bf19 100644 --- a/tests/AgentTests/SchemaDump.hs +++ b/tests/AgentTests/SchemaDump.hs @@ -48,6 +48,7 @@ schemaDumpTest = do it "verify strict tables" testVerifyStrict it "should NOT create user record for new database" testUsersMigrationNew it "should create user record for old database" testUsersMigrationOld + it "should remove duplicate ratchet key hashes before adding unique index" testRatchetKeyHashesUniqueMigration testVerifySchemaDump :: IO () testVerifySchemaDump = do @@ -114,12 +115,28 @@ testUsersMigrationOld = do `shouldReturn` ([Only (1 :: Int)]) closeDBStore st' +testRatchetKeyHashesUniqueMigration :: IO () +testRatchetKeyHashesUniqueMigration = do + let beforeUnique = takeWhile (("m20260929_ratchet_indexes" /=) . name) appMigrations + Right st <- createDBStore (DBOpts testDB [] "" False True TQOff) beforeUnique (MigrationConfig MCError Nothing) + withTransaction' st $ \db -> do + SQL.execute_ db "INSERT INTO users (user_id) VALUES (1)" + SQL.execute_ db "INSERT INTO connections (conn_id, conn_mode, user_id) VALUES (x'01', 'INV', 1), (x'02', 'INV', 1)" + SQL.execute_ db "INSERT INTO processed_ratchet_key_hashes (conn_id, hash) VALUES (x'01', x'aa'), (x'01', x'aa'), (x'01', x'bb'), (x'02', x'aa'), (x'01', x'aa')" + closeDBStore st + Right st' <- createDBStore (DBOpts testDB [] "" False True TQOff) appMigrations (MigrationConfig MCYesUp Nothing) + withTransaction' st' (`SQL.query_` "SELECT processed_ratchet_key_hash_id FROM processed_ratchet_key_hashes ORDER BY processed_ratchet_key_hash_id") + `shouldReturn` [Only (1 :: Int), Only 3, Only 4] + closeDBStore st' + skipComparisonForDownMigrations :: [String] skipComparisonForDownMigrations = [ -- on down migration idx_messages_internal_snd_id_ts index moves down to the end of the file "m20230814_indexes", -- snd_secure and last_broker_ts columns swap order on down migration - "m20250322_short_links" + "m20250322_short_links", + -- on down migration idx_processed_ratchet_key_hashes_hash index moves down to the end of the file + "m20260929_ratchet_indexes" ] getSchema :: FilePath -> FilePath -> IO String diff --git a/tests/CoreTests/CryptoTests.hs b/tests/CoreTests/CryptoTests.hs index edeb097bf..268611990 100644 --- a/tests/CoreTests/CryptoTests.hs +++ b/tests/CoreTests/CryptoTests.hs @@ -6,8 +6,9 @@ module CoreTests.CryptoTests (cryptoTests) where +import Control.Concurrent (forkIO, newEmptyMVar, putMVar, takeMVar) import Control.Concurrent.STM -import Control.Exception (evaluate) +import Control.Exception (bracket, evaluate) import Control.Monad.Except import qualified Data.Aeson as J import qualified Data.ByteString.Char8 as B @@ -22,9 +23,12 @@ import Data.Time.Clock (UTCTime (..)) import qualified Data.Text.Lazy as LT import qualified Data.Text.Lazy.Encoding as LE import Data.Type.Equality +import Data.Word (Word8) import qualified Data.X509 as X import qualified Data.X509.CertificateStore as XS import qualified Data.X509.Validation as XV +import Foreign (FunPtr, allocaBytes, fillBytes, freeHaskellFunPtr, nullPtr) +import Foreign.C.Types (CInt, CSize (..)) import qualified SMPClient import qualified Simplex.Messaging.Crypto as C import qualified Simplex.Messaging.Crypto.Lazy as LC @@ -32,9 +36,12 @@ import Simplex.Messaging.Crypto.BBS import Simplex.Messaging.Crypto.Entitlement import Simplex.Messaging.Crypto.SNTRUP761.Bindings import Simplex.Messaging.Crypto.SNTRUP761.Bindings.Defines +import Simplex.Messaging.Crypto.SNTRUP761.Bindings.FFI (c_sntrup761_keypair) +import Simplex.Messaging.Crypto.SNTRUP761.Bindings.RNG (RNGFunc) import Simplex.Messaging.Encoding (Large (..), smpDecode, smpEncode) import Simplex.Messaging.Encoding.String (strDecode, strEncode) import Simplex.Messaging.Transport.Client +import System.Timeout (timeout) import Test.Hspec hiding (fit, it) import Test.Hspec.QuickCheck (modifyMaxSuccess) import Test.QuickCheck hiding (Large) @@ -113,6 +120,7 @@ cryptoTests = do describe "sntrup761" $ do it "should enc/dec key" testSNTRUP761 it "should reject malformed KEM encodings" testSNTRUP761RejectsMalformedEncodings + it "should fail key generation with degenerate RNG" testSNTRUP761KeypairDegenerateRNG describe "BBS+" $ do it "should sign and verify" testBBSSignVerify it "should derive public key from secret key" testBBSPublicKeyDerivation @@ -298,6 +306,26 @@ testSNTRUP761 = do KEMSharedKey k' <- sntrup761Dec c sk k' `shouldBe` k +foreign import ccall "wrapper" + mkRNGFunc :: RNGFunc -> IO (FunPtr RNGFunc) + +testSNTRUP761KeypairDegenerateRNG :: IO () +testSNTRUP761KeypairDegenerateRNG = do + -- constant byte 0 draws invertible g = -(1 + x + ... + x^760), byte 0x20 draws g = 0 + keypairWithConstantRNG 0 `shouldReturn` Just 0 + keypairWithConstantRNG 0x20 `shouldReturn` Just (-1) + where + keypairWithConstantRNG :: Word8 -> IO (Maybe CInt) + keypairWithConstantRNG b = do + result <- newEmptyMVar + -- timeout cannot interrupt a foreign call, so the call runs in another thread + _ <- forkIO $ + bracket (mkRNGFunc $ \_ sz buf -> fillBytes buf b (fromIntegral sz)) freeHaskellFunPtr $ \rng -> + allocaBytes c_SNTRUP761_PUBLICKEY_SIZE $ \pkPtr -> + allocaBytes c_SNTRUP761_SECRETKEY_SIZE $ \skPtr -> + c_sntrup761_keypair pkPtr skPtr nullPtr rng >>= putMVar result + timeout 10000000 $ takeMVar result + testSNTRUP761RejectsMalformedEncodings :: IO () testSNTRUP761RejectsMalformedEncodings = do smpDecode @KEMPublicKey (smpEncode $ Large shortPublicKey) `shouldSatisfy` isLeft diff --git a/tests/CoreTests/MsgStoreTests.hs b/tests/CoreTests/MsgStoreTests.hs index 5e466ac35..64fcf7c9f 100644 --- a/tests/CoreTests/MsgStoreTests.hs +++ b/tests/CoreTests/MsgStoreTests.hs @@ -73,6 +73,7 @@ msgStoreTests = do it "should get queue and store/read messages" testGetQueue it "should write/ack messages" testWriteAckMessages it "should resolve sender ID equal to link ID of another queue" testLinkIdSenderIdCollision + it "should not add link data to secured messaging queue" testLinkDataSecuredQueue -- TODO constrain to STM stores? withMsgStore :: MsgStoreClass s => MsgStoreConfig s -> (s -> IO ()) -> IO () @@ -211,6 +212,33 @@ testWriteAckMessages ms = do void $ ExceptT $ deleteQueue ms q1 void $ ExceptT $ deleteQueue ms q2 +testLinkDataSecuredQueue :: MsgStoreClass s => s -> IO () +testLinkDataSecuredQueue ms = do + g <- C.newRandom + (sKey, _) <- atomically $ C.generateAuthKeyPair C.SEd25519 g + let st = queueStore ms + ld = (EncDataBytes "fixed data", EncDataBytes "user data") + rndId = atomically $ EntityId <$> C.randomBytes 24 g + (rId, qr) <- testNewQueueRec g QMMessaging + (cId, cqr) <- testNewQueueRec g QMContact + lnkId <- rndId + cLnkId <- rndId + runRight_ $ do + q <- ExceptT $ addQueue ms rId qr + -- the handle is read before SKEY, as in a command that raced with it + staleQ <- ExceptT $ getQueue ms SRecipient rId + ExceptT $ secureQueue st q sKey + liftIO $ addQueueLinkData st staleQ lnkId ld `shouldReturn` Left AUTH + freshQ <- ExceptT $ getQueue ms SRecipient rId + liftIO $ getQueueLinkData st freshQ lnkId `shouldReturn` Left AUTH + cq <- ExceptT $ addQueue ms cId cqr + ExceptT $ secureQueue st cq sKey + ExceptT $ addQueueLinkData st cq cLnkId ld + ld' <- ExceptT $ getQueueLinkData st cq cLnkId + liftIO $ ld' `shouldBe` ld + void $ ExceptT $ deleteQueue ms q + void $ ExceptT $ deleteQueue ms cq + -- sizes of queues, senders, links and notifiers maps type QueueMapSizes = (Int, Int, Int, Int) diff --git a/tests/CoreTests/SOCKSSettings.hs b/tests/CoreTests/SOCKSSettings.hs index 3dbf5e5e7..a56be966e 100644 --- a/tests/CoreTests/SOCKSSettings.hs +++ b/tests/CoreTests/SOCKSSettings.hs @@ -2,15 +2,18 @@ {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedLists #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PatternSynonyms #-} {-# LANGUAGE TypeApplications #-} {-# OPTIONS_GHC -fno-warn-ambiguous-fields #-} module CoreTests.SOCKSSettings where import Network.Socket (SockAddr (..), tupleToHostAddress) +import Simplex.Messaging.Agent.Client (ipAddressProtected) import Simplex.Messaging.Client +import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Encoding.String -import Simplex.Messaging.Protocol (ErrorType) +import Simplex.Messaging.Protocol (ErrorType, pattern SMPServer) import Simplex.Messaging.Transport.Client import Test.Hspec hiding (fit, it) import Util @@ -19,6 +22,7 @@ socksSettingsTests :: Spec socksSettingsTests = do describe "hostMode and requiredHostMode settings" testHostMode describe "socksMode setting, independent of hostMode setting" testSocksMode + describe "ipAddressProtected, consistent with chosen host and socksMode" testIPAddressProtected describe "socks proxy address encoding" testSocksProxyEncoding testPublicHost :: TransportHost @@ -94,6 +98,19 @@ testSocksMode = do let TransportClientConfig {socksProxy} = transportClientConfig cfg NRMInteractive host False Nothing in socksProxy +testIPAddressProtected :: Spec +testIPAddressProtected = do + it "should be protected if SOCKS proxy is used for the chosen host" $ do + protected SMAlways HMOnionViaSocks [testPublicHost] `shouldBe` True + protected SMOnion HMOnionViaSocks [testPublicHost, testOnionHost] `shouldBe` True + protected SMOnion HMPublic [testOnionHost] `shouldBe` True + it "should not be protected if SOCKS proxy is not used for the chosen host" $ do + protected SMOnion HMOnionViaSocks [testPublicHost] `shouldBe` False + protected SMOnion HMPublic [testPublicHost, testOnionHost] `shouldBe` False + where + protected socksMode hostMode hosts = + ipAddressProtected defaultNetworkConfig {socksProxy = Just defaultSocksProxyWithAuth, socksMode, hostMode} (SMPServer hosts "" (C.KeyHash "")) + testSocksProxyEncoding :: Spec testSocksProxyEncoding = do it "should decode SOCKS proxy with isolate-by-auth mode" $ do diff --git a/tests/RemoteControl.hs b/tests/RemoteControl.hs index 630e774c0..9f7ecc029 100644 --- a/tests/RemoteControl.hs +++ b/tests/RemoteControl.hs @@ -9,13 +9,15 @@ module RemoteControl where import AgentTests.FunctionalAPITests (runRight) import Control.Logger.Simple +import Control.Monad (void) +import Control.Monad.Trans.Except (runExceptT) import Crypto.Random (ChaChaDRG) import qualified Data.Aeson as J import qualified Data.ByteString.Char8 as B import qualified Data.ByteString.Lazy.Char8 as LB import Data.List (stripPrefix) import Data.List.NonEmpty (NonEmpty (..)) -import Data.Time.Clock.System (SystemTime (..)) +import Data.Time.Clock.System (SystemTime (..), getSystemTime) import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Encoding.String (StrEncoding (..)) import Simplex.Messaging.Transport (TSbChainKeys (..)) @@ -24,8 +26,10 @@ import qualified Simplex.RemoteControl.Client as HC (RCHostClient (action)) import qualified Simplex.RemoteControl.Client as RC import Simplex.RemoteControl.Discovery (mkLastLocalHost, preferAddress) import Simplex.RemoteControl.Invitation - ( RCInvitation (..), + ( RCEncInvitation (..), + RCInvitation (..), RCSignedInvitation, + signInvitation, verifySignedInvitation, ) import Simplex.RemoteControl.Types @@ -45,6 +49,7 @@ remoteControlTests = do it "should connect to existing pairing" testExistingPairing describe "Multicast discovery" $ do it "should find paired host and connect" testMulticast + it "should accept announcement only within timestamp window" testAnnouncementTimestamp testPreferAddress :: Spec testPreferAddress = do @@ -244,6 +249,26 @@ testMulticast = do Nothing -> fail "timeout" Just _ -> pure () +testAnnouncementTimestamp :: IO () +testAnnouncementTimestamp = do + drg <- C.newRandom + RCHostPairing {caKey, caCert, idPrivKey} <- RC.newRCHostPairing drg + (hostDhPubKey, dhPrivKey) <- atomically $ C.generateKeyPair @'C.X25519 drg + (skey, sessPrivKey) <- atomically $ C.generateKeyPair @'C.Ed25519 drg + (dh, ctrlDhPrivKey) <- atomically $ C.generateKeyPair @'C.X25519 drg + nonce <- atomically $ C.randomCbNonce drg + now <- systemSeconds <$> getSystemTime + let pairing = RCCtrlPairing {caKey, caCert, ctrlFingerprint = C.KeyHash "test-ca", idPubKey = C.publicKey idPrivKey, dhPrivKey, prevDhPrivKey = Nothing} + announce offset = do + let inv = RCInvitation {ca = C.KeyHash "test-ca", host = "127.0.0.1", port = 5223, v = supportedRCPVRange, app = J.String "app", ts = MkSystemTime (now + offset) 0, skey, idkey = C.publicKey idPrivKey, dh} + encInvitation <- either (fail . show) pure $ C.cbEncrypt (C.dh' hostDhPubKey ctrlDhPrivKey) nonce (strEncode $ signInvitation sessPrivKey idPrivKey inv) 900 + runExceptT . void $ RC.findRCCtrlPairing (pairing :| []) RCEncInvitation {dhPubKey = dh, nonce, encInvitation} + announce 0 `shouldReturn` Right () + announce (-3600) `shouldReturn` Right () + announce 3500 `shouldReturn` Right () + announce (-3700) `shouldReturn` Left RCEInvitation + announce 3700 `shouldReturn` Left RCEInvitation + runCtrl :: TVar ChaChaDRG -> Bool -> RCHostPairing -> MVar RCSignedInvitation -> IO (Async RCHostPairing) runCtrl drg multicast hp invVar = async . runRight $ do (_found, inv, hc, r) <- RC.connectRCHost drg hp (J.String "app") multicast Nothing Nothing diff --git a/tests/SMPClient.hs b/tests/SMPClient.hs index 792debce5..6add356c8 100644 --- a/tests/SMPClient.hs +++ b/tests/SMPClient.hs @@ -352,6 +352,16 @@ proxyCfgShortTimeout = nt = NetworkTimeout {backgroundTimeout = 4_000000, interactiveTimeout = 4_000000} in cfg' {smpAgentCfg = aCfg {smpCfg = cCfg {networkConfig = (networkConfig cCfg) {tcpConnectTimeout = nt}}}} +proxyCfgVPrev :: AStoreType -> AServerConfig +proxyCfgVPrev msType = + updateCfg (proxyCfgMS msType) $ \cfg' -> + let aCfg = smpAgentCfg cfg' + cCfg = smpCfg aCfg + in cfg' + { smpServerVRange = prevRange $ smpServerVRange cfg', + smpAgentCfg = aCfg {smpCfg = cCfg {serverVRange = prevRange $ serverVRange cCfg}} + } + withSmpServerStoreMsgLogOn :: HasCallStack => (ASrvTransport, AStoreType) -> ServiceName -> (HasCallStack => ThreadId -> IO a) -> IO a withSmpServerStoreMsgLogOn (t, msType) = withSmpServerConfigOn t $ updateCfg (cfgMS msType) $ \cfg' -> cfg' {storeNtfsFile = Just testStoreNtfsFile, serverStatsBackupFile = Just testServerStatsBackupFile} diff --git a/tests/SMPProxyTests.hs b/tests/SMPProxyTests.hs index e721efa6a..fa6a11a75 100644 --- a/tests/SMPProxyTests.hs +++ b/tests/SMPProxyTests.hs @@ -73,6 +73,8 @@ smpProxyTests = do xit "no SMP service at host/port" todo xit "bad SMP fingerprint" todo xit "batching proxy requests" todo + it "relay rejects forwarded command with changed version" $ \_ -> + testChangedFwdVersion describe "deliver message via SMP proxy" $ do let srv1 = SMPServer testHost testPort testKeyHash srv2 = SMPServer testHost2 testPort2 testKeyHash @@ -95,6 +97,11 @@ smpProxyTests = do deliverMessageViaProxy proxyServ relayServ C.SEd25519 msg1 msg2 it "max message size, X25519 keys" . twoServersFirstProxy $ deliverMessageViaProxy proxyServ relayServ C.SX25519 msg1 msg2 + describe "version compatibility" $ do + let deliver clientVR = deliverMessagesViaProxyVR clientVR srv1 srv2 C.SEd448 ["hello 1"] ["hello 2"] + it "prev client" . twoServersFirstProxy $ deliver (prevRange supportedClientSMPRelayVRange) + it "prev proxy" . twoServersPrevProxy $ deliver supportedClientSMPRelayVRange + it "prev relay" . twoServersPrevRelay $ deliver supportedClientSMPRelayVRange describe "stress test 1k" $ do let deliver n = deliverMessagesViaProxy srv1 srv2 C.SEd448 [] (map bshow [1 :: Int .. n]) it "1x1000" . twoServersFirstProxy $ deliver 1000 @@ -155,6 +162,9 @@ smpProxyTests = do twoServersFirstProxy test msType = twoServers_ (proxyCfgMS msType) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128}) test msType twoServersMoreConc test msType = twoServers_ (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {serverClientConcurrency = 128}) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128}) test msType twoServersNoConc test msType = twoServers_ (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {serverClientConcurrency = 1}) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128}) test msType + twoServersPrevProxy test msType = twoServers_ (proxyCfgVPrev msType) (cfgMS msType) test msType + twoServersPrevRelay test msType = twoServers_ (proxyCfgMS msType) (prevServerVRange $ cfgMS msType) test msType + prevServerVRange cfg' = updateCfg cfg' $ \cfg_ -> cfg_ {smpServerVRange = prevRange $ smpServerVRange cfg_} twoServers_ :: AServerConfig -> AServerConfig -> IO () -> AStoreType -> IO () twoServers_ cfg1 cfg2 runTest (ASType qsType _) = withSmpServerConfigOn (transport @TLS) cfg1 testPort $ \_ -> @@ -167,11 +177,14 @@ deliverMessageViaProxy :: (C.AlgorithmI a, C.AuthAlgorithm a) => SMPServer -> SM deliverMessageViaProxy proxyServ relayServ alg msg msg' = deliverMessagesViaProxy proxyServ relayServ alg [msg] [msg'] deliverMessagesViaProxy :: (C.AlgorithmI a, C.AuthAlgorithm a) => SMPServer -> SMPServer -> C.SAlgorithm a -> [ByteString] -> [ByteString] -> IO () -deliverMessagesViaProxy proxyServ relayServ alg unsecuredMsgs securedMsgs = do +deliverMessagesViaProxy = deliverMessagesViaProxyVR $ mkVersionRange minServerSMPRelayVersion currentClientSMPRelayVersion + +deliverMessagesViaProxyVR :: (C.AlgorithmI a, C.AuthAlgorithm a) => VersionRangeSMP -> SMPServer -> SMPServer -> C.SAlgorithm a -> [ByteString] -> [ByteString] -> IO () +deliverMessagesViaProxyVR clientVR proxyServ relayServ alg unsecuredMsgs securedMsgs = do g <- C.newRandom -- set up proxy ts <- getCurrentTime - pc' <- getProtocolClient g NRMInteractive (1, proxyServ, Nothing) defaultSMPClientConfig {serverVRange = mkVersionRange minServerSMPRelayVersion currentClientSMPRelayVersion} [] Nothing ts (\_ -> pure ()) + pc' <- getProtocolClient g NRMInteractive (1, proxyServ, Nothing) defaultSMPClientConfig {serverVRange = clientVR, proxiedRelayVRange = clientVR} [] Nothing ts (\_ -> pure ()) pc <- either (fail . show) pure pc' THAuthClient {} <- maybe (fail "getProtocolClient returned no thAuth") pure $ thAuth $ thParams pc -- set up relay @@ -448,6 +461,21 @@ requestRelaySession = testSMPClient_ "localhost" testPort supportedServerSMPRelayVRange Nothing $ \(th :: THandleSMP TLS 'TClient) -> (\(_, _, reply) -> reply) <$> sendRecv th (Nothing, "1", NoEntity, SMP.PRXY testSMPServer2 Nothing) +testChangedFwdVersion :: IO () +testChangedFwdVersion = + withSmpServerConfigOn (transport @TLS) cfg testPort $ \_ -> do + g <- C.newRandom + ts <- getCurrentTime + rc <- either (fail . show) pure =<< getProtocolClient g NRMInteractive (1, testSMPServer, Nothing) defaultSMPClientConfig [] Nothing ts (\_ -> pure ()) + THAuthClient {peerServerPubKey} <- maybe (fail "getProtocolClient returned no thAuth") pure $ thAuth $ thParams rc + (cmdPubKey, cmdPrivKey) <- atomically $ C.generateKeyPair g + nonce@(C.CbNonce corrId) <- atomically $ C.randomCbNonce g + let v = currentClientSMPRelayVersion + et <- either (fail . show) (pure . SMP.EncTransmission) $ C.cbEncrypt (C.dh' peerServerPubKey cmdPrivKey) (SMP.encTransmissionNonce v nonce) "" SMP.paddedProxiedTLength + let forward fwdVersion = forwardSMPTransmission rc (SMP.CorrId corrId) fwdVersion cmdPubKey et + _ <- runExceptT' $ forward v + runExceptT (forward $ prevVersion v) `shouldReturn` Left (PCEProtocolError SMP.CRYPTO) + -- Shared "phase 2" of the reconnection tests: start a healthy relay, confirm it is reachable -- directly (PING, not via the proxy) so a proxy failure can only mean the proxy didn't reconnect, -- let any stored connection error expire, then require the proxy to establish the session (PKEY).