diff --git a/CHANGELOG.md b/CHANGELOG.md index e161a26df..707a6ff5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +# 5.8.1 + +Agent: +- API to reconnect one server. +- Better error handling of file errors and remote control connection errors. +- Only start uploading file once all chunks were registered on the servers. + +SMP server: +- additional stats for sent message notifications. +- fix server page layout. + # 5.8.0 Version 5.8.0.10 diff --git a/README.md b/README.md index d946b8ec6..2c475ffd5 100644 --- a/README.md +++ b/README.md @@ -208,14 +208,17 @@ On Linux, you can build smp server using Docker. #### Using your distribution -1. Install [Haskell GHCup](https://www.haskell.org/ghcup/), GHC 8.10.7 and cabal: +1. Install dependencies and build tools (`GHC`, `cabal` and dev libs): ```sh - curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh - ghcup install ghc 8.10.7 - ghcup install cabal - ghcup set ghc 8.10.7 - ghcup set cabal + # On Ubuntu. Depending on your distribution, use your package manager to determine package names. + sudo apt-get update && apt-get install -y build-essential curl libffi-dev libffi7 libgmp3-dev libgmp10 libncurses-dev libncurses5 libtinfo5 pkg-config zlib1g-dev libnuma-dev libssl-dev + export BOOTSTRAP_HASKELL_GHC_VERSION=9.6.3 + export BOOTSTRAP_HASKELL_CABAL_VERSION=3.10.3.0 + curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 sh + ghcup set ghc "${BOOTSTRAP_HASKELL_GHC_VERSION}" + ghcup set cabal "${BOOTSTRAP_HASKELL_CABAL_VERSION}" + source ~/.ghcup/env ``` 2. Build the project: @@ -224,10 +227,20 @@ On Linux, you can build smp server using Docker. git clone https://github.com/simplex-chat/simplexmq cd simplexmq git checkout stable - # On Ubuntu. Depending on your distribution, use your package manager to determine package names. - apt-get update && apt-get install -y build-essential libgmp3-dev zlib1g-dev cabal update - cabal install + cabal build exe:smp-server exe:xftp-server + ``` + +3. List compiled binaries: + + `smp-server` + ```sh + cabal list-bin exe:smp-server + ``` + + `xftp-server` + ```sh + cabal list-bin exe:xftp-server ``` - Initialize SMP server with `smp-server init [-l] -n ` or `smp-server init [-l] --ip ` - depending on how you initialize it, either FQDN or IP will be used for server's address. diff --git a/apps/smp-server/static/index.html b/apps/smp-server/static/index.html index e3889de2a..e276ff596 100644 --- a/apps/smp-server/static/index.html +++ b/apps/smp-server/static/index.html @@ -15,6 +15,12 @@ status (receive/send): NONE/NONE1. request connectionfrom agent2. create Alice's SMP queuestatus: NEW/NONE3. out-of-band invitation4. accept connectionstatus: NONE/NEW5. secure Alice's SMP queuestatus: NONE/SECURED6. create Bob's SMP queuestatus: NEW/SECURED7. confirm Alice's SMP queuestatus: NEW/CONFIRMEDstatus: CONFIRMED/NEW8. secure Bob's SMP queuestatus: CONFIRMED/SECUREDstatus: CONFIRMED/CONFIRMED9. notify Aliceabout connection success(no HELLO needed in v6)status: ACTIVE/ACTIVE10. notify Bobabout connection successstatus: CONFIRMED/CONFIRMEDstatus: ACTIVE/ACTIVEcreateConnectionNEW: create SMP queueallow sender to secureIDS: SMP queue IDsINV: invitationto connectOOB: invitation to connectjoinConnection:via invitation infoSKEY: secure queue (this command needs to be proxied)NEW: create SMP queueallow sender to secureIDS: SMP queue IDsSEND: Bob's info without sender's key (SMP confirmation with reply queues)MSG: Bob's info withoutsender server keyACK: confirm messageCONF: connection request IDand Bob's infoallowConnection: accept connection request,send Alice's infoSKEY: secure queue (this command needs to be proxied)SEND: Alice's info without sender's server key (SMP confirmation without reply queues)CON: connectedMSG: Alice's info withoutsender's server keyINFO: Alice's infoACK: confirm messageCON: connected \ No newline at end of file diff --git a/protocol/diagrams/duplex-messaging/duplex-creating-v2.mmd b/protocol/diagrams/duplex-messaging/duplex-creating-v2.mmd deleted file mode 100644 index 09ee913ff..000000000 --- a/protocol/diagrams/duplex-messaging/duplex-creating-v2.mmd +++ /dev/null @@ -1,71 +0,0 @@ -sequenceDiagram - participant A as Alice - participant AA as Alice's
agent - participant AS as Alice's
server - participant BS as Bob's
server - participant BA as Bob's
agent - participant B as Bob - - note over AA, BA: status (receive/send): NONE/NONE - - note over A, AA: 1. request connection
from agent - A ->> AA: NEW: create
duplex connection - - note over AA, AS: 2. create Alice's SMP queue - AA ->> AS: NEW: create SMP queue - AS ->> AA: IDS: SMP queue IDs - note over AA: status: NEW/NONE - - AA ->> A: INV: invitation
to connect - - note over A, B: 3. out-of-band invitation - A ->> B: OOB: invitation to connect - - note over BA, B: 4. accept connection - B ->> BA: JOIN:
via invitation info - note over BA: status: NONE/NEW - - note over BA, BS: 5. create Bob's SMP queue - BA ->> BS: NEW: create SMP queue - BS ->> BA: IDS: SMP queue IDs - note over BA: status: NEW/NEW - - note over BA, AA: 6. establish Alice's SMP queue - BA ->> AS: SEND: Bob's info and sender server key (SMP confirmation with reply queues) - note over BA: status: NEW/CONFIRMED - - AS ->> AA: MSG: Bob's info and
sender server key - note over AA: status: CONFIRMED/NONE - AA ->> AS: ACK: confirm message - AA ->> A: CONF: connection request ID
and Bob's info - A ->> AA: LET: accept connection request,
send Alice's info - AA ->> AS: KEY: secure queue - note over AA: status: SECURED/NONE - - AA ->> BS: SEND: Alice's info and sender's server key (SMP confirmation without reply queues) - note over AA: status: SECURED/CONFIRMED - - BS ->> BA: MSG: Alice's info and
sender's server key - note over BA: status: CONFIRMED/CONFIRMED - BA ->> B: INFO: Alice's info - BA ->> BS: ACK: confirm message - BA ->> BS: KEY: secure queue - note over BA: status: SECURED/CONFIRMED - - BA ->> AS: SEND: HELLO: only needs to be sent once in v2 - - note over BA: status: SECURED/ACTIVE - note over BA, B: 7a. notify Bob
about connection success - BA ->> B: CON: connected - - AS ->> AA: MSG: HELLO: Alice's agent
knows Bob can send - note over AA: status: SECURED/ACTIVE - AA ->> AS: ACK: confirm message - note over A, AA: 7a. notify Alice
about connection success - AA ->> A: CON: connected - - AA ->> BS: SEND: HELLO: only needs to be sent once in v2 - note over AA: status: ACTIVE/ACTIVE - BS ->> BA: MSG: HELLO: Bob's agent
knows Alice can send - note over BA: status: ACTIVE/ACTIVE - BA ->> BS: ACK: confirm message diff --git a/protocol/diagrams/duplex-messaging/duplex-creating.mmd b/protocol/diagrams/duplex-messaging/duplex-creating.mmd index 738bad325..5cddf6aa4 100644 --- a/protocol/diagrams/duplex-messaging/duplex-creating.mmd +++ b/protocol/diagrams/duplex-messaging/duplex-creating.mmd @@ -8,8 +8,8 @@ sequenceDiagram note over AA, BA: status (receive/send): NONE/NONE - note over A, AA: 1. request connection from agent - A ->> AA: NEW: create
duplex connection + note over A, AA: 1. request connection
from agent + A ->> AA: createConnection note over AA, AS: 2. create Alice's SMP queue AA ->> AS: NEW: create SMP queue @@ -17,63 +17,58 @@ sequenceDiagram note over AA: status: NEW/NONE AA ->> A: INV: invitation
to connect - note over AA: status: PENDING/NONE note over A, B: 3. out-of-band invitation A ->> B: OOB: invitation to connect note over BA, B: 4. accept connection - B ->> BA: JOIN:
via invitation info + B ->> BA: joinConnection:
via invitation info note over BA: status: NONE/NEW - note over BA, AA: 5. establish Alice's SMP queue - BA ->> AS: SEND: Bob's info and sender server key (SMP confirmation) - note over BA: status: NONE/CONFIRMED - activate BA + note over BA, BS: 5. create Bob's SMP queue + BA ->> BS: NEW: create SMP queue + BS ->> BA: IDS: SMP queue IDs + note over BA: status: NEW/NEW + + note over BA, AA: 6. confirm Alice's SMP queue + BA ->> AS: SEND: Bob's info and sender server key (SMP confirmation with reply queues) + note over BA: status: NEW/CONFIRMED + AS ->> AA: MSG: Bob's info and
sender server key note over AA: status: CONFIRMED/NONE AA ->> AS: ACK: confirm message AA ->> A: CONF: connection request ID
and Bob's info - A ->> AA: LET: accept connection request,
send Alice's info + A ->> AA: allowConnection: accept connection request,
send Alice's info AA ->> AS: KEY: secure queue note over AA: status: SECURED/NONE - BA ->> AS: SEND: HELLO: try sending until successful - deactivate BA - note over BA: status: NONE/ACTIVE - AS ->> AA: MSG: HELLO: Alice's agent
knows Bob can send - note over AA: status: ACTIVE/NONE - AA ->> AS: ACK: confirm message + AA ->> BS: SEND: Alice's info and sender's server key (SMP confirmation without reply queues) + note over AA: status: SECURED/CONFIRMED - note over BA, BS: 6. create Bob's SMP queue - BA ->> BS: NEW: create SMP queue - BS ->> BA: IDS: SMP queue IDs - note over BA: status: NEW/ACTIVE - - note over AA, BA: 7. establish Bob's SMP queue - BA ->> AS: SEND: REPLY: invitation to the connect - note over BA: status: PENDING/ACTIVE - AS ->> AA: MSG: REPLY: invitation
to connect - note over AA: status: ACTIVE/NEW - AA ->> AS: ACK: confirm message - - AA ->> BS: SEND: Alice's info and sender's server key - note over AA: status: ACTIVE/CONFIRMED - activate AA + note over BA, AA: 7. confirm Bob's SMP queue BS ->> BA: MSG: Alice's info and
sender's server key - note over BA: status: CONFIRMED/ACTIVE + note over BA: status: CONFIRMED/CONFIRMED BA ->> B: INFO: Alice's info BA ->> BS: ACK: confirm message BA ->> BS: KEY: secure queue + note over BA: status: SECURED/CONFIRMED + + BA ->> AS: SEND: HELLO message + note over BA: status: SECURED/ACTIVE - AA ->> BS: SEND: HELLO: try sending until successful - deactivate AA + AS ->> AA: MSG: HELLO: Alice's agent
knows Bob can send + note over AA: status: SECURED/ACTIVE + AA ->> AS: ACK: confirm message + AA ->> BS: SEND: HELLO + + note over A, AA: 8. notify Alice
about connection success + AA ->> A: CON: connected note over AA: status: ACTIVE/ACTIVE + BS ->> BA: MSG: HELLO: Bob's agent
knows Alice can send note over BA: status: ACTIVE/ACTIVE BA ->> BS: ACK: confirm message - note over A, B: 8. notify users about connection success - AA ->> A: CON: connected + note over BA, B: 9. notify Bob
about connection success BA ->> B: CON: connected diff --git a/protocol/diagrams/duplex-messaging/duplex-creating.svg b/protocol/diagrams/duplex-messaging/duplex-creating.svg index 138935d3b..bea3bb23d 100644 --- a/protocol/diagrams/duplex-messaging/duplex-creating.svg +++ b/protocol/diagrams/duplex-messaging/duplex-creating.svg @@ -1 +1,3 @@ -AliceAlice'sagentAlice'sserverBob'sserverBob'sagentBobstatus (receive/send): NONE/NONE1. request connection from agentNEW: createduplex connection2. create Alice's SMP queueNEW: create SMP queueIDS: SMP queue IDsstatus: NEW/NONEINV: invitationto connectstatus: PENDING/NONE3. out-of-band invitationOOB: invitation to connect4. accept connectionJOIN:via invitation infostatus: NONE/NEW5. establish Alice's SMP queueSEND: Bob's info and sender server key (SMP confirmation)status: NONE/CONFIRMEDMSG: Bob's info andsender server keystatus: CONFIRMED/NONEACK: confirm messageREQ: connection request IDand Bob's infoACPT: accept connection request,send Alice's infoKEY: secure queuestatus: SECURED/NONESEND: HELLO: try sending until successfulstatus: NONE/ACTIVEMSG: HELLO: Alice's agentknows Bob can sendstatus: ACTIVE/NONEACK: confirm message6. create Bob's SMP queueNEW: create SMP queueIDS: SMP queue IDsstatus: NEW/ACTIVE7. establish Bob's SMP queueSEND: REPLY: invitation to the connectstatus: PENDING/ACTIVEMSG: REPLY: invitationto connectstatus: ACTIVE/NEWACK: confirm messageSEND: Alice's info and sender's server keystatus: ACTIVE/CONFIRMEDMSG: Alice's info andsender's server keystatus: CONFIRMED/ACTIVEINFO: Alice's infoACK: confirm messageKEY: secure queuestatus: SECURED/ACTIVESEND: HELLO: try sending until successfulstatus: ACTIVE/ACTIVEMSG: HELLO: Bob's agentknows Alice can sendstatus: ACTIVE/ACTIVEACK: confirm message8. notify users about connection successCON: connectedCON: connectedAliceAlice'sagentAlice'sserverBob'sserverBob'sagentBob \ No newline at end of file + + +BobBob'sagentBob'sserverAlice'sserverAlice'sagentAliceBobBob'sagentBob'sserverAlice'sserverAlice'sagentAlicestatus (receive/send): NONE/NONE1. request connectionfrom agent2. create Alice's SMP queuestatus: NEW/NONE3. out-of-band invitation4. accept connectionstatus: NONE/NEW5. secure Alice's SMP queuestatus: NONE/SECURED6. create Bob's SMP queuestatus: NEW/SECURED7. confirm Alice's SMP queuestatus: NEW/CONFIRMEDstatus: CONFIRMED/NEW8. secure Bob's SMP queuestatus: CONFIRMED/SECUREDstatus: CONFIRMED/CONFIRMED9. notify Aliceabout connection success(no HELLO needed in v6)status: ACTIVE/ACTIVE10. notify Bobabout connection successstatus: CONFIRMED/CONFIRMEDstatus: ACTIVE/ACTIVEcreateConnectionNEW: create SMP queueallow sender to secureIDS: SMP queue IDsINV: invitationto connectOOB: invitation to connectjoinConnection:via invitation infoSKEY: secure queue (this command needs to be proxied)NEW: create SMP queueallow sender to secureIDS: SMP queue IDsSEND: Bob's info without sender's key (SMP confirmation with reply queues)MSG: Bob's info withoutsender server keyACK: confirm messageCONF: connection request IDand Bob's infoallowConnection: accept connection request,send Alice's infoSKEY: secure queue (this command needs to be proxied)SEND: Alice's info without sender's server key (SMP confirmation without reply queues)CON: connectedMSG: Alice's info withoutsender's server keyINFO: Alice's infoACK: confirm messageCON: connected \ No newline at end of file diff --git a/protocol/diagrams/duplex-messaging/queue-rotation-fast.mmd b/protocol/diagrams/duplex-messaging/queue-rotation-fast.mmd new file mode 100644 index 000000000..75887dd0b --- /dev/null +++ b/protocol/diagrams/duplex-messaging/queue-rotation-fast.mmd @@ -0,0 +1,17 @@ +sequenceDiagram + participant A as Alice + participant R as Current server
that has A's
receive queue + participant R' as New server
that has the new A's
receive queue + participant S as Server
that has A's send queue
(B's receive queue) + participant B as Bob + + A ->> R': NEW: create new queue
(allow SKEY) + A ->> S: SEND: QADD (R'): send address
of the new queue(s) + S ->> B: MSG: QADD (R') + B ->> R': SKEY: secure new queue + B ->> R': SEND: QTEST + R' ->> A: MSG: QTEST + A ->> R: DEL: delete the old queue + B ->> R': SEND: send messages to the new queue + R' ->> A: MSG: receive messages from the new queue + \ No newline at end of file diff --git a/protocol/diagrams/duplex-messaging/queue-rotation-fast.svg b/protocol/diagrams/duplex-messaging/queue-rotation-fast.svg new file mode 100644 index 000000000..823074823 --- /dev/null +++ b/protocol/diagrams/duplex-messaging/queue-rotation-fast.svg @@ -0,0 +1,3 @@ + + +BobServerthat has A's send queue(B's receive queue)New serverthat has the new A'sreceive queueCurrent serverthat has A'sreceive queueAliceBobServerthat has A's send queue(B's receive queue)New serverthat has the new A'sreceive queueCurrent serverthat has A'sreceive queueAliceNEW: create new queue(allow SKEY)SEND: QADD (R'): send addressof the new queue(s)MSG: QADD (R')SKEY: secure new queueSEND: QTESTMSG: QTESTDEL: delete the old queueSEND: send messages to the new queueMSG: receive messages from the new queue \ No newline at end of file diff --git a/protocol/diagrams/duplex-messaging/queue-rotation.mmd b/protocol/diagrams/duplex-messaging/queue-rotation.mmd new file mode 100644 index 000000000..9572a2c48 --- /dev/null +++ b/protocol/diagrams/duplex-messaging/queue-rotation.mmd @@ -0,0 +1,21 @@ +sequenceDiagram + participant A as Alice + participant R as Current server
that has A's
receive queue + participant R' as New server
that has the new A's
receive queue + participant S as Server
that has A's send queue
(B's receive queue) + participant B as Bob + + A ->> R': NEW: create new queue + A ->> S: SEND: QADD (R'): send address
of the new queue(s) + S ->> B: MSG: QADD (R') + B ->> R: SEND: QKEY (R'): sender's key
for the new queue(s) + R ->> A: MSG: QKEY(R') + A ->> R': KEY: secure new queue + A ->> S: SEND: QUSE (R'): instruction to use new queue(s) + S ->> B: MSG: QUSE (R') + B ->> R': SEND: QTEST + R' ->> A: MSG: QTEST + A ->> R: DEL: delete the old queue + B ->> R': SEND: send messages to the new queue + R' ->> A: MSG: receive messages from the new queue + \ No newline at end of file diff --git a/protocol/diagrams/duplex-messaging/queue-rotation.svg b/protocol/diagrams/duplex-messaging/queue-rotation.svg new file mode 100644 index 000000000..d477cd622 --- /dev/null +++ b/protocol/diagrams/duplex-messaging/queue-rotation.svg @@ -0,0 +1,3 @@ + + +BobServerthat has A's send queue(B's receive queue)New serverthat has the new A'sreceive queueCurrent serverthat has A'sreceive queueAliceBobServerthat has A's send queue(B's receive queue)New serverthat has the new A'sreceive queueCurrent serverthat has A'sreceive queueAliceNEW: create new queueSEND: QADD (R'): send addressof the new queue(s)MSG: QADD (R')SEND: QKEY (R'): sender's keyfor the new queue(s)MSG: QKEY(R')KEY: secure new queueSEND: QUSE (R'): instruction to use new queue(s)MSG: QUSE (R')SEND: QTESTMSG: QTESTDEL: delete the old queueSEND: send messages to the new queueMSG: receive messages from the new queue \ No newline at end of file diff --git a/protocol/diagrams/notifications/register-token-detailed.mmd b/protocol/diagrams/notifications/register-token-detailed.mmd new file mode 100644 index 000000000..bceef296f --- /dev/null +++ b/protocol/diagrams/notifications/register-token-detailed.mmd @@ -0,0 +1,30 @@ +sequenceDiagram + participant M as mobile app + participant C as chat core + participant A as agent + participant P as push server + participant APN as APN + + note over M, APN: get device token + M ->> APN: registerForRemoteNotifications() + APN ->> M: device token + + note over M, P: register device token with push server + M ->> C: /_ntf register + C ->> A: registerNtfToken() + A ->> P: TNEW + P ->> A: ID (tokenId) + A ->> C: registered + C ->> M: registered + + note over M, APN: verify device token + P ->> APN: E2E encrypted code
in background
notification + APN ->> M: deliver background notification with e2ee verification token + M ->> C: /_ntf verify + C ->> A: verifyNtfToken() + A ->> P: TVFY code + P ->> A: OK / ERR + A ->> C: verified + C ->> M: verified + + note over M, APN: now token ID can be used diff --git a/protocol/diagrams/notifications/register-token.mmd b/protocol/diagrams/notifications/register-token.mmd index bceef296f..caa05d716 100644 --- a/protocol/diagrams/notifications/register-token.mmd +++ b/protocol/diagrams/notifications/register-token.mmd @@ -1,30 +1,26 @@ sequenceDiagram - participant M as mobile app - participant C as chat core + participant C as client app participant A as agent - participant P as push server - participant APN as APN + participant P as SimpleX
Notification
Server + participant APN as Apple
Push Notifications
Server - note over M, APN: get device token - M ->> APN: registerForRemoteNotifications() - APN ->> M: device token + note over C, APN: get device token + C ->> APN: registerForRemoteNotifications() + APN ->> C: device token - note over M, P: register device token with push server - M ->> C: /_ntf register - C ->> A: registerNtfToken() + note over C, P: register device token with push server + C ->> A: registerToken A ->> P: TNEW P ->> A: ID (tokenId) A ->> C: registered - C ->> M: registered - note over M, APN: verify device token + note over C, APN: verify device token P ->> APN: E2E encrypted code
in background
notification - APN ->> M: deliver background notification with e2ee verification token - M ->> C: /_ntf verify - C ->> A: verifyNtfToken() + APN ->> C: deliver background notification with e2ee verification token + C ->> A: verifyToken
() A ->> P: TVFY code P ->> A: OK / ERR A ->> C: verified - C ->> M: verified - note over M, APN: now token ID can be used + note over C, APN: now token ID can be used + \ No newline at end of file diff --git a/protocol/diagrams/notifications/register-token.svg b/protocol/diagrams/notifications/register-token.svg new file mode 100644 index 000000000..662a185da --- /dev/null +++ b/protocol/diagrams/notifications/register-token.svg @@ -0,0 +1,3 @@ + + +ApplePush NotificationsServerSimpleXNotificationServeragentclient appApplePush NotificationsServerSimpleXNotificationServeragentclient appget device tokenregister device token with push serververify device tokennow token ID can be usedregisterForRemoteNotifications()device tokenregisterTokenTNEWID (tokenId)registeredE2E encrypted codein backgroundnotificationdeliver background notification with e2ee verification tokenverifyToken(<e2ee code>)TVFY codeOK / ERRverified \ No newline at end of file diff --git a/protocol/diagrams/notifications/subscription-detailed.mmd b/protocol/diagrams/notifications/subscription-detailed.mmd new file mode 100644 index 000000000..0db7cd7cc --- /dev/null +++ b/protocol/diagrams/notifications/subscription-detailed.mmd @@ -0,0 +1,40 @@ +sequenceDiagram + participant M as mobile app + participant C as chat core + participant A as agent + participant S as SMP server + participant N as NTF server + participant APN as APN + + note over M, APN: register subscription + + alt register existing + M -->> A: on /_ntf register, for subscribed queues + else create new connection + A -->> S: NEW / JOIN + note over A, S: ...
Connection handshake
... + S -->> A: CON + end + A ->> S: NKEY nKey + S ->> A: NID nId + A ->> N: SNEW tknId dhKey (smpServer, nId, nKey) + N ->> A: ID subId dhKey + N ->> S: NSUB nId + S ->> N: OK [/ NMSG] + + note over M, APN: notify about message + + S ->> N: NMSG + N ->> APN: APNSMutableContent
ntfQueue, nonce + APN ->> M: UNMutableNotificationContent + note over M, S: ...
Client awaken, message is received
... + S ->> M: message + note over M: mutate notification + + note over M, APN: change APN token + + APN ->> M: new device token + M -->> C: /_ntf_sub update tkn + C -->> A: updateNtfToken() + A -->> N: TUPD tknId newDeviceToken + note over M, N: ...
Verify token
... diff --git a/protocol/diagrams/notifications/subscription.mmd b/protocol/diagrams/notifications/subscription.mmd index 0db7cd7cc..8b4db971c 100644 --- a/protocol/diagrams/notifications/subscription.mmd +++ b/protocol/diagrams/notifications/subscription.mmd @@ -1,17 +1,16 @@ sequenceDiagram - participant M as mobile app - participant C as chat core + participant C as client app participant A as agent participant S as SMP server participant N as NTF server participant APN as APN - note over M, APN: register subscription + note over C, APN: register subscription alt register existing - M -->> A: on /_ntf register, for subscribed queues + C -->> A: registerToken else create new connection - A -->> S: NEW / JOIN + A -->> S: create/joinConnection note over A, S: ...
Connection handshake
... S -->> A: CON end @@ -20,21 +19,20 @@ sequenceDiagram A ->> N: SNEW tknId dhKey (smpServer, nId, nKey) N ->> A: ID subId dhKey N ->> S: NSUB nId - S ->> N: OK [/ NMSG] + S ->> N: OK / NMSG:
confirm subscription - note over M, APN: notify about message + note over C, APN: notify about message S ->> N: NMSG N ->> APN: APNSMutableContent
ntfQueue, nonce - APN ->> M: UNMutableNotificationContent - note over M, S: ...
Client awaken, message is received
... - S ->> M: message - note over M: mutate notification + APN ->> C: UNMutableNotificationContent + note over C, S: ...
Client awaken, message is received
... + S ->> C: message + note over C: show notification - note over M, APN: change APN token + note over C, APN: change APN token - APN ->> M: new device token - M -->> C: /_ntf_sub update tkn - C -->> A: updateNtfToken() + APN ->> C: new device token + C -->> A: updateToken() A -->> N: TUPD tknId newDeviceToken - note over M, N: ...
Verify token
... + note over C, N: ...
Verify token
... diff --git a/protocol/diagrams/notifications/subscription.svg b/protocol/diagrams/notifications/subscription.svg new file mode 100644 index 000000000..49840ce58 --- /dev/null +++ b/protocol/diagrams/notifications/subscription.svg @@ -0,0 +1,3 @@ + + +APNNTF serverSMP serveragentclient appAPNNTF serverSMP serveragentclient appregister subscription...Connection handshake...alt[register existing][create new connection]notify about message...Client awaken, message is received...show notificationchange APN token...Verify token...registerTokencreate/joinConnectionCONNKEY nKeyNID nIdSNEW tknId dhKey (smpServer, nId, nKey)ID subId dhKeyNSUB nIdOK / NMSG:confirm subscriptionNMSGAPNSMutableContentntfQueue, nonceUNMutableNotificationContentmessagenew device tokenupdateToken()TUPD tknId newDeviceToken \ No newline at end of file diff --git a/protocol/diagrams/simplex-messaging/simplex-creating-fast.mmd b/protocol/diagrams/simplex-messaging/simplex-creating-fast.mmd new file mode 100644 index 000000000..48a59988d --- /dev/null +++ b/protocol/diagrams/simplex-messaging/simplex-creating-fast.mmd @@ -0,0 +1,23 @@ +sequenceDiagram + participant B as Bob (sender) + participant S as server (queue RID) + participant A as Alice (recipient) + + note over A: creating queue
("public" key RK
for msg retrieval) + A ->> S: 1. create queue ("NEW") + S ->> A: respond with queue RID and SID ("IDS") + + note over A: out-of-band msg
(sender's queue SID
and "public" key EK
to encrypt msgs) + A -->> B: 2. send out-of-band message + + note over B: secure queue
(with "public" key SK for
sending messages) + B ->> S: 3. confirm queue ("SKEY" command authorized with SK) + + note over B: confirm queue
(public key
for e2e encryption
and any optional
encrypted info.) + B ->> S: 4. confirm queue ("SEND" command authorized with SK) + + S ->> A: 5. deliver Bob's message (MSG) + note over A: decrypt message
("private" key EK) + A ->> S: acknowledge message (ACK) + + note over S: 6. simplex
queue RID
is ready to use! diff --git a/protocol/diagrams/simplex-messaging/simplex-creating-fast.svg b/protocol/diagrams/simplex-messaging/simplex-creating-fast.svg new file mode 100644 index 000000000..37337435f --- /dev/null +++ b/protocol/diagrams/simplex-messaging/simplex-creating-fast.svg @@ -0,0 +1,3 @@ + + +Alice (recipient)server (queue RID)Bob (sender)Alice (recipient)server (queue RID)Bob (sender)creating queue("public" key RKfor msg retrieval)out-of-band msg(sender's queue SIDand "public" key EKto encrypt msgs)secure queue(with "public" key SK forsending messages)confirm queue(public keyfor e2e encryptionand any optionalencrypted info.)decrypt message("private" key EK)6. simplexqueue RIDis ready to use!1. create queue ("NEW")respond with queue RID and SID ("IDS")2. send out-of-band message3. confirm queue ("SKEY" command authorized with SK)4. confirm queue ("SEND" command authorized with SK)5. deliver Bob's message (MSG)acknowledge message (ACK) \ No newline at end of file diff --git a/protocol/diagrams/simplex-messaging/simplex-creating.mmd b/protocol/diagrams/simplex-messaging/simplex-creating.mmd index 5116c3bed..f38c174e8 100644 --- a/protocol/diagrams/simplex-messaging/simplex-creating.mmd +++ b/protocol/diagrams/simplex-messaging/simplex-creating.mmd @@ -10,11 +10,13 @@ sequenceDiagram note over A: out-of-band msg
(sender's queue SID
and "public" key EK
to encrypt msgs) A -->> B: 2. send out-of-band message - note over B: confirm queue
("public" key SK for
sending messages
and any optional
info encrypted with
"public" key EK) + note over B: confirm queue
("public" key SK for
sending messages,
public key for
e2e encryption
and any optional
encrypted info) B ->> S: 3. confirm queue ("SEND" command not signed) - S ->> A: 4. deliver Bob's message + S ->> A: 4. deliver Bob's message (MSG) note over A: decrypt message
("private" key EK) + A ->> S: acknowledge message (ACK) + A ->> S: 5. secure queue ("KEY", RK-signed) note over S: 6. simplex
queue RID
is ready to use! diff --git a/protocol/diagrams/simplex-messaging/simplex-creating.svg b/protocol/diagrams/simplex-messaging/simplex-creating.svg index 325fbb6cf..33dd288b3 100644 --- a/protocol/diagrams/simplex-messaging/simplex-creating.svg +++ b/protocol/diagrams/simplex-messaging/simplex-creating.svg @@ -1 +1,3 @@ -Bob (sender)server (queue RID)Alice (recipient)creating queue("public" key RKfor msg retrieval)1. create queue ("NEW")respond with queue RID and SID ("IDS")out-of-band msg(sender's queue SIDand "public" key EKto encrypt msgs)2. send out-of-band messageconfirm queue("public" key SK forsending messagesand any optionalinfo encrypted with"public" key EK)3. confirm queue ("SEND" command not signed)4. deliver Bob's messagedecrypt message("private" key EK)5. secure queue ("KEY", RK-signed)6. simplexqueue RIDis ready to use!Bob (sender)server (queue RID)Alice (recipient) \ No newline at end of file + + +Alice (recipient)server (queue RID)Bob (sender)Alice (recipient)server (queue RID)Bob (sender)creating queue("public" key RKfor msg retrieval)out-of-band msg(sender's queue SIDand "public" key EKto encrypt msgs)confirm queue("public" key SK forsending messages,public key fore2e encryptionand any optionalencrypted info)decrypt message("private" key EK)6. simplexqueue RIDis ready to use!1. create queue ("NEW")respond with queue RID and SID ("IDS")2. send out-of-band message3. confirm queue ("SEND" command not signed)4. deliver Bob's message (MSG)acknowledge message (ACK)5. secure queue ("KEY", RK-signed) \ No newline at end of file diff --git a/protocol/diagrams/xftp/xftp-receiving-file.mmd b/protocol/diagrams/xftp/xftp-receiving-file.mmd new file mode 100644 index 000000000..d1944368b --- /dev/null +++ b/protocol/diagrams/xftp/xftp-receiving-file.mmd @@ -0,0 +1,18 @@ +sequenceDiagram + participant B as Bob (recipient) + participant S as XFTP server(s) + + note over B: having received file description
from sender + + loop for each chunk + B ->> S: 1a. download chunk ("FGET") + S ->> B: send chunk body ("FILE") + + opt + B ->> S: 1b. acknowledge chunk reception ("FACK") + note over S: delete recipient ID + S ->> B: respond with ok ("OK") + end + end + + note over B: 2. combine chunks into a file
3. decrypt file using key from file description
4. extract file name and unpad the file
5. validate file digest with the file description diff --git a/protocol/diagrams/xftp/xftp-receiving-file.svg b/protocol/diagrams/xftp/xftp-receiving-file.svg new file mode 100644 index 000000000..aceeb653d --- /dev/null +++ b/protocol/diagrams/xftp/xftp-receiving-file.svg @@ -0,0 +1,3 @@ + + +XFTP server(s)Bob (recipient)XFTP server(s)Bob (recipient)having received file descriptionfrom senderdelete recipient IDopt​loop[for each chunk]2. combine chunks into a file3. decrypt file using key from file description4. extract file name and unpad the file5. validate file digest with the file description1a. download chunk ("FGET")send chunk body ("FILE")1b. acknowledge chunk reception ("FACK")respond with ok ("OK") \ No newline at end of file diff --git a/protocol/diagrams/xftp/xftp-sending-file.mmd b/protocol/diagrams/xftp/xftp-sending-file.mmd new file mode 100644 index 000000000..8e0b58498 --- /dev/null +++ b/protocol/diagrams/xftp/xftp-sending-file.mmd @@ -0,0 +1,23 @@ +sequenceDiagram + participant A as Alice (sender) + participant S as XFTP server(s) + participant B as recipient(s) + + note over A: 1. prepare file:
encrypt,
split into chunks,
generate recipient
keys, etc. + + loop for each chunk + A ->> S: 2a. register chunk ("FNEW") + S ->> A: respond with sender's and recipients' chunk IDs ("SIDS") + + opt + A ->> S: 2b. request additional recipient IDs ("FADD") + S ->> A: respond with added recipients' chunk IDs ("RIDS") + end + + A ->> S: 2c. upload chunk to chosen server ("FPUT") + S ->> A: respond with ok ("OK") + end + + note over A: 3. prepare file description(s) + + A -->> B: 4. send file description(s) out-of-band diff --git a/protocol/diagrams/xftp/xftp-sending-file.svg b/protocol/diagrams/xftp/xftp-sending-file.svg new file mode 100644 index 000000000..c3df658f5 --- /dev/null +++ b/protocol/diagrams/xftp/xftp-sending-file.svg @@ -0,0 +1,3 @@ + + +recipient(s)XFTP server(s)Alice (sender)recipient(s)XFTP server(s)Alice (sender)1. prepare file:encrypt,split into chunks,generate recipientkeys, etc.opt​loop[for each chunk]3. prepare file description(s)2a. register chunk ("FNEW")respond with sender's and recipients' chunk IDs ("SIDS")2b. request additional recipient IDs ("FADD")respond with added recipients' chunk IDs ("RIDS")2c. upload chunk to chosen server ("FPUT")respond with ok ("OK")4. send file description(s) out-of-band \ No newline at end of file diff --git a/protocol/diagrams/xrcp/session.mmd b/protocol/diagrams/xrcp/session.mmd new file mode 100644 index 000000000..2e98fbb1b --- /dev/null +++ b/protocol/diagrams/xrcp/session.mmd @@ -0,0 +1,44 @@ +sequenceDiagram + participant CI as Controller UI + participant CC as Controller Core + participant HC as Host Core + participant HI as Host UI + +note over CI, HI: 1. Session invitation +CI->>CC: "Link a mobile" +CC-->>CI: Session invitation URI +note over CC: Listen for TCP connection +activate CC +HI->>HC: Session invitation URI + +note over CI, HI: 2. Establishing TLS connection +HC-->>CC: TCP connect +note over CC, HC: TLS handshake +par + note over CC: validate client X509 credentials + CC->>CI: session code from tlsUnique + CI-->>CC: user confirmation +and + note over HC: validate server X509 credentials + HC->>HI: session code from tlsUnique + HI-->>HC: user confirmation +end + +note over CI, HI: 3. Session verification and protocol negotiation +HC->>CC: host HELLO +note over CC: validate version, CA fingerprint +alt + CC-->>HC: controller ERROR +else + CC-->>HC: controller HELLO + note over CC, HC: update stored keys +end +deactivate CC + +note over CI, HI: 4. Session operation +loop + CI->>CC: command + CC->>HC: XRCP command + HC-->>CC: XRCP response + CC-->>CI: response +end diff --git a/protocol/diagrams/xrcp/session.svg b/protocol/diagrams/xrcp/session.svg new file mode 100644 index 000000000..136b56fc6 --- /dev/null +++ b/protocol/diagrams/xrcp/session.svg @@ -0,0 +1 @@ +Host UIHost CoreController CoreController UIHost UIHost CoreController CoreController UI1. Session invitationListen for TCP connection2. Establishing TLS connectionTLS handshakevalidate client X509 credentialsvalidate server X509 credentialspar​3. Session verification and protocol negotiationvalidate version, CA fingerprintupdate stored keysalt​4. Session operationloop​"Link a mobile"Session invitation URISession invitation URITCP connectsession code from tlsUniqueuser confirmationsession code from tlsUniqueuser confirmationhost HELLOcontroller ERRORcontroller HELLOcommandXRCP commandXRCP responseresponse \ No newline at end of file diff --git a/protocol/overview-tjr.md b/protocol/overview-tjr.md index 7f1e72768..d30f55235 100644 --- a/protocol/overview-tjr.md +++ b/protocol/overview-tjr.md @@ -1,4 +1,4 @@ -Revision 1, 2022-01-01 +Revision 2, 2024-06-22 Evgeny Poberezkin @@ -13,19 +13,23 @@ Evgeny Poberezkin - [Technical Details](#technical-details) - [Trust in Servers](#trust-in-servers) - [Client -> Server Communication](#client---server-communication) + - [2-hop Onion Message Routing](#2-hop-onion-message-routing) - [SimpleX Messaging Protocol](#simplex-messaging-protocol) - [SimpleX Agents](#simplex-agents) - [Encryption Primitives Used](#encryption-primitives-used) - [Threat model](#threat-model) - [Acknowledgements](#acknowledgements) + ## Introduction #### What is SimpleX SimpleX as a whole is a platform upon which applications can be built. [SimpleX Chat](https://github.com/simplex-chat/simplex-chat) is one such application that also serves as an example and reference application. - - [SimpleX Messaging Protocol](https://github.com/simplex-chat/simplexmq/blob/master/protocol/simplex-messaging.md) (SMP) is a protocol to send messages in one direction to a recipient, relying on a server in-between. The messages are delivered via uni-directional queues created by recipients. + - [SimpleX Messaging Protocol](./simplex-messaging.md) (SMP) is a protocol to send messages in one direction to a recipient, relying on a server in-between. The messages are delivered via uni-directional queues created by recipients. + + - SMP protocol allows to send message via a SMP server playing proxy role using 2-hop onion routing (referred to as "private routing" in messaging clients) to protect transport information of the sender (IP address and session) from the server chosen (and possibly controlled) by the recipient. - SMP runs over a transport protocol (shown below as TLS) that provides integrity, server authentication, confidentiality, and transport channel binding. @@ -35,7 +39,9 @@ SimpleX as a whole is a platform upon which applications can be built. [SimpleX - SimpleX Client libraries speak SMP to SimpleX Servers and provide a low-level API not generally intended to be used by applications. - - SimpleX Agents interface with SimpleX Clients to provide a more high-level API intended to be used by applications. Typically they are embedded as libraries, but are designed so they can also be abstracted into local services. + - SimpleX Agents interface with SimpleX Clients to provide a more high-level API intended to be used by applications. Typically they are embedded as libraries, but can also be abstracted into local services. + + - SimpleX Agents communicate with other agents inside e2e encrypted envelopes provided by SMP protocol - the syntax and semantics of the messages exchanged by the agent are defined by [SMP agent protocol](./agent-protocol.md) *Diagram showing the SimpleX Chat app, with logical layers of the chat application interfacing with a SimpleX Agent library, which in turn interfaces with a SimpleX Client library. The Client library in turn speaks the Messaging Protocol to a SimpleX Server.* @@ -72,10 +78,11 @@ SimpleX as a whole is a platform upon which applications can be built. [SimpleX - Low latency: the delay introduced by the network should not be higher than 100ms-1s in addition to the underlying TCP network latency. -2. Provide better communication security and privacy than the alternative instant messaging solutions. In particular SimpleX provides better privacy of metadata (who talks to whom and when) and better security against active network attackers and malicious servers. +2. Provide better communication security and privacy than the alternative instant messaging solutions. In particular SimpleX provides better privacy of metadata (who talks to whom and when) and better security against active network attackers and malicious servers. 3. Balance user experience with privacy requirements, prioritizing experience of mobile device users. + #### In Comparison SimpleX network has a design similar to P2P networks, but unlike most P2P networks it consists of clients and servers without depending on any centralized component. @@ -91,53 +98,73 @@ In comparison to more traditional messaging applications (e.g. WhatsApp, Signal, - users can change servers with minimal disruption - even after an in-use server disappears, simply by changing the configuration on which servers the new queues are created. + ## Technical Details #### Trust in Servers Clients communicate directly with servers (but not with other clients) using SimpleX Messaging Protocol (SMP) running over some transport protocol that provides integrity, server authentication, confidentiality, and transport channel binding. By default, we assume this transport protocol is TLS. -Users use multiple servers, and choose where to receive their messages. Accordingly, they send messages to their communication partners' chosen servers. +Users use multiple servers, and choose where to receive their messages. Accordingly, they send messages to their communication partners' chosen servers either directly, if this is a known/trusted server, or via another SMP server providing proxy functionality to protect IP address and session of the sender. -Although end-to-end encryption is always present, users place a degree of trust in servers. This trust decision is very similar to a user's choice of email provider; however the trust placed in a SimpleX server is significantly less. Notably, there is no re-used identifier or credential between queues on the same (or different) servers. While a user *may* re-use a connection to fetch from multiple queues, or connect to a server from the same IP address, both are choices a user may opt into to break the promise of un-correlatable queues. +Although end-to-end encryption is always present, users place a degree of trust in servers they connect to. This trust decision is very similar to a user's choice of email provider; however the trust placed in a SimpleX server is significantly less. Notably, there is no re-used identifier or credential between queues on the same (or different) servers. While a user *may* re-use a transport connection to fetch messages from multiple queues, or connect to a server from the same IP address, both are choices a user may opt into to break the promise of un-correlatable queues. Users may trust a server because: -- They deploy and control the servers themselves from the available open-source code. This has the trade-offs of strong trust in the server but limited metadata obfuscation to a passive network observer. Techniques such as noise traffic, traffic mixing (incurring latency), and using an onion routing transport protocol can mitigate that latter. +- They deploy and control the servers themselves from the available open-source code. This has the trade-offs of strong trust in the server but limited metadata obfuscation to a passive network observer. Techniques such as noise traffic, traffic mixing (incurring latency), and using an onion routing transport protocol can mitigate that. - They use servers from a trusted commercial provider. The more clients the provider has, the less metadata about the communication times is leaked to the network observers. -- Users trust their contacts and the servers they chose. +By default, servers do not retain access logs, and permanently delete messages and queues when requested. Messages persist only in memory until they cross a threshold of time, typically on the order of days.[0] There is still a risk that a server maliciously records all queues and messages (even though encrypted) sent via the same transport connection to gain a partial knowledge of the user’s communications graph and other meta-data. -By default, servers do not retain access logs, and permanently delete messages and queues when requested. Messages persist only in memory until they cross a threshold of time, typically on the order of days.[0] There is still a risk that a server maliciously records all queues and messages (even though encrypted) sent via the same transport connection to gain a partial knowledge of the user’s communications graph and other meta-data. +SimpleX supports measures (managed transparently to the user at the agent level) to mitigate the trust placed in servers. These include rotating the queues in use between users, noise traffic, supporting overlay networks such as Tor, and isolating traffic to different queues to different transport connections (and Tor circuits, if Tor is used). -SimpleX supports measures (managed transparently to the user at the agent level) to mitigate the trust placed in servers. These include rotating the queues in use between users, noise traffic, and supporting overlay networks such as Tor. - -[0] While configurable by servers, a minimum value is enforced by the default software. SimpleX Agents provide redundant routing over queues to mitigate against message loss. +[0] While configurable by servers, a minimum value is enforced by the default software. SimpleX Agents can provide redundant routing over queues to mitigate against message loss. #### Client -> Server Communication Utilizing TLS grants the SimpleX Messaging Protocol (SMP) server authentication and metadata protection to a passive network observer. But SMP does not rely on the transport protocol for message confidentiality or client authentication. The SMP protocol itself provides end-to-end confidentiality, authentication, and integrity of messages between communicating parties. -Servers have long-lived, self-signed, offline certificates whose hash is pre-shared with clients over secure channels - either provided with the client library or provided in the secure introduction between clients. The offline certificate signs an online certificate used in the transport protocol handshake. [0] +Servers have long-lived, self-signed, offline certificates whose hash is pre-shared with clients over secure channels - either provided with the client library or provided in the secure introduction between clients, as part of the server address. The offline certificate signs an online certificate used in the transport protocol handshake. [0] If the transport protocol's confidentiality is broken, incoming and outgoing messages to the server cannot be correlated by message contents. Additionally, because of encryption at the SMP layer, impersonating the server is not sufficient to pass (and therefore correlate) a message from a sender to recipient - the only attack possible is to drop the messages. Only by additionally *compromising* the server can one pass and correlate messages. It's important to note that the SMP protocol does not do server authentication. Instead we rely upon the fact that an attacker who tricks the transport protocol into authenticating the server incorrectly cannot do anything with the SMP messages except drop them. -After the connection is established, the client sends blocks of a fixed size 16Kb, and the server replies with the blocks of the same size to reduce metadata observable to a network adversary. The protocol has been designed to make traffic correlation attacks difficult, adapting ideas from Tor, remailers, and more general onion and mix networks. It does not try to replace Tor though - SimpleX servers can be deployed as onion services and SimpleX clients can communicate with servers over Tor to further improve participants privacy. +After the connection is established, the client sends blocks of a fixed size 16KB, and the server replies with the blocks of the same size to reduce metadata observable to a network adversary. The protocol has been designed to make traffic correlation attacks difficult, adapting ideas from Tor, remailers, and more general onion and mix networks. It does not try to replace Tor though - SimpleX servers can be deployed as onion services and SimpleX clients can communicate with servers over Tor to further improve participants privacy. -By using fixed-size blocks, oversized for the expected content, the vast majority of traffic is uniform in nature. When enough traffic is transiting a server simultaneously, the server acts as a (very) low-latency mix node. We can't rely on this behavior to make a security claim, but we have engineered to take advantage of it when we can. As mentioned, this holds true even if the transport connection is compromised. +By using fixed-size blocks, oversized for the expected content, the vast majority of traffic is uniform in nature. When enough traffic is transiting a server simultaneously, the server acts as a low-latency mix node. We can't rely on this behavior to make a security claim, but we have engineered to take advantage of it when we can. As mentioned, this holds true even if the transport connection is compromised. -The protocol does not protect against attacks targeted at particular users with known identities - e.g., if the attacker wants to prove that two known users are communicating, they can achieve it. At the same time, it substantially complicates large-scale traffic correlation, making determining the real user identities much less effective. +The protocol does not protect against attacks targeted at particular users with known identities - e.g., if the attacker wants to prove that two known users are communicating, they can achieve it by observing their local traffic. At the same time, it substantially complicates large-scale traffic correlation, making determining the real user identities much less effective. [0] Future versions of SMP may add support for revocation lists of certificates, presently this risk is mitigated by the SMP protocol itself. +#### 2-hop Onion Message Routing + +As SimpleX Messaging Protocol servers providing messaging queues are chosen by the recipients, in case senders connect to these servers directly the server owners (who potentially can be the recipients themselves) can learn senders' IP addresses (if Tor is not used) and which other queues on the same server are accessed by the user in the same transport connection (even if Tor is used). + +While the clients support isolating the messages sent to different queues into different transport connections (and Tor circuits), this is not practical, as it consumes additional traffic and system resources. + +To mitigate this problem SimpleX Messaging Protocol servers support 2-hop onion message routing when the SMP server chosen by the sender forwards the messages to the servers chosen by the recipients, thus protecting both the senders IP addresses and sessions, even if connection isolation and Tor are not used. + +The design of 2-hop onion message routing prevents these potential attacks: + +- MITM by proxy (SMP server that forwards the messages). + +- Identification by the proxy which and how many queues the sender sends messages to (as messages are additionally e2e encrypted between the sender and the destination SMP server). + +- Correlation of messages sent to different queues via the same user session (as random correlation IDs and keys are used for each message). + +See more details about 2-hop onion message routing design in [SimpleX Messaging Protocol](./simplex-messaging.md#proxying-sender-commands) + +Also see [Threat model](#threat-model) + + #### SimpleX Messaging Protocol -SMP is initialized with an in-person or out-of-band introduction message, where Alice provides Bob with details of a server (including IP, port, and hash of the long-lived offline certificate), a queue ID, and Alice's public key for her receiving queue. These introductions are similar to the PANDA key-exchange, in that if observed, the adversary can race to establish the communication channel instead of the intended participant. [0] +SMP is initialized with an in-person or out-of-band introduction message, where Alice provides Bob with details of a server (including IP address or host name, port, and hash of the long-lived offline certificate), a queue ID, and Alice's public keys to agree e2e encryption. These introductions are similar to the PANDA key-exchange, in that if observed, the adversary can race to establish the communication channel instead of the intended participant. [0] Because queues are uni-directional, Bob provides an identically-formatted introduction message to Alice over Alice's now-established receiving queue. @@ -145,6 +172,7 @@ When setting up a queue, the server will create separate sender and recipient qu [0] Users can additionally create public 'contact queues' that are only used to receive connection requests. + #### SimpleX Agents SimpleX agents provide higher-level operations compared to SimpleX Clients, who are primarily concerned with creating queues and communicating with servers using SMP. Agent operations include: @@ -157,9 +185,10 @@ SimpleX agents provide higher-level operations compared to SimpleX Clients, who - Noise traffic + #### Encryption Primitives Used -- Ed448 to sign/verify commands to SMP servers (Ed25519 is also supported via client/server configuration). +- Ed25519 or Curve25519 to authorize/verify commands to SMP servers (authorization algorithm is set via client/server configuration). - Curve25519 for DH exchange to agree: - the shared secret between server and recipient (to encrypt message bodies - it avoids shared cipher-text in sender and recipient traffic) - the shared secret between sender and recipient (to encrypt messages end-to-end in each queue - it avoids shared cipher-text in redundant queues). @@ -170,42 +199,44 @@ SimpleX agents provide higher-level operations compared to SimpleX Clients, who - AES-GCM AEAD cipher, - SHA512-based HKDF for key derivation. + ## Threat Model #### Global Assumptions - - A user protects their local database and key material - - The user's application is authentic, and no local malware is running - - The cryptographic primitives in use are not broken + - A user protects their local database and key material. + - The user's application is authentic, and no local malware is running. + - The cryptographic primitives in use are not broken. - A user's choice of servers is not directly tied to their identity or otherwise represents distinguishing information about the user. + - The user's client uses 2-hop onion message routing. #### A passive adversary able to monitor the traffic of one user *can:* - - identify that and when a user is using SimpleX + - identify that and when a user is using SimpleX. - - block SimpleX traffic - - - determine which servers the user communicates with + - determine which servers the user receives the messages from. - observe how much traffic is being sent, and make guesses as to its purpose. *cannot:* - - see who sends messages to the user and who the user sends the messages to + - see who sends messages to the user and who the user sends the messages to. + + - determine the servers used by users' contacts. #### A passive adversary able to monitor a set of senders and recipients *can:* - - identify who and when is using SimpleX + - identify who and when is using SimpleX. - - learn which SimpleX Messaging Protocol servers are used as receive queues for which users + - learn which SimpleX Messaging Protocol servers are used as receive queues for which users. - - learn when messages are sent and received + - learn when messages are sent and received. - - perform traffic correlation attacks against senders and recipients and correlate senders and recipients within the monitored set, frustrated by the number of users on the servers + - perform traffic correlation attacks against senders and recipients and correlate senders and recipients within the monitored set, frustrated by the number of users on the servers. - observe how much traffic is being sent, and make guesses as to its purpose @@ -217,43 +248,83 @@ SimpleX agents provide higher-level operations compared to SimpleX Clients, who *can:* -- learn when a queue recipient or sender is online +- learn when a queue recipient is online -- know how many messages are sent via the queue (although some may be noise) +- know how many messages are sent via the queue (although some may be noise or not content messages). -- perform queue correlation (matching multiple queues to a single user) via either a re-used transport connection, user's IP Address, or connection timing regularities +- learn which messages would trigger notifications even if a user does not use [push notifications](./push-notifications.md). -- learn a user's IP address, track them through other IP addresses they use to access the same queue, and infer information (e.g. employer) based on the IP addresses, as long as Tor is not used. +- perform the correlation of the queue used to receive messages (matching multiple queues to a single user) via either a re-used transport connection, user's IP Address, or connection timing regularities. -- drop all future messages inserted into a queue, detectable only over other, redundant queues +- learn a recipient's IP address, track them through other IP addresses they use to access the same queue, and infer information (e.g. employer) based on the IP addresses, as long as Tor is not used. + +- drop all future messages inserted into a queue, detectable only over other, redundant queues. - lie about the state of a queue to the recipient and/or to the sender (e.g. suspended or deleted when it is not). -- spam a user with invalid messages +- spam a user with invalid messages. *cannot:* -- undetectably add, duplicate, or corrupt individual messages +- undetectably add, duplicate, or corrupt individual messages. -- undetectably drop individual messages, so long as a subsequent message is delivered +- undetectably drop individual messages, so long as a subsequent message is delivered. -- learn the contents of messages +- learn the contents or type of messages. -- distinguish noise messages from regular messages except via timing regularities +- distinguish noise messages from regular messages except via timing regularities. -- compromise the user's end-to-end encryption with an active attack +- compromise the users' end-to-end encryption with an active attack. + +- learn a sender's IP address, track them through other IP addresses they use to access the same queue, and infer information (e.g. employer) based on the IP addresses, even if Tor is not used (provided messages are sent via proxy SMP server). + +- perform senders' queue correlation (matching multiple queues to a single sender) via either a re-used transport connection, user's IP Address, or connection timing regularities, unless it has additional information from the proxy SMP server (provided messages are sent via proxy SMP server). + +#### SimpleX Messaging Protocol server that proxies the messages to another SMP server + +*can:* + +- learn a sender's IP address, as long as Tor is not used. + +- learn when a sender with a given IP address is online. + +- know how many messages are sent from a given IP address and to a given destination SMP server. + +- drop all messages from a given IP address or to a given destination server. + +- unless destination SMP server detects repeated public DH keys of senders, replay messages to a destination server within a single session, causing either duplicate message delivery (which will be detected and ignored by the receiving clients), or, when receiving client is not connected to SMP server, exhausting capacity of destination queues used within the session. + +*cannot:* + +- perform queue correlation (matching multiple queues to a single user), unless it has additional information from the destination SMP server. + +- undetectably add, duplicate, or corrupt individual messages. + +- undetectably drop individual messages, so long as a subsequent message is delivered. + +- learn the contents or type of messages. + +- learn which messages would trigger notifications. + +- learn the destination queues of messages. + +- distinguish noise messages from regular messages except via timing regularities. + +- compromise the user's end-to-end encryption with another user via an active attack. + +- compromise the user's end-to-end encryption with the destination SMP servers via an active attack. #### An attacker who obtained Alice's (decrypted) chat database *can:* -- see the history of all messages exchanged by Alice with her communication partners +- see the history of all messages exchanged by Alice with her communication partners. -- see shared profiles of contacts and groups +- see shared profiles of contacts and groups. -- surreptitiously receive new messages sent to Alice via existing queues; until communication queues are rotated or the Double-Ratchet advances forward +- surreptitiously receive new messages sent to Alice via existing queues; until communication queues are rotated or the Double-Ratchet advances forward. -- prevent Alice from receiving all new messages sent to her - either surreptitiously by emptying the queues regularly or overtly by deleting them +- prevent Alice from receiving all new messages sent to her - either surreptitiously by emptying the queues regularly or overtly by deleting them. - send messages from the user to their contacts; recipients will detect it as soon as the user sends the next message, because the previous message hash won’t match (and potentially won’t be able to decrypt them in case they don’t keep the previous ratchet keys). @@ -269,41 +340,41 @@ SimpleX agents provide higher-level operations compared to SimpleX Clients, who *can:* -- spam the user with messages +- spam the user with messages. -- forever retain messages from the user +- forever retain messages from the user. *cannot:* -- cryptographically prove to a third-party that a message came from a user (assuming the user’s device is not seized) +- cryptographically prove to a third-party that a message came from a user (assuming the user’s device is not seized). -- prove that two contacts they have is the same user +- prove that two contacts they have is the same user. -- cannot collaborate with another of the user's contacts to confirm they are communicating with the same user +- cannot collaborate with another of the user's contacts to confirm they are communicating with the same user. #### An attacker who observes Alice showing an introduction message to Bob *can:* - - Impersonate Bob to Alice + - Impersonate Bob to Alice. *cannot:* - - Impersonate Alice to Bob + - Impersonate Alice to Bob. #### An attacker with Internet access *can:* -- Denial of Service SimpleX messaging servers +- Denial of Service SimpleX messaging servers. -- spam a user's public “contact queue” with connection requests +- spam a user's public “contact queue” with connection requests. *cannot:* -- send messages to a user who they are not connected with +- send messages to a user who they are not connected with. -- enumerate queues on a SimpleX server +- enumerate queues on a SimpleX server. ## Acknowledgements diff --git a/protocol/pqdr.md b/protocol/pqdr.md new file mode 100644 index 000000000..27f7082c8 --- /dev/null +++ b/protocol/pqdr.md @@ -0,0 +1,222 @@ +Version 1, 2024-06-22 + +# Post-quantum resistant augmented double ratchet algorithm (PQDR) + +## Table of contents + +- [Overview](#overview) +- [Comparison with the other approaches](#comparison-with-the-other-approaches) + - [PQXDH for post-quantum key agreement](#pqxdh-for-post-quantum-key-agreement) (Signal) + - [Hybrid Signal protocol for post-quantum encryption](#hybrid-signal-protocol-for-post-quantum-encryption) (Tutanota) +- [Augmented double ratchet algorithm](#augmented-double-ratchet-algorithm) +- [Double ratchet with encrypted headers augmented with double PQ KEM](#double-ratchet-with-encrypted-headers-augmented-with-double-pq-kem) + - [Initialization](#initialization) + - [Encrypting messages](#encrypting-messages) + - [Decrypting messages](#decrypting-messages) +- [Implementation considerations](#implementation-considerations) +- [Chosen KEM algorithm](#chosen-kem-algorithm) +- [Summary](#summary) + +## Overview + +It is a reasonable assumption that "record-now-decrypt-later" attacks are ongoing, so the users want to use cryptographic schemes for end-to-end encryption that are augmented with some post-quantum algorithm that is believed to be resistant to quantum computers. + +SimpleX Chat uses [double-ratchet with header encryption](https://signal.org/docs/specifications/doubleratchet/#double-ratchet-with-header-encryption) to provide end-to-end encryption to messages and files. This document describes augmented algorithm with post-quantum key encapsulation mechanism (KEM) making it resistant to quantum computers. + +Double-ratchet algorithm is a state of the art solution for end to end encryption offering a set of qualities that is not present in any other algorithm: + +- perfect forward secrecy, i.e. compromise of session or long term keys does not lead to the ability to decrypt any of the past messages. +- deniability (also known as repudiation), i.e. the fact that the recipient of the message while having the proof of message authenticity, cannot prove to a third party that the sender actually sent this message. +- break-in recovery (also know as post-compromise security or future secrecy), i.e. the ability of the end-to-end encryption security to recover from the compromise of the long term keys. This is achieved by generating a new random key pair whenever a new DH key is received (DH ratchet step). + +It is desirable to preserve all these qualities when augmenting the algorithm with a post-quantum algorithm, and having these qualities resistant to both conventional and quantum computers. + +## Comparison with the other approaches + +### PQXDH for post-quantum key agreement + +[The solution](https://signal.org/docs/specifications/pqxdh/) recently [introduced by Signal](https://signal.org/blog/pqxdh/) augments the initial key agreement ([X3DH](https://signal.org/docs/specifications/x3dh/)) that is made prior to double ratchet algorithm. This is believed to provide protection from "record-now-decrypt-later" attack, but if the attacker at any point obtains long term keys from any of the devices, the break-in recovery will not be post-quantum resistant, and the attacker with quantum computer will be able to decrypt all the subsequent messages. + +### Hybrid Signal protocol for post-quantum encryption + +[The solution](https://eprint.iacr.org/2021/875.pdf) [proposed by Tutanota](https://tutanota.com/blog/posts/pqmail-update/) aims to preserve the break-in recovery property of double ratchet, but in doing so it: +- replaces rather than augments DH key agreement with post-quantum KEM mechanism, making it potentially vulnerable to conventional computers. +- adds signature to the DH ratchet step, to compensate for not keeping DH key agreement, but losing the deniability property for some of the messages. + +## Augmented double ratchet algorithm + +The double ratchet algorithm is augmented with post-quantum KEM mechanism, preserving all properties of the double ratchet algorithm. + +It is possible, because although double ratchet uses DH (which is a non-interactive key exchanges), it uses it "interactively", when the new DH keys are generated by both parties in turns. Parties of double-ratchet encrypted communication can run two post-quantum key encapsulation mechanisms in parallel with both DH and KEM key agreements in each DH ratchet step, making break-in recovery of double ratchet algorithm post-quantum resistant, without losing deniability or resistance to conventional computers. + +Specifically, [double ratchet with encrypted headers](https://signal.org/docs/specifications/doubleratchet/#double-ratchet-with-header-encryption) is augmented with some post-quantum key encapsulation mechanism (KEM) as described below. A possible algorithm for PQ KEM is [NTRU-prime](https://ntruprime.cr.yp.to), that is currently adopted in SSH and has available implementations. It is important though that the proposed scheme can be used with any PQ KEM algorithm. + +The downside of the scheme is its substantial size overhead, as the encapsulation key and encapsulated shared secret are added to the header of each message. For the algorithm described below NTRU-prime adds ~2-4kb to each message (depending on the key size and the chosen variant). See [this table](https://ntruprime.cr.yp.to/security.html) for key and ciphertext sizes and the assessment of the security level for various key sizes. + +It is possible to reduce size overhead by using only one KEM agreement and making only one of two ratchet steps providing post-quantum resistant break-in recovery. + +## Double ratchet with encrypted headers augmented with double PQ KEM + +Algorithm below assumes that in addition to shared secret from the initial key agreement, there will be an encapsulation key available from the party that published its keys (Bob). + +### Initialization + +The double ratchet initialization is defined in pseudo-code. This pseudo-code is identical to Signal algorithm specification except for that parts that add post-quantum key agreement. + +``` +// Alice obtained Bob's keys and initializes ratchet first +def RatchetInitAlicePQ2HE(state, SK, bob_dh_public_key, shared_hka, shared_nhkb, bob_pq_kem_encapsulation_key): + state.DHRs = GENERATE_DH() + state.DHRr = bob_dh_public_key + // below added for post-quantum KEM + state.PQRs = GENERATE_PQKEM() + state.PQRr = bob_pq_kem_encapsulation_key + state.PQRss = random // shared secret for KEM + state.PQRct = PQKEM-ENC(state.PQRr, state.PQRss) // encapsulated additional shared secret + // above added for KEM + // the next line augments DH key agreement with PQ shared secret + state.RK, state.CKs, state.NHKs = KDF_RK_HE(SK, DH(state.DHRs, state.DHRr) || state.PQRss) + state.CKr = None + state.Ns = 0 + state.Nr = 0 + state.PN = 0 + state.MKSKIPPED = {} + state.HKs = shared_hka + state.HKr = None + state.NHKr = shared_nhkb + +// Bob initializes ratchet second, having received Alice's connection request +def RatchetInitBobPQ2HE(state, SK, bob_dh_key_pair, shared_hka, shared_nhkb, bob_pq_kem_key_pair): + state.DHRs = bob_dh_key_pair + state.DHRr = None + // below added for KEM + state.PQRs = bob_pq_kem_key_pair + state.PQRr = None + state.PQRss = None + state.PQRct = None + // above added for KEM + state.RK = SK + state.CKs = None + state.CKr = None + state.Ns = 0 + state.Nr = 0 + state.PN = 0 + state.MKSKIPPED = {} + state.HKs = None + state.NHKs = shared_nhkb + state.HKr = None + state.NHKr = shared_hka +``` + +`GENERATE_PQKEM` generates decapsulation/encapsulation key pair. + +`PQKEM-ENC` is key encapsulation algorithm. + +Other than commented lines, the above adds parameters `bob_pq_kem_encapsulation_key` and `bob_pq_kem_key_pair` to the ratchet initialization. Otherwise it is identical to the original double ratchet initialization. + +### Encrypting messages + +``` +def RatchetEncryptPQ2HE(state, plaintext, AD): + state.CKs, mk = KDF_CK(state.CKs) + // encapsulation key from PQRs and encapsulated shared secret is added to header + header = HEADER_PQ2( + dh = state.DHRs.public, + kem = state.PQRs.public, // added for KEM #2 + ct = state.PQRct // added for KEM #1 + pn = state.PN, + n = state.Ns, + ) + enc_header = HENCRYPT(state.HKs, header) + state.Ns += 1 + return enc_header, ENCRYPT(mk, plaintext, CONCAT(AD, enc_header)) +``` + +Other than adding encapsulation key and encapsulated shared secret into the header, the above is identical to the original double ratchet message encryption step. + +### Decrypting messages + +``` +def RatchetDecryptPQ2HE(state, enc_header, ciphertext, AD): + plaintext = TrySkippedMessageKeysHE(state, enc_header, ciphertext, AD) + if plaintext != None: + return plaintext + header, dh_ratchet = DecryptHeader(state, enc_header) // DecryptHeader is the same as in double ratchet specification + if dh_ratchet: + SkipMessageKeysHE(state, header.pn) // SkipMessageKeysHE is the same as in double ratchet specification + DHRatchetPQ2HE(state, header) + SkipMessageKeysHE(state, header.n) + state.CKr, mk = KDF_CK(state.CKr) + state.Nr += 1 + return DECRYPT(mk, ciphertext, CONCAT(AD, enc_header)) + +// DecryptHeader is the same as in double ratchet specification +def DecryptHeader(state, enc_header): + header = HDECRYPT(state.HKr, enc_header) + if header != None: + return header, False + header = HDECRYPT(state.NHKr, enc_header) + if header != None: + return header, True + raise Error() + +def DHRatchetPQ2HE(state, header): + state.PN = state.Ns + state.Ns = 0 + state.Nr = 0 + state.HKs = state.NHKs + state.HKr = state.NHKr + state.DHRr = header.dh + // save new encapsulation key from header + state.PQRr = header.kem + // decapsulate shared secret from header - KEM #2 + ss = PQKEM-DEC(state.PQRs.private, header.ct) + // use decapsulated shared secret with receiving ratchet + state.RK, state.CKr, state.NHKr = KDF_RK_HE(state.RK, DH(state.DHRs, state.DHRr) || ss) + state.DHRs = GENERATE_DH() + // below is added for KEM + state.PQRs = GENERATE_PQKEM() // generate new PQ key pair + state.PQRss = random // shared secret for KEM + state.PQRct = PQKEM-ENC(state.PQRr, state.PQRss) // encapsulated additional shared secret KEM #1 + // above is added for KEM + // use new shared secret with sending ratchet + state.RK, state.CKs, state.NHKs = KDF_RK_HE(state.RK, DH(state.DHRs, state.DHRr) || state.PQRss) +``` + +`PQKEM-DEC` is key decapsulation algorithm. + +`DHRatchetPQ2HE` augments both DH agreements with decapsulated shared secret from the received header and with the new shared secret, respectively. The new shared secret together with the new encapsulation key are saved in the state and will be added to the header in the next sent message. + +Other than augmenting DH key agreements with the shared secrets from KEM, the above is identical to the original double ratchet DH ratchet step. + +It is worth noting that while DH agreements work as ping-pong, when the new received DH key is used for both DH agreements (and only the sent DH key is updated for the second DH key agreement), PQ KEM agreements in the proposed scheme work as a "parallel ping-pong", with two balls in play all the time (two KEM agreements run in parallel). + +## Implementation considerations for SimpleX Messaging Protocol + +As SimpleX Messaging Protocol pads messages to a fixed size, using 16kb transport blocks, the size increase introduced by this scheme can be compensated for by using ZSTD encryption of JSON bodies and image previews encoded as base64. While there may be some rare cases of random texts that would fail to compress, in all real scenarios it would not cause the message size reduction. + +Sharing the initial keys in case of SimpleX Chat it is equivalent to sharing the invitation link. As encapsulation key is large, it may be inconvenient to share it in the link in some contexts, e.g. when QR codes are used. + +It is possible to postpone sharing the encapsulation key until the first message from Alice (confirmation message in SMP protocol), the party sending connection request. The upside here is that the invitation link size would not increase. The downside is that the user profile shared in this confirmation will not be encrypted with PQ-resistant algorithm. + +Another consideration is pairwise ratchets in groups. Key generation in sntrup761 is quite slow - on slow devices it can be as slow as 10-20 keys per second, so using this primitive in groups larger than 10-20 members would result in slow performance. + +For backward compatibility the implementation must support adding PQ-resistant key agreement to the existing connections. + +It is also beneficial to support removing PQ-resistant key agreement from the connections that have them, e.g. as the group size grows. + +### Chosen KEM algorithm + +The implementation uses Streamlined NTRU-Prime 761 (sntrup761) that was also used for OpenSSH for a long time. + +It was chosen over ML-KEM (Kyber) standardized by NIST for several reasons: + +- sntrup761 was used in OpenSSH for a long period of time. +- ML-KEM standardization process raised [concerns](https://groups.google.com/a/list.nist.gov/g/pqc-forum/c/WFRDl8DqYQ4) [amongst](https://blog.cr.yp.to/20231003-countcorrectly.html) the experts. +- ML-KEM (if modified) is likely to have conflicts with the existing patents, unlike sntrup761. + +It was chosen over non-interactive CTIDH due to its slower implementation, and lack of optimized code for aarch64 CPUs used in mobile devices. + +## Summary + +If chosen PQ KEM proves secure against quantum computer attacks, then the proposed augmented double ratchet will also be secure against quantum computer attack, including break-in recovery property, while keeping deniability and forward secrecy, because the [same proof](https://eprint.iacr.org/2016/1013.pdf) as for double ratchet algorithm would hold here, provided chosen KEM is secure. diff --git a/protocol/push-notifications.md b/protocol/push-notifications.md new file mode 100644 index 000000000..6d5e1dea0 --- /dev/null +++ b/protocol/push-notifications.md @@ -0,0 +1,398 @@ +Version 2, 2024-06-22 + +# Overview of push notifications for SimpleX Messaging Servers + +## Table of contents + +- [Introduction](#introduction) +- [Participating servers](#participating-servers) +- [Register device token to receive push notifications](#register-device-token-to-receive-push-notifications) +- [Subscribe to connection notifications](#subscribe-to-connection-notifications) +- [SimpleX Notification Server protocol](#simplex-notification-server-protocol) + - [Register new notification token](#register-new-notification-token) + - [Verify notification token](#verify-notification-token) + - [Check notification token status](#check-notification-token-status) + - [Replace notification token](#replace-notification-token) + - [Delete notification token](#delete-notification-token) + - [Subscribe to periodic notifications](#subscribe-to-periodic-notifications) + - [Create SMP message notification subscription](#create-smp-message-notification-subscription) + - [Check notification subscription status](#check-notification-subscription-status) + - [Delete notification subscription](#delete-notification-subscription) + - [Error responses](#error-responses) +- [Threat model](#threat-model) + +## Introduction + +SimpleX Messaging servers already operate as push servers and deliver the messages to subscribed clients as soon as they are sent to the servers. + +The reason for push notifications is to support instant message notifications on iOS that does not allow background services. + +## Participating servers + +The diagram below shows which servers participate in message notification delivery. + +While push provider (e.g., APN) can learn how many notifications are delivered to the user, it cannot access message content, even encrypted, or any message metadata - the notifications are e2e encrypted between SimpleX Notification Server and the user's device. + +``` + User's iOS device Internet Servers +--------------------- . ------------------------ . ----------------------------- + . . + . . can be self-hosted now ++--------------+ . . +----------------+ +| SimpleX Chat | -------------- TLS --------------- | SimpleX | +| client |------> SimpleX Messaging Protocol (SMP) ------> | Messaging | ++--------------+ ---------------------------------- | Server | + ^ | . . +----------------+ + | | . . . . . | . . . + | | . . | V | + | | . . |SMP| TLS + | | . . | | | SimpleX + | | . . . . . V . . . NTF Server + | | . . +----------------------------------+ + | | . . | +---------------+ | + | | -------------- TLS --------------- | | SimpleX | can be | + | |-----------> Notification Server Protocol -----> | | Notifications | self-hosted | + | ---------------------------------- | | Subscriber | in the future | + | . . | +---------------+ | + | . . | | | + | . . | V | + | . . | +---------------+ | + | . . | | SimpleX | | + | . . | | Push | | + | . . | | Server | | + | . . | +---------------+ | + | . . +----------------------------------+ + | . . . . . | . . . + | . . | V | + | . . |SMP| TLS + | . . | | | + | . . . . . V . . . + | -------------- TLS --------------- +-----------------+ + |----------------- Notification delivery <-------| Apple PN server | + ---------------------------------- +-----------------+ + . . +``` + +## Register device token to receive push notifications + +This diagram shows the process of registering a device to receive PUSH notifications via Apple Push Notification (APN) servers. + +![Register device notification token](./diagrams/notifications/register-token.svg) + +## Subscribe to connection notifications + +This diagram shows the process of subscription to notifications, notification delivery and device token update. + +![Subscribe to notifications](./diagrams/notifications/subscription.svg) + +## SimpleX Notification Server protocol + +To manage notification subscriptions to SMP servers, SimpleX Notification Server provides an RPC protocol with a similar design to SimpleX Messaging Protocol server. + +This protocol sends requests and responses in a fixed size blocks of 512 bytes over TLS, uses the same [syntax of protocol transmissions](./simplex-messaging.md#smp-transmission-and-transport-block-structure) as SMP protocol, and has the same transport [handshake syntax](./simplex-messaging.md#transport-handshake) (except the server certificate is not included in the handshake). + +Protocol commands have this syntax: + +``` +ntfServerTransmission = +ntfServerCmd = newTokenCmd / verifyTokenCmd / checkTokenCmd / + replaceTokenCmd / deleteTokenCmd / cronCmd / + newSubCmd / checkSubCmd / deleteSubCmd +``` +### Register new notification token + +This command should be used after the client app obtains a token from push notifications provider to register the token with the server. + +Having received this command the server will deliver a test notification via the push provider to validate that the client has this token. + +The command syntax: + +```abnf +newTokenCmd = %s"TNEW" SP newToken +newToken = %s"T" deviceToken authPubKey clientDhPubKey +deviceToken = pushProvider tokenString +pushProvider = apnsDev / apnsProd / apnsNull +apnsDev = "AD" ; APNS token for development environment +apnsProd = "AP" ; APNS token for production environment +apnsNull = "AN" ; token that does not trigger any notification delivery - used for server testing +tokenString = shortString +authPubKey = length x509encoded ; Ed25519 key used to verify clients commands +clientDhPubKey = length x509encoded ; X25519 key to agree e2e encryption between the server and client +shortString = length *OCTET +length = 1*1 OCTET +``` + +The server response syntax: + +```abnf +tokenIdResp = %s"IDTKN" SP entityId serverDhPubKey +entityId = shortString +serverDhPubKey = length x509encoded ; X25519 key to agree e2e encryption between the server and client +``` + +### Verify notification token + +This command is used to verify the token after the device receives the test notification from the push provider. + +The command syntax: + +```abnf +verifyTokenCmd = %s"TVFY" SP regCode +regCode = shortString +``` + +The response to this command is `okResp` or `errorResp` + +```abnf +okResp = %s"OK" +``` + +### Check notification token status + +This command is used to check the token status: + +```abnf +checkTokenCmd = %s"TCHK" +``` + +The response to this command: + +```abnf +tokenStatusResp = %s"TKN" SP tokenStatus +tokenStatus = %s"NEW" / %s"REGISTERED" / %s"INVALID" / %s"CONFIRMED" / %s"ACTIVE" / %s"EXPIRED" +``` + +### Replace notification token + +This command should be used when push provider issues a new notification token. + +It happens when: +- the app data is migrated to another device. +- the app is re-installed on the same device. +- can happen periodically, at push provider discretion. + +This command allows to replace the token without re-registering and re-subscribing all notification subscriptions. + +Using this command triggers the same verification flow as registering a new token. + +The command syntax: + +```abnf +replaceTokenCmd = %s"TRPL" SP deviceToken +``` + +The response to this command is `okResp` or `errorResp`. + +### Delete notification token + +The command syntax: + +```abnf +deleteTokenCmd = %s"TDEL" +``` + +The response to this command is `okResp` or `errorResp`. + +After this command all message notification subscriptions will be removed and no more notifications will be sent. + +### Subscribe to periodic notifications + +This command enables or disables periodic notifications sent to the client device irrespective of message notifications. + +This is useful for two reasons: +- it provides better privacy from notification server, as while the server learns the device token, it doesn't learn anything else about user communications. +- it allows to receive messages when notifications were dropped by push provider, e.g. while the device was offline, or lost by notification server, e.g. while it was restarting. + +The command syntax: + +```abnf +cronCmd = %s"TCRN" SP interval +interval = 2*2 OCTET ; Word16, minutes +``` + +The interval for periodic notifications is set in minutes, with the minimum of 20 minutes. The client should pass `0` to disable periodic notifications. + +### Create SMP message notification subscription + +This command makes notification server subscribe to message notifications from SMP server and to deliver them to push provider: + +```abnf +newSubCmd = %s"SNEW" newSub +newSub = %s "S" tokenId smpServer notifierId notifierKey +tokenId = shortString ; returned in response to `TNEW` command +smpServer = smpServer = hosts port fingerprint +hosts = length 1*host +host = shortString +port = shortString +fingerprint = shortString +notifierId = shortString ; returned by SMP server in response to `NKEY` SMP command +notifierKey = length x509encoded ; private key used to authorize requests to subscribe to message notifications +``` + +The response syntax: + +```abnf +subIdResp = %s"IDSUB" SP entityId +``` + +### Check notification subscription status + +This command syntax: + +```abnf +checkSubCmd = %s"SCHK" +``` + +The response: + +```abnf +subStatusResp = %s"SUB" SP subStatus +subStatus = %s"NEW" / %s"PENDING" / ; e.g., after SMP server disconnect/timeout while ntf server is retrying to connect + %s"ACTIVE" / %s"INACTIVE" / %s"END" / ; if another server subscribed to notifications + %s"AUTH" / subErrStatus +subErrStatus = %s"ERR" SP shortString +``` + +### Delete notification subscription + +The command syntax: + +```abnf +deleteSubCmd = %s"SDEL" +``` + +The response to this command is `okResp` or `errorResp`. + +After this command no more message notifications will be sent from this queue. + +### Error responses + +All commands can return error response: + +```abnf +errorResp = %s"ERR" SP errorType +``` + +Where `errorType` has the same syntax as in [SimpleX Messaging Protocol](./simplex-messaging.md#error-responses) + +## Threat Model + +This threat model compliments SimpleX Messaging Protocol [threat model](./overview-tjr.md#threat-model) + +#### A passive adversary able to monitor the traffic of one user + +*can:* + + - identify that and a user is using SimpleX push notifications. + +*cannot:* + + - determine which servers a user subscribed to the notifications from. + +#### A passive adversary able to monitor a set of senders and recipients + + *can:* + + - perform more efficient traffic correlation attacks against senders and recipients and correlate senders and recipients within the monitored set, frustrated by the number of users on the servers. + +#### SimpleX Messaging Protocol server + +*can:* + +- learn which messages trigger push notifications. + +- learn IP address of SimpleX notification servers used by the user. + +- drop message notifications. + +- spam a user with invalid notifications. + +*cannot:* + +- learn user device token for push notifications. + +- learn which queues belong to the same users with any additional efficiency compared with not using push notifications. + +#### SimpleX Notification Server subscribed to message notifications + +*can:* + +- learn a user device token. + +- learn how many messaging queues and servers a user receives messages from. + +- learn how many message notifications are delivered to the user from each queue. + +- undetectably drop notifications. + +- spam a user with background notifications. + +*cannot:* + +- learn queue addresses for receiving or sending messages. + +- learn the contents or type of messages (not even encrypted). + +- learn anything about messages sent without notification flag. + +- spam a user with visible notifications (provided the client app can filter push notifications). + +- add, duplicate, or corrupt individual messages that will be shown to the user. + +#### SimpleX Notification Server subscribed ONLY to periodic notifications + +*can:* + +- learn a user device token. + +- drop periodic notifications. + +- spam a user with background notifications. + +*cannot:* + +- learn how many messaging queues and servers a user receives messages from. + +- learn how many message notifications are delivered to the user from each queue. + +- learn queue addresses for receiving or sending messages. + +- learn the contents or type of messages (not even encrypted). + +- learn anything about messages sent without notification flag. + +- spam a user with visible notifications (provided the client app can filter push notifications). + +- add, duplicate, or corrupt individual messages that will be shown to the user. + +#### A user’s contact + +*cannot:* + +- determine if a user uses push notifications or not. + +#### Push notification provider (e.g., APN) + +*can:* + +- learn that a user uses SimpleX app. + +- learn how many notifications are delivered to user's device. + +- drop notifications (in fact, APN coalesces notifications delivered while user's device is offline, delivering only the last one). + +*cannot:* + +- learn which SimpleX Messaging Protocol servers are used by a user (notifications are e2e encrypted). + +- learn which or how many messaging queues a user receives notifications from. + +- learn the contents or type of messages (not even encrypted, notifications only contain encrypted metadata). + +#### An attacker with Internet access + +*cannot:* + +- register notification token not present on attacker's device. + +- enumerate tokens or subscriptions on a SimpleX Notification Server. diff --git a/protocol/simplex-messaging.md b/protocol/simplex-messaging.md index 48843cab3..16e4e6606 100644 --- a/protocol/simplex-messaging.md +++ b/protocol/simplex-messaging.md @@ -1,3 +1,5 @@ +Version 9, 2024-06-22 + # Simplex Messaging Protocol (SMP) ## Table of contents @@ -9,39 +11,51 @@ - [Simplex queue](#simplex-queue) - [SMP queue URI](#smp-queue-uri) - [SMP procedure](#smp-procedure) +- [Fast SMP procedure](#fast-smp-procedure) - [SMP qualities and features](#smp-qualities-and-features) - [Cryptographic algorithms](#cryptographic-algorithms) +- [Deniable client authentication scheme](#deniable-client-authentication-scheme) - [Simplex queue IDs](#simplex-queue-ids) - [Server security requirements](#server-security-requirements) - [Message delivery notifications](#message-delivery-notifications) -- [SMP Transmission structure](#smp-transmission-structure) +- [SMP Transmission and transport block structure](#smp-transmission-and-transport-block-structure) - [SMP commands](#smp-commands) - [Correlating responses with commands](#correlating-responses-with-commands) - - [Command authentication](#command-authentication) + - [Command verification](#command-verification) - [Keep-alive command](#keep-alive-command) - [Recipient commands](#recipient-commands) - [Create queue command](#create-queue-command) - [Subscribe to queue](#subscribe-to-queue) - - [Secure queue command](#secure-queue-command) + - [Secure queue by recipient](#secure-queue-by-recipient) - [Enable notifications command](#enable-notifications-command) - [Disable notifications command](#disable-notifications-command) + - [Get message command](#get-message-command) - [Acknowledge message delivery](#acknowledge-message-delivery) - [Suspend queue](#suspend-queue) - [Delete queue](#delete-queue) + - [Get queue state](#get-queue-state) - [Sender commands](#sender-commands) + - [Secure queue by sender](#secure-queue-by-sender) - [Send message](#send-message) + - [Proxying sender commands](#proxying-sender-commands) + - [Request proxied session](#request-proxied-session) + - [Send command via proxy](#send-command-via-proxy) + - [Forward command to destination server](#forward-command-to-destination-server) - [Notifier commands](#notifier-commands) - [Subscribe to queue notifications](#subscribe-to-queue-notifications) - [Server messages](#server-messages) - - [Queue IDs response](#queue-ids-response) - [Deliver queue message](#deliver-queue-message) - - [Notifier queue ID response](#notifier-queue-id-response) - [Deliver message notification](#deliver-message-notification) - [Subscription END notification](#subscription-end-notification) - [Error responses](#error-responses) - [OK response](#ok-response) -- [Appendices](#appendices) - - [Appendix A. Transport connection with the SMP server](#appendix-a) +- [Transport connection with the SMP server](#transport-connection-with-the-SMP-server) + - [General transport protocol considerations](#general-transport-protocol-considerations) + - [TLS transport encryption](#tls-transport-encryption) + - [Server certificate](#server-certificate) + - [ALPN to agree handshake version](#alpn-to-agree-handshake-version) + - [Transport handshake](#transport-handshake) + - [Additional transport privacy](#additional-transport-privacy) ## Abstract @@ -51,11 +65,11 @@ It's designed with the focus on communication security and integrity, under the It is designed as a low level protocol for other application protocols to solve the problem of secure and private message transmission, making [MITM attack][1] very difficult at any part of the message transmission system. -This document describes SMP protocol versions 3 and 4, the previous versions are discontinued. +This document describes SMP protocol versions 6 and 7, the previous versions are discontinued. ## Introduction -The objective of Simplex Messaging Protocol (SMP) is to facilitate the secure and private unidirectional transfer of messages from senders to recipients via persistent simplex queues managed by the message broker (server). +The objective of Simplex Messaging Protocol (SMP) is to facilitate the secure and private unidirectional transfer of messages from senders to recipients via persistent simplex queues managed by the message brokers (servers). SMP is independent of any particular transmission system and requires only a reliable ordered data stream channel. While this document describes transport over TCP, other transports are also possible. @@ -67,7 +81,7 @@ The protocol is designed with the focus on privacy and security, to some extent SMP does not use any form of participants' identities and provides [E2EE][2] without the possibility of [MITM attack][1] relying on two pre-requisites: -- the users can establish a secure encrypted transport connection with the SMP server. [Appendix A](#appendix-a) describes SMP transport protocol of such connection over TCP, but any other transport connection protocol can be used. +- the users can establish a secure encrypted transport connection with the SMP server. [Transport connection](#transport-connection-with-the-smp-server) section describes SMP transport protocol of such connection over TCP, but any other transport connection protocol can be used. - the recipient can pass a single message to the sender via a pre-existing secure and private communication channel (out-of-band message) - the information in this message is used to encrypt messages and to establish connection with SMP server. @@ -77,13 +91,13 @@ The SMP model has three communication participants: the recipient, the message b SMP server manages multiple "simplex queues" - data records on the server that identify communication channels from the senders to the recipients. The same communicating party that is the sender in one queue, can be the recipient in another - without exposing this fact to the server. -The queue record consists of 2 unique random IDs generated by the server, one for the recipient and another for the sender, and 2 keys to authenticate the recipient and the sender respectively, provided by the client. The users of SMP protocol must use a unique key for each queue, to avoid the possibility of aggregating and analyzing their queues in case SMP server is compromised. +The queue record consists of 2 unique random IDs generated by the server, one for the recipient and another for the sender, and 2 keys to verify the recipient's and the sender's commands, provided by the clients. The users of SMP protocol must use a unique ephemeral keys for each queue, to prevent aggregating their queues by keys in case SMP server is compromised. Creating and using the queue requires sending commands to the SMP server from the recipient and the sender - they are described in detail in [SMP commands](#smp-commands) section. ## Out-of-band messages -The out-of-band message with the queue information is sent via some trusted alternative channel from the recipient to the sender. This message is used to share one or several [queue URIs](#smp-queue-uri) that parties can use to establish the initial connection, the encryption scheme and, it can include the public key(s) for end-to-end encryption. +The out-of-band message with the queue information is sent via some trusted alternative channel from the recipient to the sender. This message is used to share one or several [queue URIs](#smp-queue-uri) that parties can use to establish the initial connection, the encryption scheme, including the public key(s) for end-to-end encryption. The approach to out-of-band message passing and their syntax should be defined in application-level protocols. @@ -91,9 +105,9 @@ The approach to out-of-band message passing and their syntax should be defined i The simplex queue is the main unit of SMP protocol. It is used by: -- Sender of the queue (who received out-of-band message) to send messages to the server using sender's queue ID, signed by sender's key. +- Sender of the queue (who received out-of-band message) to send messages to the server using sender's queue ID, authorized by sender's key. -- Recipient of the queue (who created the queue and sent out-of-band message) will use it to retrieve messages from the server, signing the commands by the recipient key. Recipient decrypts the messages with the key negotiated during the creation of the queue. +- Recipient of the queue (who created the queue and sent out-of-band message) will use it to retrieve messages from the server, authorizing the commands by the recipient key. Recipient decrypts the messages with the key negotiated during the creation of the queue. - Participant identities are not shared with the server - new unique keys and queue IDs are used for each queue. @@ -101,7 +115,7 @@ This simplex queue can serve as a building block for more complex communication This approach is based on the concept of [unidirectional networks][4] that are used for applications with high level of information security. -Access to each queue is controlled with unique (not shared with other queues) asymmetric key pairs, separate for the sender and the recipient. The sender and the receiver have private keys, and the server has associated public keys to authenticate participants' commands by verifying cryptographic signatures. +Access to each queue is controlled with unique (not shared with other queues) asymmetric key pairs, separate for the sender and the recipient. The sender and the receiver have private keys, and the server has associated public keys to authenticate participants' commands by verifying cryptographic authorizations. The messages sent over the queue are end-to-end encrypted using the DH secret agreed via out-of-band message and SMP confirmation. @@ -115,17 +129,21 @@ The protocol uses different IDs for sender and recipient in order to provide an ## SMP queue URI -The SMP queue URIs MUST include server identity, queue hostname, an optional port, sender queue ID and the public key that the clients must use to verify responses. Server identity is used to establish secure connection protected from MITM attack with SMP server (see [Appendix A](#appendix-a) for SMP transport protocol). +The SMP queue URIs MUST include server identity, queue hostname, an optional port, sender queue ID, and the recipient's public key to agree shared secret for e2e encryption, and an optional query string parameter `k=s` to indicate that the queue can be secured by the sender using `SKEY` command (see [Fast SMP procedure](#fast-smp-procedure) and [Secure queue by sender](#secure-queue-by-sender)). Server identity is used to establish secure connection protected from MITM attack with SMP server (see [Transport connection](#transport-connection-with-the-smp-server) for SMP transport protocol). The [ABNF][8] syntax of the queue URI is: ```abnf -queueURI = %s"smp://" smpServer "/" queueId "#" recipientDhPublicKey -smpServer = serverIdentity "@" srvHost [":" port] -srvHost = ; RFC1123, RFC5891 +queueURI = %s"smp://" smpServer "/" queueId "#/?" versionParam keyParam [sndSecureParam] +smpServer = serverIdentity "@" srvHosts [":" port] +srvHosts = ["," srvHosts] ; RFC1123, RFC5891 port = 1*DIGIT serverIdentity = base64url queueId = base64url +versionParam = %s"v=" versionRange +versionRange = 1*DIGIT / 1*DIGIT "-" 1*DIGIT +keyParam = %s"&dh=" recipientDhPublicKey +sndSecureParam = %s"&k=s" base64url = ; RFC4648, section 5 recipientDhPublicKey = x509UrlEncoded ; the recipient's Curve25519 key for DH exchange to derive the secret @@ -139,7 +157,7 @@ x509UrlEncoded = `port` is optional, the default TCP port for SMP protocol is 5223. -`serverIdentity` is a required hash of the server certificate SPKI block (without line breaks, header and footer) used by the client to validate server certificate during transport handshake (see [Appendix A](#appendix-a)) +`serverIdentity` is a required hash of the server certificate SPKI block (without line breaks, header and footer) used by the client to validate server certificate during transport handshake (see [Transport connection](#transport-connection-with-the-smp-server)) ## SMP procedure @@ -149,17 +167,17 @@ To create and start using a simplex queue Alice and Bob follow these steps: 1. Alice creates a simplex queue on the server: - 1. Decides which SMP server to use (can be the same or different server that Alice uses for other queues) and opens secure encrypted transport connection to the chosen SMP server (see [Appendix A](#appendix-a)). + 1. Decides which SMP server to use (can be the same or different server that Alice uses for other queues) and opens secure encrypted transport connection to the chosen SMP server (see [Transport connection](#transport-connection-with-the-smp-server)). - 2. Generates a new random public/private key pair (encryption key - `EK`) that she did not use before for Bob to encrypt the messages. + 2. Generates a new random public/private key pair (encryption key - `EK`) that she did not use before to agree a shared secret with Bob to encrypt the messages. - 3. Generates another new random public/private key pair (recipient key - `RK`) that she did not use before for her to sign commands and to decrypt the transmissions received from the server. + 3. Generates another new random public/private key pair (recipient key - `RK`) that she did not use before for her to authorize commands to the server. 4. Generates one more random key pair (recipient DH key - `RDHK`) to negotiate symmetric key that will be used by the server to encrypt message bodies delivered to Alice (to avoid shared cipher-text inside transport connection). - 5. Sends `"NEW"` command to the server to create a simplex queue (see `create` in [Create queue command](#create-queue-command)). This command contains previously generated unique "public" keys `RK` and `RDHK`. `RK` will be used to verify the following commands related to the same queue signed by its private counterpart, for example to subscribe to the messages received to this queue or to update the queue, e.g. by setting the key required to send the messages (initially Alice creates the queue that accepts unsigned messages, so anybody could send the message via this queue if they knew the queue sender's ID and server address). + 5. Sends `"NEW"` command to the server to create a simplex queue (see `create` in [Create queue command](#create-queue-command)). This command contains previously generated unique "public" keys `RK` and `RDHK`. `RK` will be used by the server to verify the subsequent commands related to the same queue authorized by its private counterpart, for example to subscribe to the messages received to this queue or to update the queue, e.g. by setting the key required to send the messages (initially Alice creates the queue that accepts unauthorized messages, so anybody could send the message via this queue if they knew the queue sender's ID and server address). - 6. The server sends `"IDS"` response with queue IDs (`queueIds`): + 6. The server sends `IDS` response with queue IDs (`queueIds`): - Recipient ID `RID` for Alice to manage the queue and to receive the messages. @@ -167,27 +185,29 @@ To create and start using a simplex queue Alice and Bob follow these steps: - Server public DH key (`SDHK`) to negotiate a shared secret for message body encryption, that Alice uses to derive a shared secret with the server `SS`. -2. Alice sends an out-of-band message to Bob via the alternative channel that both Alice and Bob trust (see [protocol abstract](#simplex-messaging-protocol-abstract)). The message must include: +2. Alice sends an out-of-band message to Bob via the alternative channel that both Alice and Bob trust (see [protocol abstract](#simplex-messaging-protocol-abstract)). The message must include [SMP queue URI](#smp-queue-uri) with: - - Unique "public" key (`EK`) that Bob must use for E2E key agreement. + - Unique "public" key (`EK`) that Bob must use to agree a shared secret for E2E encryption. - - SMP server hostname and information to open secure encrypted transport connection (see [Appendix A](#appendix-a)). + - SMP server hostname and information to open secure encrypted transport connection (see [Transport connection](#transport-connection-with-the-smp-server)). - Sender queue ID `SID` for Bob to use. 3. Bob, having received the out-of-band message from Alice, connects to the queue: - 1. Generates a new random public/private key pair (sender key - `SK`) that he did not use before for him to sign messages sent to Alice's server. + 1. Generates a new random public/private key pair (sender key - `SK`) that he did not use before for him to authorize messages sent to Alice's server and another key pair for e2e encryption agreement. 2. Prepares the confirmation message for Alice to secure the queue. This message includes: - - Previously generated "public" key `SK` that will be used by Alice's server to authenticate Bob's messages, once the queue is secured. + - Previously generated "public" key `SK` that will be used by Alice's server to verify Bob's messages, once the queue is secured. + + - Public key to agree a shared secret with Alice for e2e encryption. - Optionally, any additional information (application specific, e.g. Bob's profile name and details). - 3. Encrypts the confirmation body with the "public" key `EK` (that Alice provided via the out-of-band message). + 3. Encrypts the confirmation body with the shared secret agreed using public key `EK` (that Alice provided via the out-of-band message). - 4. Sends the encrypted message to the server with queue ID `SID` (see `send` in [Send message](#send-message)). This initial message to the queue must not be signed - signed messages will be rejected until Alice secures the queue (below). + 4. Sends the encrypted message to the server with queue ID `SID` (see `send` in [Send message](#send-message)). This initial message to the queue must not be authorized - authorized messages will be rejected until Alice secures the queue (below). 4. Alice receives Bob's message from the server using recipient queue ID `RID` (possibly, via the same transport connection she already has opened - see `message` in [Deliver queue message](#deliver-queue-message)): @@ -195,13 +215,13 @@ To create and start using a simplex queue Alice and Bob follow these steps: 2. She decrypts received message with [key agreed with sender using] "private" key `EK`. - 3. Even though anybody could have sent the message to the queue with ID `SID` before it is secured (e.g. if communication is compromised), Alice would ignore all messages until the decryption succeeds (i.e. the result contains the expected message format). Optionally, in the client application, she also may identify Bob using the information provided, but it is out of scope of SMP protocol. + 3. Anybody can send the message to the queue with ID `SID` before it is secured (e.g. if communication is compromised), so it's a "race" to secure the queue. Optionally, in the client application, Alice may identify Bob using the information provided, but it is out of scope of SMP protocol. 5. Alice secures the queue `RID` with `"KEY"` command so only Bob can send messages to it (see [Secure queue command](#secure-queue-command)): - 1. She sends the `KEY` command with `RID` signed with "private" key `RK` to update the queue to only accept requests signed by "private" key `SK` provided by Bob. This command contains unique "public" key `SK` previously generated by Bob. + 1. She sends the `KEY` command with `RID` signed with "private" key `RK` to update the queue to only accept requests authorized by "private" key `SK` provided by Bob. This command contains unique "public" key `SK` previously generated by Bob. - 2. From this moment the server will accept only signed commands to `SID`, so only Bob will be able to send messages to the queue `SID` (corresponding to `RID` that Alice has). + 2. From this moment the server will accept only authorized commands to `SID`, so only Bob will be able to send messages to the queue `SID` (corresponding to `RID` that Alice has). 3. Once queue is secured, Alice deletes `SID` and `SK` - even if Alice's client is compromised in the future, the attacker would not be able to send messages pretending to be Bob. @@ -217,19 +237,19 @@ Bob now can securely send messages to Alice: 1. Bob sends the message: - 1. He encrypts the message to Alice with "public" key `EK` (provided by Alice, only known to Bob, used only for one simplex queue). + 1. He encrypts the message to Alice with the agreed shared secret (using "public" key `EK` provided by Alice, only known to Bob, used only for one simplex queue). - 2. He signs `"SEND"` command to the server queue `SID` using the "private" key `SK` (that only he knows, used only for this queue). + 2. He authorizes `"SEND"` command to the server queue `SID` using the "private" key `SK` (that only he knows, used only for this queue). - 3. He sends the command to the server (see `send` in [Send message](#send-message)), that the server will authenticate using the "public" key `SK` (that Alice earlier received from Bob and provided to the server via `"KEY"` command). + 3. He sends the command to the server (see `send` in [Send message](#send-message)), that the server will verify using the "public" key `SK` (that Alice earlier received from Bob and provided to the server via `"KEY"` command). 2. Alice receives the message(s): - 1. She signs `"SUB"` command to the server to subscribe to the queue `RID` with the "private" key `RK` (see `subscribe` in [Subscribe to queue](#subscribe-to-queue)). + 1. She authorizes `"SUB"` command to the server to subscribe to the queue `RID` with the "private" key `RK` (see `subscribe` in [Subscribe to queue](#subscribe-to-queue)). - 2. The server, having authenticated Alice's command with the "public" key `RK` that she provided, delivers Bob's message(s) (see `message` in [Deliver queue message](#deliver-queue-message)). + 2. The server, having verified Alice's command with the "public" key `RK` that she provided, delivers Bob's message(s) (see `message` in [Deliver queue message](#deliver-queue-message)). - 3. She decrypts Bob's message(s) with the "private" key `EK` (that only she has). + 3. She decrypts Bob's message(s) with the shared secret agreed using "private" key `EK`. 4. She acknowledges the message reception to the server with `"ACK"` so that the server can delete the message and deliver the next messages. @@ -245,15 +265,27 @@ This flow is show on sequence diagram below. Sequence diagram does not show E2E encryption - server knows nothing about encryption between the sender and the receiver. -A higher level protocol application protocol should define the semantics that allow to use two simplex queues (or two sets of queues for redundancy) for the bi-directional or any other communication scenarios. +A higher level application protocol should define the semantics that allow to use two simplex queues (or two sets of queues for redundancy) for the bi-directional or any other communication scenarios. The SMP is intentionally unidirectional - it provides no answer to how Bob will know that the transmission succeeded, and whether Alice received any messages. There may be a scenario when Alice wants to securely receive the messages from Bob, but she does not want Bob to have any proof that she received any messages - this low-level protocol can be used in this scenario, as all Bob knows as a fact is that he was able to send one unsigned message to the server that Alice provided, and now he can only send messages signed with the key `SK` that he sent to the server - it does not prove that any message was received by Alice. -For practical purposes of bi-directional conversation, now that Bob can securely send encrypted messages to Alice, Bob can create the second simplex queue that will allow Alice to send messages to Bob in the same way, sending the second queue details via the first queue. If both Alice and Bob have their respective unique "public" keys (Alice's and Bob's `EK`s of two separate queues), or pass additional keys to sign the messages, the conversation can be both encrypted and signed. +For bi-directional conversation, now that Bob can securely send encrypted messages to Alice, Bob can create the second simplex queue that will allow Alice to send messages to Bob in the same way, sending the second queue details via the first queue. If both Alice and Bob have their respective unique "public" keys (Alice's and Bob's `EK`s of two separate queues), or pass additional keys to sign the messages, the conversation can be both encrypted and signed. The established queues can also be used to change the encryption keys providing [forward secrecy][5], or to negotiate using other SMP queue(s). -This protocol also can be used for off-the-record messaging, as Alice and Bob can use multiple queues between them and only information they pass to each other allows proving their identity, so if they want to share anything off-the-record they can initiate a new queue without linking it to any other information they exchanged. As a result, this protocol provides better anonymity and better protection from [MITM][1] than [OTR][6] protocol. +This protocol also can be used for off-the-record messaging, as Alice and Bob can use multiple queues between them and only information they pass to each other allows proving their identity, so if they want to share anything off-the-record they can initiate a new queue without linking it to any other information they exchanged. + +## Fast SMP procedure + +V9 of SMP protocol added support for creating messaging queue in fewer steps, with the sender being able to send the messages without waiting for the recipient to be online to secure the key. + +In step 1.5 of [SMP procedure](#smp-procedure) the client must use sndSecure parameter set to `T` (true) to allow sender securing the queue. + +In step 2, the [SMP queue URI](#smp-queue-uri) should include parameter indicating that the sender can secure the queue. + +In step 3.2, prior to sending the confirmation message Bob secures the queue using `SKEY` command. Confirmation message is now sent with sender authorization and Bob can continue sending the messages without Alice being online. This also allows faster negotiation of duplex connections. + +![Creating queue](./diagrams/simplex-messaging/simplex-creating-fast.svg) ## SMP qualities and features @@ -269,11 +301,11 @@ Simplex Messaging Protocol: - Multiple servers, that can be deployed by the system users, can be used to send and retrieve messages. - - Servers do not communicate with each other and do not "know" about other servers. + - Servers do not communicate with each other, except when used as proxy to forward commands to another server, and do not "know" about other servers. - Clients only communicate with servers (excluding the initial out-of-band message), so the message passing is asynchronous. - - For each queue, the message recipient defines the server through which the sender should send messages. + - For each queue, the message recipient defines the server through which the sender should send messages. To protect transport anonymity the sender can use their chosen server to forward commands to the server chosen by the recipient. - While multiple servers and multiple queues can be used to pass each message, it is in scope of application level protocol(s), and out of scope of this protocol. @@ -285,50 +317,56 @@ Simplex Messaging Protocol: - Each queue is created and managed by the queue recipient. - - Asymmetric encryption is used to sign and verify the requests to send and receive the messages. + - Asymmetric encryption is used to authorize and verify the requests to send and receive the messages. - - One unique "public" key is used by the servers to authenticate requests to send the messages into the queue, and another unique "public" key - to retrieve the messages from the queue. "Unique" here means that each "public" key is used only for one queue and is not used for any other context - effectively, this key is not public and does not represent any participant identity. + - One ephemeral public key is used by the servers to verify requests to send the messages into the queue, and another ephemeral public key - to verify requests to retrieve the messages from the queue. These ephemeral keys are used only for one queue, and are not used for any other context - this key does not represent any participant identity. - - Both recipient and sender "public" keys are provided to the server by the queue recipient. "Public" key `RK` is provided when the queue is created, public key `SK` is provided when the queue is secured. + - Both recipient and sender public keys are provided to the server by the queue recipient. "Public" key `RK` is provided when the queue is created, public key `SK` is provided when the queue is secured. V9 of SMP protocol allows senders to provide their key to the server directly or via proxy, to avoid waiting until the recipient is online to secure the queue. - - The "public" keys known to the server and used to authenticate commands from the participants are unrelated to the keys used to encrypt and decrypt the messages - the latter keys are also unique per each queue but they are only known to participants, not to the servers. + - The "public" keys known to the server and used to verify commands from the participants are unrelated to the keys used to encrypt and decrypt the messages - the latter keys are also unique per each queue but they are only known to participants, not to the servers. - Messaging graph can be asymmetric: Bob's ability to send messages to Alice does not automatically lead to the Alice's ability to send messages to Bob. ## Cryptographic algorithms -Simplex messaging clients and servers must cryptographically sign commands, responses and messages for the following operations: +Simplex messaging clients must cryptographically authorize commands for the following operations: - With the recipient's key `RK` (server to verify): - create the queue (`NEW`) - subscribe to queue (`SUB`) - secure the queue (`KEY`) - enable queue notifications (`NKEY`) + - disable queue notifications (`NDEL`) - acknowledge received messages (`ACK`) - suspend the queue (`OFF`) - delete the queue (`DEL`) - With the sender's key `SK` (server to verify): + - secure queue (`SKEY`) - send messages (`SEND`) - With the optional notifier's key: - subscribe to message notifications (`NSUB`) -- With the server's key (for recipient and sender to verify) - - queue IDs response (`IDS`) - - notifier queue ID response (`NID`) - - delivered messages (`MSG`) - - `OK` and `ERR` responses (excluding error responses not related to a queue) -To sign/verify transmissions clients and servers MUST use Ed25519 or Ed448 algorithm defined in [RFC8709][15]. +To authorize/verify transmissions clients and servers MUST use either signature algorithm Ed25519 algorithm defined in [RFC8709][15] or [deniable authentication scheme](#deniable-client-authentication-scheme) based on NaCL crypto_box. -To encrypt/decrypt message bodies delivered to the recipients, servers/clients MUST use x25519 or x448 algorithm defined in [RFC8709][15] to derive the shared secret (TODO encryption scheme). +It is recommended that clients use signature algorithm for the recipient commands and deniable authentication scheme for sender commands (to have non-repudiation quality in the whole protocol stack). -Clients MUST encrypt message bodies sent via SMP servers - the protocol for this end-to-end encryption should be chosen by the clients using SMP protocol. +To encrypt/decrypt message bodies delivered to the recipients, servers/clients MUST use NaCL crypto_box. -The reasons to use these algorithms: +Clients MUST encrypt message bodies sent via SMP servers using use NaCL crypto_box. -- Faster operation than RSA algorithms. -- DH key exchange provides forward secrecy. +## Deniable client authentication scheme -Future versions of the protocol may allow different cryptographic algorithms. +While e2e encryption algorithms used in the client applications have repudiation quality, which is the desirable default, using signature algorithm for command authorization has non-repudiation quality. + +SMP protocol supports repudiable authenticators to authorize client commands. These authenticators use NaCl crypto_box that proves authentication and third party unforgeability and, unlike signature, provides repudiation guarantee. See [crypto_box docs](https://nacl.cr.yp.to/box.html). + +When queue is created or secured, the recipient would provide a DH key (X25519) to the server (either their own or received from the sender, in case of KEY command), and the server would provide its own random X25519 key per session in the handshake header. The authenticator is computed in this way: + +```abnf +transmission = authenticator authorized +authenticator = crypto_box(sha512(authorized), secret = dh(client long term queue key, server session key), nonce = correlation ID) +authorized = sessionIdentifier corrId queueId protocol_command ; same as the currently signed part of the transmission +``` ## Simplex queue IDs @@ -351,7 +389,7 @@ Simplex messaging server implementations MUST NOT create, store or send to any o ## Message delivery notifications -Supporting message delivery while the client mobile app is not running requires sending push notifications with the device token. All alternative mechanisms for background message delivery are unreliable, particularly on iOS platform. Obviously, supporting push notification delivery by simply subscribing to messages would reduce meta-data privacy as it allows to see all queues that a given device uses. +Supporting message delivery while the client mobile app is not running requires sending push notifications with the device token. All alternative mechanisms for background message delivery are unreliable, particularly on iOS platform. To protect the privacy of the recipients, there are several commands in SMP protocol that allow enabling and subscribing to message notifications from SMP queues, using separate set of "notifier keys" and via separate queue IDs - as long as SMP server is not compromised, these notifier queue IDs cannot be correlated with recipient or sender queue IDs. @@ -362,14 +400,14 @@ The clients can optionally instruct a dedicated push notification server to subs - `subscribeNotifications` (`"NSUB"`) - see [Subscribe to queue notifications](#subscribe-to-queue-notifications). - `messageNotification` (`"NMSG"`) - see [Deliver message notification](#deliver-message-notification). -[`SEND` command](#send-message) includes the notification flag to instruct SMP server whether to send the notification - this flag is forwarded to the recepient inside encrypted envelope, together with the timestamp and the message body, so even if TLS is compromised this flag cannot be used for traffic correlation. +[`SEND` command](#send-message) includes the notification flag to instruct SMP server whether to send the notification - this flag is forwarded to the recipient inside encrypted envelope, together with the timestamp and the message body, so even if TLS is compromised this flag cannot be used for traffic correlation. ## SMP Transmission and transport block structure Each transport block has a fixed size of 16384 bytes for traffic uniformity. -From SMP version 4 each block can contain multiple transmissions, version 3 blocks have 1 transmission. -Some parts of SMP transmission are padded to a fixed size; this padding is uniformly added as a word16 encoded in network byte order - see `paddedString` syntax. +Each block can contain multiple transmissions. +Some parts of SMP transmission are padded to a fixed size; the size of the unpadded string is prepended as a word16 encoded in network byte order - see `paddedString` syntax. In places where some part of the transmission should be padded, the syntax for `paddedNotation` is used: @@ -383,26 +421,7 @@ paddedNotation = ; paddedLength - required length after padding, including 2 bytes for originalLength ``` -Each transmission/block for SMP v3 between the client and the server must have this format/syntax: - -```abnf -paddedTransmission = -transmission = signature signed -signed = sessionIdentifier corrId queueId smpCommand -; corrId is required in client commands and server responses, -; it is empty in server notifications. -corrId = length *OCTET -queueId = length *OCTET -; empty queue ID is used with "create" command and in some server responses -signature = length *OCTET -; empty signature can be used with "send" before the queue is secured with secure command -; signature is always empty with "ping" and "serverMsg" -length = 1*1 OCTET -``` - -`base64` encoding should be used with padding, as defined in section 4 of [RFC 4648][9] - -Transport block for SMP v4 has this syntax: +Transport block for SMP transmission between the client and the server must have this syntax: ```abnf paddedTransportBlock = @@ -410,6 +429,24 @@ transportBlock = transmissionCount transmissions transmissionCount = 1*1 OCTET ; equal or greater than 1 transmissions = transmissionLength transmission [transmissions] transmissionLength = 2*2 OCTET ; word16 encoded in network byte order + +transmission = authorization authorized +authorized = sessionIdentifier corrId entityId smpCommand +corrId = %x18 24*24 OCTET / %x0 "" + ; corrId is required in client commands and server responses, + ; it is empty (0-length) in server notifications. + ; %x18 is 24 - the random correlation ID must be 24 bytes as it is used as a nonce for NaCL crypto_box in some contexts. +entityId = shortString ; queueId or proxySessionId + ; empty entityId ID is used with "create" command and in some server responses +authorization = shortString ; signature or authenticator + ; empty authorization can be used with "send" before the queue is secured with secure command + ; authorization is always empty with "ping" and server responses +sessionIdentifier = "" ; +sessionIdentifierForAuth = shortString + ; sessionIdentifierForAuth MUST be included in authorized transmission body. + ; From v7 of SMP protocol but it is no longer used in the transmission to save space and fit more transmissions in the transport block. +shortString = length *OCTET ; length prefixed bytearray 0-255 bytes +length = 1*1 OCTET ``` ## SMP commands @@ -417,11 +454,16 @@ transmissionLength = 2*2 OCTET ; word16 encoded in network byte order Commands syntax below is provided using [ABNF][8] with [case-sensitive strings extension][8a]. ```abnf -smpCommand = ping / recipientCmd / send / subscribeNotifications / serverMsg -recipientCmd = create / subscribe / secure / enableNotifications / disableNotifications / - acknowledge / suspend / delete +smpCommand = ping / recipientCmd / senderCommand / + proxyCommand / subscribeNotifications / serverMsg +recipientCmd = create / subscribe / rcvSecure / + enableNotifications / disableNotifications / getMessage + acknowledge / suspend / delete / getQueueInfo +senderCommand = send / sndSecure +proxyCommand = proxySession / proxyCommand / relayCommand serverMsg = queueIds / message / notifierId / messageNotification / - unsubscribed / ok / error + proxySessionKey / proxyResponse / relayResponse + unsubscribed / queueInfo/ ok / error ``` The syntax of specific commands and responses is defined below. @@ -432,9 +474,9 @@ The server should send `queueIds`, `error` and `ok` responses in the same order If the transport connection is closed before some responses are sent, these responses should be discarded. -### Command authentication +### Command verification -SMP servers must authenticate all transmissions (excluding `ping` and initial `send` commands) by verifying the client signatures. Command signature should be generated by applying the algorithm specified for the queue to the `signed` block of the transmission, using the key associated with the queue ID (recipient's, sender's or notifier's, depending on which queue ID is used). +SMP servers must verify all transmissions (excluding `ping` and initial `send` commands) by verifying the client authorizations. Command authorization should be generated by applying the algorithm specified for the queue to the `signed` block of the transmission, using the key associated with the queue ID (recipient's, sender's or notifier's, depending on which queue ID is used). ### Keep-alive command @@ -452,37 +494,42 @@ Sending any of the commands in this section (other than `create`, that is sent w #### Create queue command -This command is sent by the recipient to the SMP server to create a new queue. The syntax is: +This command is sent by the recipient to the SMP server to create a new queue. + +Servers SHOULD support basic auth with this command, to allow only server owners and trusted users to create queues on the destiation servers. + +The syntax is: ```abnf -create = %s"NEW " recipientSignaturePublicKey recipientDhPublicKey -recipientSignaturePublicKey = length x509encoded -; the recipient's Ed25519 or Ed448 public key to verify commands for this queue - +create = %s"NEW " recipientAuthPublicKey recipientDhPublicKey basicAuth subscribe sndSecure +recipientAuthPublicKey = length x509encoded +; the recipient's Ed25519 or X25519 public key to verify commands for this queue recipientDhPublicKey = length x509encoded ; the recipient's Curve25519 key for DH exchange to derive the secret ; that the server will use to encrypt delivered message bodies ; using [NaCl crypto_box][16] encryption scheme (curve25519xsalsa20poly1305). +basicAuth = "0" / "1" shortString ; server password +subscribeMode = %s"S" / %s"C" ; S - create and subscribe, C - only create +sndSecure = %s"T" / %s"F" ; T - sender can secure the queue, from v9 x509encoded = - length = 1*1 OCTET ``` -If the queue is created successfully, the server must send `queueIds` response with the recipient's and sender's queue IDs and public keys to sign all responses and messages and to encrypt delivered message bodies: +If the queue is created successfully, the server must send `queueIds` response with the recipient's and sender's queue IDs and public key to encrypt delivered message bodies: ```abnf -queueIds = %s"IDS " recipientId senderId srvDhPublicKey +queueIds = %s"IDS " recipientId senderId srvDhPublicKey sndSecure serverDhPublicKey = length x509encoded ; the server's Curve25519 key for DH exchange to derive the secret ; that the server will use to encrypt delivered message bodies to the recipient -recipientId = length *OCTET ; 16-24 bytes -senderId = length *OCTET ; 16-24 bytes +recipientId = shortString ; 16-24 bytes +senderId = shortString ; 16-24 bytes ``` -Once the queue is created, the recipient gets automatically subscribed to receive the messages from that queue, until the transport connection is closed. The `subscribe` command is needed only to start receiving the messages from the existing queue when the new transport connection is opened. +Once the queue is created, depending on `subscribeMode` parameter of `NEW` command the recipient gets automatically subscribed to receive the messages from that queue, until the transport connection is closed. To start receiving the messages from the existing queue when the new transport connection is opened the client must use `subscribe` command. -`NEW` transmission MUST be signed using the private part of the `recipientSignaturePublicKey` – this verifies that the client has the private key that will be used to sign subsequent commands for this queue. +`NEW` transmission MUST be authorized using the private part of the `recipientAuthPublicKey` – this verifies that the client has the private key that will be used to authorize subsequent commands for this queue. `IDS` response transmission MUST be sent with empty queue ID (the third part of the transmission). @@ -500,19 +547,23 @@ The first message will be delivered either immediately or as soon as it is avail This transmission and its response MUST be signed. -#### Secure queue command +#### Secure queue by recipient + +This command is only used until v8 of SMP protocol. V9 uses [SKEY](#secure-queue-by-sender). This command is sent by the recipient to the server to add sender's key to the queue: ```abnf -secure = %s"KEY " senderSignaturePublicKey -senderSignaturePublicKey = length x509encoded -; the sender's Ed25519 or Ed448 key to verify SEND commands for this queue +rcvSecure = %s"KEY " senderAuthPublicKey +senderAuthPublicKey = length x509encoded +; the sender's Ed25519 or X25519 key to verify SEND commands for this queue ``` `senderKey` is received from the sender as part of the first message - see [Send Message](#send-message) command. -Once the queue is secured only signed messages can be sent to it. +Once the queue is secured only authorized messages can be sent to it. + +This command MUST be used in transmission with recipient queue ID. #### Enable notifications command @@ -521,7 +572,7 @@ This command is sent by the recipient to the server to add notifier's key to the ```abnf enableNotifications = %s"NKEY " notifierKey recipientNotificationDhPublicKey notifierKey = length x509encoded -; the notifier's Ed25519 or Ed448 public key public key to verify NSUB command for this queue +; the notifier's Ed25519 or X25519 public key to verify NSUB command for this queue recipientNotificationDhPublicKey = length x509encoded ; the recipient's Curve25519 key for DH exchange to derive the secret @@ -533,7 +584,7 @@ The server will respond with `notifierId` response if notifications were enabled ```abnf notifierId = %s"NID " notifierId srvNotificationDhPublicKey -notifierId = length *OCTET ; 16-24 bytes +notifierId = shortString ; 16-24 bytes srvNotificationDhPublicKey = length x509encoded ; the server's Curve25519 key for DH exchange to derive the secret ; that the server will use to encrypt notification metadata to the recipient (encryptedNMsgMeta in NMSG) @@ -555,17 +606,30 @@ The server must respond `ok` to this command if it was successful. Once notifier's credentials are removed server will no longer send "NMSG" for this queue to notifier. +#### Get message command + +The client can use this command to receive one message without subscribing to the queue. This command is used when processing push notifications. + +The client MUST NOT use `SUB` and `GET` command on the same queue in the same transport connection - doing so would create an error. + +```abnf +getMessage = %s"GET" +``` + #### Acknowledge message delivery The recipient should send the acknowledgement of message delivery once the message was stored in the client, to notify the server that the message should be deleted: ```abnf -acknowledge = %s"ACK" +acknowledge = %s"ACK" SP msgId +msgId = shortString ``` -Even if acknowledgement is not sent by the recipient, the server should limit the time of message storage, whether it was delivered to the recipient or not. +Client must send message ID to acknowledge a particular message - to prevent double acknowledgement (e.g., when command response times out) resulting in message being lost. If the message was not delivered or if the ID of the message does not match the last delivered message, the server SHOULD respond with `ERR NO_MSG` error. -Having received the acknowledgement, SMP server should immediately delete the message and then send the next available message or respond with `ok` if there are no more messages available in this simplex queue. +The server should limit the time the message is stored, even if the message was not delivered or if acknowledgement is not sent by the recipient. + +Having received the acknowledgement, SMP server should delete the message and then send the next available message or respond with `ok` if there are no more messages available in this simplex queue. #### Suspend queue @@ -593,10 +657,67 @@ All undelivered messages must be deleted as soon as this command is received, be delete = %s"DEL" ``` +#### Get queue state + +This command is used by the queue recipient to get the debugging information about the current state of the queue. + +The response to that command is `INFO`. + +```abnf +getQueueInfo = %s"QUE" +queueInfo = %s"INFO " info +info = +``` + +The format of queue information is implementation specific, and is not part of the specification. For information, [JTD schema][17] for queue information returned by the reference implementation of SMP server is: + +```json +{ + "properties": { + "qiSnd": {"type": "boolean"}, + "qiNtf": {"type": "boolean"}, + "qiSize": {"type": "uint16"} + }, + "optionalProperties": { + "qiSub": { + "properties": { + "qSubThread": {"enum": ["noSub", "subPending", "subThread", "prohibitSub"]} + }, + "optionalProperties": { + "qDelivered": {"type": "string", "metadata": {"description": "message ID"}} + } + }, + "qiMsg": { + "properties": { + "msgId": {"type": "string"}, + "msgTs": {"type": "timestamp"}, + "msgType": {"enum": ["message", "quota"]} + } + } + } +} +``` + ### Sender commands Currently SMP defines only one command that can be used by senders - `send` message. This command must be used with sender's ID, if recipient's ID is used the server must respond with `"ERR AUTH"` response (see [Error responses](#error-responses)). +#### Secure queue by sender + +This command is used from v8 of SMP protocol. V8 and earlier uses [KEY](#secure-queue-by-recipient). + +This command is sent by the sender to the server to add sender's key to the queue: + +```abnf +sndSecure = %s"SKEY " senderAuthPublicKey +senderAuthPublicKey = length x509encoded +; the sender's Ed25519 or X25519 key to verify SEND commands for this queue +``` + +Once the queue is secured only authorized messages can be sent to it. + +This command MUST be used in transmission with sender queue ID. + #### Send message This command is sent to the server by the sender both to confirm the queue after the sender received out-of-band message from the recipient and to send messages after the queue is secured: @@ -605,64 +726,79 @@ This command is sent to the server by the sender both to confirm the queue after send = %s"SEND " msgFlags SP smpEncMessage msgFlags = notificationFlag reserved notificationFlag = %s"T" / %s"F" -smpEncMessage = smpPubHeader sentMsgBody ; message up to 16088 bytes -smpPubHeader = smpClientVersion ("1" senderPublicDhKey / "0") -smpClientVersion = word16 +smpEncMessage = smpEncClientMessage / smpEncConfirmation ; message up to 16064 bytes + +smpEncClientMessage = smpPubHeaderNoKey msgNonce sentClientMsgBody ; message up to 16064 bytes +smpPubHeaderNoKey = smpClientVersion "0" +sentClientMsgBody = 16016*16016 OCTET + +smpEncConfirmation = smpPubHeaderWithKey msgNonce sentConfirmationBody +smpPubHeaderWithKey = smpClientVersion "1" senderPublicDhKey + ; sender's Curve25519 public key to agree DH secret for E2E encryption in this queue + ; it is only sent in confirmation message +sentConfirmationBody = 15920*15920 OCTET ; E2E-encrypted smpClientMessage padded to 16016 bytes before encryption senderPublicDhKey = length x509encoded -; sender's Curve25519 public key to agree DH secret for E2E encryption in this queue -; it is only sent in confirmation message + +smpClientVersion = word16 x509encoded = -sentMsgBody = 16032*16032 OCTET -; E2E-encrypted smpClientMessage padded to 16032 bytes before encryption +msgNonce = 24*24 OCTET word16 = 2*2 OCTET ``` -The first message is sent to confirm the queue - it should contain sender's server key (see decrypted message syntax below) - this first message must be sent without signature. +The first message is sent to confirm the queue - it should contain sender's server key (see decrypted message syntax below) - this first message may be sent without authorization. -Once the queue is secured (see [Secure queue command](#secure-queue-command)), the following send commands must be sent with the signature. +Once the queue is secured (see [Secure queue by sender](#secure-queue-by-sender)), the subsequent `SEND` commands must be sent with the authorization. The server must respond with `"ERR AUTH"` response in the following cases: - the queue does not exist or is suspended -- the queue is secured but the transmission does NOT have a signature -- the queue is NOT secured but the transmission has a signature +- the queue is secured but the transmission does NOT have a authorization +- the queue is NOT secured but the transmission has a authorization -Until the queue is secured, the server should accept any number of unsigned messages - it both enables the legitimate sender to resend the confirmation in case of failure and also allows the simplex messaging client to ignore any confirmation messages that may be sent by the attackers (assuming they could have intercepted the queue ID in the server response, but do not have a correct encryption key passed to sender in out-of-band message). +The server must respond with `"ERR QUOTA"` response when queue capacity is exceeded. The number of messages that the server can hold is defined by the server configuration. When sender reaches queue capacity the server will not accept any further messages until the recipient receives ALL messages from the queue. After the last message is delivered, the server will deliver an additional special message indicating that the queue capacity was reached. See [Deliver queue message](#deliver-queue-message) -The body should be encrypted with the recipient's "public" key (`EK`); once decrypted it must have this format: +Until the queue is secured, the server should accept any number of unsigned messages (up to queue capacity) - it allows the sender to resend the confirmation in case of failure. + +The body should be encrypted with the shared secret based on recipient's "public" key (`EK`); once decrypted it must have this format: ```abnf -sentMsgBody = -smpClientMessage = smpPrivHeader clientMsgBody -smpPrivHeader = emptyHeader / smpConfirmationHeader -emptyHeader = " " -smpConfirmationHeader = %s"K" senderKey +sentClientMsgBody = +smpClientMessage = emptyHeader clientMsgBody +emptyHeader = "_" +clientMsgBody = *OCTET ; up to 16016 - 2 + +sentConfirmationBody = +smpConfirmation = smpConfirmationHeader confirmationBody +smpConfirmationHeader = emptyHeader / %s"K" senderKey + ; emptyHeader is used when queue is already secured by sender +confirmationBody = *OCTET ; up to 15920 - 2 senderKey = length x509encoded -; the sender's Ed25519 or Ed448 public key to sign SEND commands for this queue -clientMsgBody = *OCTET ; up to 16016 in case of emptyHeader + ; the sender's Ed25519 or X25519 public key to authorize SEND commands for this queue ``` `clientHeader` in the initial unsigned message is used to transmit sender's server key and can be used in the future revisions of SMP protocol for other purposes. -SMP transmission structure for sent messages: +SMP transmission structure for directly sent messages: ``` -------- transmission (= 16384 bytes) +------- transmissions (= 16384 bytes) + 1 | transmission count (= 1) 2 | originalLength - 276- | signature sessionId corrId queueId %s"SEND" SP (1+114 + 1+32? + 1+32 + 1+24 + 4+1 = 210) - ....... smpEncMessage (= 16088 bytes = 16384 - 296 bytes) + 299- | authorization sessionId corrId queueId %s"SEND" SP (1+114 + 1+32? + 1+24 + 1+24 + 4+1 = 203) + ....... smpEncMessage (= 16064 bytes = 16384 - 320 bytes) 8- | smpPubHeader (for messages it is only version and '0' to mean "no DH key" = 3 bytes) 24 | nonce for smpClientMessage 16 | auth tag for smpClientMessage - ------- smpClientMessage (E2E encrypted, = 16032 bytes = 16088 - 48) + ------- smpClientMessage (E2E encrypted, = 16016 bytes = 16064 - 48) 2 | originalLength - 12- | smpPrivHeader + 2- | smpPrivHeader ....... - | clientMsgBody (<= 16016 bytes = 16032 - 14) + | clientMsgBody (<= 16012 bytes = 16016 - 4) ....... 0+ | smpClientMessage pad ------- smpClientMessage end | + 0+ | message pad ....... smpEncMessage end 18+ | transmission pad ------- transmission end @@ -671,20 +807,23 @@ SMP transmission structure for sent messages: SMP transmission structure for received messages: ``` -------- transmission (= 16384 bytes) +------- transmissions (= 16384 bytes) + 1 | transmission count (= 1) 2 | originalLength - 276- | signature sessionId corrId queueId %s"MSG" SP msgId timestamp (1+114 + 1+32? + 1+32 + 1+24 + 3+1 + 24+1 + 8 = 243) + 283- | authorization sessionId corrId queueId %s"MSG" SP msgId (1+114 + 1+32? + 1+24 + 1+24 + 3+1 + 1+24 = 227) 16 | auth tag (msgId is used as nonce) - ------- serverEncryptedMsg (= 16090 bytes = 16384 - 294 bytes) + ------- serverEncryptedMsg (= 16082 bytes = 16384 - 302 bytes) 2 | originalLength - ....... smpEncMessage (= 16088 bytes = 16090 - 2 bytes) - 16- | smpPubHeader (empty header for the message) + 8 | timestamp + 8- | message flags + ....... smpEncMessage (= 16064 bytes = 16082 - 18 bytes) + 8- | smpPubHeader (empty header for the message) 24 | nonce for smpClientMessage 16 | auth tag for smpClientMessage - ------- smpClientMessage (E2E encrypted, = 16032 bytes = 16088 - 56 bytes) + ------- smpClientMessage (E2E encrypted, = 16016 bytes = 16064 - 48 bytes) 2 | originalLength - 16- | smpPrivHeader (empty header for the message) - ....... clientMsgBody (<= 16016 bytes = 16032 - 16) + 2- | smpPrivHeader (empty header for the message) + ....... clientMsgBody (<= 16012 bytes = 16016 - 4) -- TODO move internal structure (below) to agent protocol 20- | agentPublicHeader (the size is for user messages post handshake, without E2E X3DH keys - it is version and 'M' for the messages - 3 bytes in total) ....... E2E double-ratchet encrypted (<= 15996 bytes = 16016 - 20) @@ -719,6 +858,149 @@ SMP transmission structure for received messages: ------- transmission end ``` +### Proxying sender commands + +To protect transport (IP address and session) anonymity of the sender from the server chosen (and, potentially, controlled) by the recipient SMP v8 added support for proxying sender's command to the recipient's server via the server chosen by the sender. + +Sequence diagram for sending the message and `SKEY` commands via SMP proxy: + +``` +------------- ------------- ------------- ------------- +| sending | | SMP | | SMP | | receiving | +| client | | proxy | | server | | client | +------------- ------------- ------------- ------------- + | `PRXY` | | | + | -------------------------> | | | + | | ------------------------------> | | + | | SMP handshake | | + | | <------------------------------ | | + | `PKEY` | | | + | <------------------------- | | | + | | | | + | `PFWD` (s2r) | | | + | -------------------------> | | | + | | `RFWD` (p2r) | | + | | ------------------------------> | | + | | `RRES` (p2r) | | + | | <------------------------------ | | + | `PRES` (s2r) | | `MSG` | + | <------------------------- | | -----------------------> | + | | | `ACK` | + | | | <----------------------- | + | | | | + | | | | +``` + +1. The client requests (`PRXY` command) the chosen server to connect to the destination SMP server and receives (`PKEY` response) the session information, including server certificate and the session key signed by this certificate. To protect client session anonymity the proxy MUST re-use the same session with all clients that request connection with any given destination server. + +2. The client encrypts the transmission (`SKEY` or `SEND`) to the destination server using the shared secret computed from per-command random key and server's session key and sends it to proxying server in `PFWD` command. + +3. Proxy additionally encrypts the body to prevent correlation by ciphertext (in case TLS is compromised) and forwards it to proxy in `RFWD` command. + +4. Proxy receives the double-encrypted response from the destination server, removes one encryption layer and forwards it to the client. + +The diagram below shows the encryption layers for `PFWD`/`RFWD` commands and `RRES`/`PRES` responses: + +- s2r - encryption between client and SMP relay, with relay key returned in relay handshake, with MITM by proxy mitigated by verifying the certificate fingerprint included in the relay address. This encryption prevents proxy server from observing commands and responses - proxy does not know how many different queues a connected client sends messages and commands to. +- e2e - end-to-end encryption per SMP queue, with additional client encryption inside it. +- p2r - additional encryption between proxy and SMP relay with the shared secret agreed in the handshake, to mitigate traffic correlation inside TLS. +- r2c - additional encryption between SMP relay and client to prevent traffic correlation inside TLS. + +``` +----------------- ----------------- -- TLS -- ----------------- ----------------- +| | -- TLS -- | | -- p2r -- | | -- TLS -- | | +| | -- s2r -- | | -- s2r -- | | -- r2c -- | | +| sending | -- e2e -- | | -- e2e -- | | -- e2e -- | receiving | +| client | MSG | SMP proxy | MSG | SMP server | MSG | client | +| | -- e2e -- | | -- e2e -- | | -- e2e -- | | +| | -- s2r -- | | -- s2r -- | | -- r2c -- | | +| | -- TLS -- | | -- p2r -- | | -- TLS -- | | +----------------- ----------------- -- TLS -- ----------------- ----------------- +``` + +SMP proxy is not another type of the server, it is a role that any SMP server can play when forwarding the commands. + +#### Request proxied session + +The sender uses this command to request the session with the destination proxy. + +Servers SHOULD support basic auth with this command, to allow only server owners and trusted users to proxy commands to the destination servers. + +```abnf +proxySession = %s"PRXY" SP smpServer basicAuth +smpServer = hosts port fingerprint +hosts = length 1*host +host = shortString +port = shortString +fingerprint = shortString +basicAuth = "0" / "1" shortString ; server password +``` + +```abnf +proxySessionKey = %s"PKEY" SP sessionId smpVersionRange certChain signedKey +sessionId = shortString + ; Session ID (tlsunique) of the proxy with the destination server. + ; This session ID should be used as entity ID in transmission with `PFWD` command +certChain = length 1*cert +cert = originalLength x509encoded +signedKey = originalLength x509encoded ; key signed with certificate +originalLength = 2*2 OCTET +``` + +When the client receives PKEY response it MUST validate that: +- the fingerprint of the received certificate matches fingerprint in the server address - it mitigates MITM attack by proxy. +- the server session key is correctly signed with the received certificate. + +The proxy server may respond with error response in case the destination server is not available or in case it has an earlier version that does not support proxied commands. + +#### Send command via proxy + +Sender can send `SKEY` and `SEND` commands via proxy after obtaining the session ID with `PRXY` command (see [Request proxied session](#request-proxied-session)). + +Transmission sent to proxy server should use session ID as entity ID and use a random correlation ID of 24 bytes as a nonce for crypto_box encryption of transmission to the destination server. The random ephemeral X25519 key to encrypt transmission should be unique per command, and it should be combined with the key sent by the server in the handshake header to proxy and to the client in `PKEY` command. + +Encrypted transmission should use the received session ID from the connection between proxy server and destination server in the authorized body. + +```abnf +proxyCommand = %s"PFWD" SP smpVersion commandKey +smpVersion = 2*2 OCTET +commandKey = length x509encoded +``` + +The proxy server will forward the encrypted transmission in `RFWD` command (see below). + +Having received the `RRES` response from the destination server, proxy server will forward `PRES` response to the client. `PRES` response should use the same correlation ID as `PFWD` command. The destination server will use this correlation ID increased by 1 as a nonce for encryption of the response. + +```abnf +proxyResponse = %s"PRES" SP +``` + +#### Forward command to destination server + +Having received `PFWD` command from the client, the server should additionally encrypt it (without padding, as the received transmission is already encrypted by the client and padded to a fixed size) together with the correlation ID, sender command key, and protocol version, and forward it to the destination server as `RFWD` command: + +Transmission forwarded to relay uses empty entity ID and its unique random correlation ID is used as a nonce to encrypt forwarded transmission. Correlation ID increased by 1 is used by the destination server as a nonce to encrypt responses. + +```abnf +relayCommand = %s"RFWD" SP +forwardedTransmission = fwdCorrId fwdSmpVersion fwdCommandKey transmission +fwdCorrId = length 24*24 OCTET + ; `fwdCorrId` - correlation ID used in `PFWD` command transmission - it is used as a nonce for client encryption, + ; and `fwdCorrId + 1` is used as a nonce for the destination server response encryption. +fwdSmpVersion = 2*2 OCTET +fwdCommandKey = length x509encoded +transmission = *OCTET ; note that it is not prefixed with the length +``` + +The destination server having received this command decrypts both encryption layers (proxy and client), verifies client authorization as usual, processes it, and send the double encrypted `RRES` response to proxy. + +The shared secret for encrypting transmission bodies between proxy server and destination server is agreed from proxy and destination server keys exchanged in handshake headers - proxy and server use the same shared secret during the session for the encryption between them. + + +```abnf +relayResponse = %s"RRES" SP +``` + ### Notifier commands #### Subscribe to queue notifications @@ -735,45 +1017,40 @@ The first message notification will be delivered either immediately or as soon a ### Server messages -#### Queue IDs response +This section includes server events and generic command responses used for several commands. -Server must respond with this message when the new queue is created. - -See its syntax in [Create queue command](#create-queue-command) +The syntax for command-specific responses is shown together with the commands. #### Deliver queue message -The server must deliver messages to all subscribed simplex queues on the currently open transport connection. The syntax for the message delivery is: - -```abnf -message = %s"MSG " msgId encryptedRcvMsgBody -encryptedMsgBody = ; server-encrypted padded sent msgBody -paddedSentMsgBody = ; maxMessageLength = 16088 -encryptedRcvMsgBody = ; server-encrypted meta-data and padded sent msgBody -rcvMsgBody = timestamp msgFlags SP paddedSentMsgBody -msgId = length 24*24OCTET -timestamp = 8*8OCTET -``` - -`msgId` - unique message ID generated by the server based on cryptographically strong random bytes. It should be used by the clients to detect messages that were delivered more than once (in case the transport connection was interrupted and the server did not receive the message delivery acknowledgement). Message ID is used as a nonce for server/recipient encryption of message bodies. - -`timestamp` - system time when the server received the message from the sender as **a number of seconds** since Unix epoch (1970-01-01) encoded as 64-bit integer in network byte order. If a client system/language does not support 64-bit integers, until 2106 it is safe to simply skip the first 4 zero bytes and decode 32-bit unsigned integer (or as signed integer until 2038). - -`paddedSentMsgBody` - see syntax in [Send message](#send-message) - When server delivers the messages to the recipient, message body should be encrypted with the secret derived from DH exchange using the keys passed during the queue creation and returned with `queueIds` response. This is done to prevent the possibility of correlation of incoming and outgoing traffic of SMP server inside transport protocol. -#### Notifier queue ID response +The server must deliver messages to all subscribed simplex queues on the currently open transport connection. The syntax for the message delivery is: -Server must respond with this message when queue notifications are enabled. +```abnf +message = %s"MSG" SP msgId encryptedRcvMsgBody +encryptedRcvMsgBody = + ; server-encrypted padded sent msgBody + ; maxMessageLength = 16064 +rcvMsgBody = timestamp msgFlags SP sentMsgBody / msgQuotaExceeded +msgQuotaExceeded = %s"QUOTA" SP timestamp +msgId = length 24*24OCTET +timestamp = 8*8OCTET +``` -See its syntax in [Enable notifications command](#enable-notifications-command) +If the sender exceeded queue capacity the recipient will receive a special message indicating the quota was exceeded. This can be used in the higher level protocol to notify sender client that it can continue sending messages. + +`msgId` - unique message ID generated by the server based on cryptographically strong random bytes. It should be used by the clients to detect messages that were delivered more than once (in case the transport connection was interrupted and the server did not receive the message delivery acknowledgement). Message ID is used as a nonce for server/recipient encryption of message bodies. + +`timestamp` - system time when the server received the message from the sender as **a number of seconds** since Unix epoch (1970-01-01) encoded as 64-bit integer in network byte order. If a client system/language does not support 64-bit integers, until 2106 it is safe to simply skip the first 4 zero bytes and decode 32-bit unsigned integer (or as signed integer until 2038). + +`sentMsgBody` - message sent by `SEND` command. See [Send message](#send-message). #### Deliver message notification -The server must deliver message notifications to all simplex queues that were subscribed with `subscribeNotifications` command ("NSUB") on the currently open transport connection. The syntax for the message notification delivery is: +The server must deliver message notifications to all simplex queues that were subscribed with `subscribeNotifications` command (`NSUB`) on the currently open transport connection. The syntax for the message notification delivery is: ```abnf messageNotification = %s"NMSG " nmsgNonce encryptedNMsgMeta @@ -802,28 +1079,60 @@ No further messages should be delivered to unsubscribed transport connection. #### Error responses -- incorrect block format, encoding or signature size (`BLOCK`). +- incorrect block format, encoding or authorization size (`BLOCK`). - missing or different session ID - tls-unique binding of TLS transport (`SESSION`) - command errors (`CMD`): - - error parsing command (`SYNTAX`) - - prohibited command (`PROHIBITED`) - any server response sent from client or `ACK` sent without active subscription or without message delivery. - - transmission has no required signature or queue ID (`NO_AUTH`) + - unknown command (`UNKNOWN`). + - error parsing command (`SYNTAX`). + - prohibited command (`PROHIBITED`): + - `ACK` sent without active subscription or without message delivery. + - `GET` and `SUB` used in the same transport connection with the same queue. + - transmission has no required authorization or queue ID (`NO_AUTH`) - transmission has unexpected credentials (`HAS_AUTH`) - - transmission has no required queue ID (`NO_QUEUE`) -- authentication error (`AUTH`) - incorrect signature, unknown (or suspended) queue, sender's ID is used in place of recipient's and vice versa, and some other cases (see [Send message](#send-message) command). + - transmission has no required queue ID (`NO_ENTITY`) +- proxy server errors (`PROXY`): + - `PROTOCOL` - any error. + - `BASIC_AUTH` - incorrect basic auth. + - `NO_SESSION` - no destination server session with passed ID. + - `BROKER` - destination server error: + - `RESPONSE` - invalid server response (failed to parse). + - `UNEXPECTED` - unexpected response. + - `NETWORK` - network error. + - `TIMEOUT` - command response timeout. + - `HOST` - no compatible server host (e.g. onion when public is required, or vice versa) + - `TRANSPORT` - handshake or other transport error: + - `BLOCK` - error parsing transport block. + - `VERSION` - incompatible client or server version. + - `LARGE_MSG` - message too large. + - `SESSION` - incorrect session ID. + - `NO_AUTH` - absent server key - when the server did not provide a DH key to authorize commands for the queue that should be authorized with a DH key. + - `HANDSHAKE` - transport handshake error: + - `PARSE` - handshake syntax (parsing) error. + - `IDENTITY` - incorrect server identity (certificate fingerprint does not match server address). + - `BAD_AUTH` - incorrect or missing server credentials in handshake. +- authentication error (`AUTH`) - incorrect authorization, unknown (or suspended) queue, sender's ID is used in place of recipient's and vice versa, and some other cases (see [Send message](#send-message) command). - message queue quota exceeded error (`QUOTA`) - too many messages were sent to the message queue. Further messages can only be sent after the recipient retrieves the messages. -- sent message is too large (> 16088) to be delivered (`LARGE_MSG`). +- sent message is too large (> 16064) to be delivered (`LARGE_MSG`). - internal server error (`INTERNAL`). The syntax for error responses: ```abnf error = %s"ERR " errorType -errorType = %s"BLOCK" / %s"SESSION" / %s"CMD " cmdError / %s"AUTH" / %s"LARGE_MSG" /%s"INTERNAL" +errorType = %s"BLOCK" / %s"SESSION" / %s"CMD" SP cmdError / %s"PROXY" proxyError / + %s"AUTH" / %s"QUOTA" / %s"LARGE_MSG" / %s"INTERNAL" cmdError = %s"SYNTAX" / %s"PROHIBITED" / %s"NO_AUTH" / %s"HAS_AUTH" / %s"NO_ENTITY" +proxyError = %s"PROTOCOL" SP errorType / %s"BROKER" SP brokerError / + %s"BASIC_AUTH" / %s"NO_SESSION" +brokerError = %s"RESPONSE" SP shortString / %s"UNEXPECTED" SP shortString / + %s"NETWORK" / %s"TIMEOUT" / %s"HOST" / + %s"TRANSPORT" SP transportError +transportError = %s"BLOCK" / %s"VERSION" / %s"LARGE_MSG" / %s"SESSION" / %s"NO_AUTH" / + %s"HANDSHAKE" SP handshakeError +handshakeError = %s"PARSE" / %s"IDENTITY" / %s"BAD_AUTH" ``` -Server implementations must aim to respond within the same time for each command in all cases when `"ERR AUTH"` response is required to prevent timing attacks (e.g., the server should perform signature verification even when the queue does not exist on the server or the signature of different size is sent, using any RSA key with the same size as the signature size). +Server implementations must aim to respond within the same time for each command in all cases when `"ERR AUTH"` response is required to prevent timing attacks (e.g., the server should verify authorization even when the queue does not exist on the server or the authorization of different type is sent, using any dummy key compatible with the used authorization). ### OK response @@ -833,22 +1142,12 @@ When the command is successfully executed by the server, it should respond with ok = %s"OK" ``` -## Appendices +## Transport connection with the SMP server -### Appendix A. - -**SMP transport protocol.** +### General transport protocol considerations Both the recipient and the sender can use TCP or some other, possibly higher level, transport protocol to communicate with the server. The default TCP port for SMP server is 5223. -For scenarios when meta-data privacy is critical, it is recommended that clients: - -- communicating over Tor network, -- establish a separate connection for each SMP queue, -- send noise traffic (using PING command). - -In addition to that, the servers can be deployed as Tor onion services. - The transport protocol should provide the following: - server authentication (by matching server certificate hash with `serverIdentity`), @@ -856,40 +1155,110 @@ The transport protocol should provide the following: - integrity (preventing data modification by the attacker without detection), - unique channel binding (`sessionIdentifier`) to include in the signed part of SMP transmissions. -By default, the client and server communicate using [TLS 1.3 protocol][13] restricted to: +### TLS transport encryption + +The client and server communicate using [TLS 1.3 protocol][13] restricted to: - TLS_CHACHA20_POLY1305_SHA256 cipher suite (for better performance on mobile devices), -- ed25519 and ed448 EdDSA algorithms for signatures, -- x25519 and x448 ECDHE groups for key exchange. -- servers must send the chain of exactly 2 self-signed certificates in the handshake, with the first (offline) certificate one signing the second (online) certificate. Offline certificate fingerprint is used as a server identity - it is a part of SMP server address. +- ed25519 EdDSA algorithms for signatures, +- x25519 ECDHE groups for key exchange. +- servers must send the chain of 2, 3 or 4 self-signed certificates in the handshake (see [Server certificate](#server-certificate)), with the first (offline) certificate one signing the second (online) certificate. Offline certificate fingerprint is used as a server identity - it is a part of SMP server address. - The clients must abort the connection in case a different number of certificates is sent. - server and client TLS configuration should not allow resuming the sessions. During TLS handshake the client must validate that the fingerprint of the online server certificate is equal to the `serverIdentity` the client received as part of SMP server address; if the server identity does not match the client must abort the connection. +### Server certificate + +Servers use self-signed certificates that the clients validate by comparing the fingerprint of one of the certificates in the chain with the certificate fingerprint present in the server address. + +Clients SHOULD support the chains of 2, 3 and 4 server certificates: + +**2 certificates**: +1. offline server certificate: + - its fingerprint is present in the server address. + - its private key is not stored on the server. +2. online server certificate: + - it must be signed by offline certificate. + - its private key is stored on the server and is used in TLS session. + +**3 certificates**: +1. offline server certificate - same as with 2 certificates. +2. online server certificate: + - it must be signed by offline certificate. + - its private key is stored on the server. +3. session certificate: + - generated automatically on every server start and/or on schedule. + - signed by online server certificate. + - its private key is used in TLS session. + +**4 certificates**: +0. offline operator identity certificate: + - used for all servers operated by the same entity. + - its private key is not stored on the server. +1. offline server certificate: + - signed by offline operator certificate. + - same as with 2 certificates. +2. online server certificate - same as with 3 certificates. +3. session certificate - same as with 3 certificates. + +### ALPN to agree handshake version + +Client and server use [ALPN extension][18] of TLS to agree handshake version. + +Server SHOULD send `smp/1` protocol name and the client should confirm this name in order to use the current protocol version. This is added to allow support of older clients without breaking backward compatibility and to extend or modify handshake syntax. + +If the client does not confirm this protocol name, the server would fall back to v6 of SMP protocol. + +### Transport handshake + Once TLS handshake is complete, client and server will exchange blocks of fixed size (16384 bytes). -The first block sent by the server should be `serverHello` and the client should respond with `clientHello` - these blocks are used to agree SMP protocol version: +The first block sent by the server should be `paddedServerHello` and the client should respond with `paddedClientHello` - these blocks are used to agree SMP protocol version: ```abnf -serverHello = minSmpVersion maxSmpVersion sessionIdentifier pad +paddedServerHello = +serverHello = smpVersionRange sessionIdentifier [serverCert signedServerKey] ignoredPart +smpVersionRange = minSmpVersion maxSmpVersion minSmpVersion = smpVersion maxSmpVersion = smpVersion -sessionIdentifier = length *OCTET +sessionIdentifier = shortString ; unique session identifier derived from transport connection handshake -; it should be included in all SMP transmissions sent in this transport connection. +; it should be included in authorized part of all SMP transmissions sent in this transport connection, +; but it must not be sent as part of the transmission in the current protocol version. +serverCert = originalLength x509encoded +signedServerKey = originalLength x509encoded ; signed by server certificate -clientHello = smpVersion pad +paddedClientHello = +clientHello = smpVersion [clientKey] ignoredPart ; chosen SMP protocol version - it must be the maximum supported version ; within the range offered by the server +clientKey = length x509encoded smpVersion = 2*2OCTET ; Word16 version number - +originalLength = 2*2OCTET +ignoredPart = *OCTET pad = *OCTET ``` +`signedServerKey` is used to compute a shared secret to authorize client transmission - it is combined with the per-queue key that was used when the queue was created. + +`clientKey` is used only by SMP proxy server when it connects to the destination server to agree shared secret for the additional encryption layer, end user clients do not use this key. + +`ignoredPart` in handshake allows to add additional parameters in handshake without changing protocol version - the client and servers must ignore any extra bytes within the original block length. + For TLS transport client should assert that `sessionIdentifier` is equal to `tls-unique` channel binding defined in [RFC 5929][14] (TLS Finished message struct); we pass it in `serverHello` block to allow communication over some other transport protocol (possibly, with another channel binding). +### Additional transport privacy + +For scenarios when meta-data privacy is critical, it is recommended that clients: + +- communicating over Tor network, +- establish a separate connection for each SMP queue, +- send noise traffic (using PING command). + +In addition to that, the servers can be deployed as Tor onion services. + [1]: https://en.wikipedia.org/wiki/Man-in-the-middle_attack [2]: https://en.wikipedia.org/wiki/End-to-end_encryption [3]: https://en.wikipedia.org/wiki/QR_code @@ -906,3 +1275,5 @@ For TLS transport client should assert that `sessionIdentifier` is equal to `tls [14]: https://datatracker.ietf.org/doc/html/rfc5929#section-3 [15]: https://www.rfc-editor.org/rfc/rfc8709.html [16]: https://nacl.cr.yp.to/box.html +[17]: https://datatracker.ietf.org/doc/html/rfc8927 +[18]: https://datatracker.ietf.org/doc/html/rfc7301 diff --git a/protocol/xftp.md b/protocol/xftp.md new file mode 100644 index 000000000..d6d271837 --- /dev/null +++ b/protocol/xftp.md @@ -0,0 +1,632 @@ +Version 2, 2024-06-22 + +# SimpleX File Transfer Protocol + +## Table of contents + +- [Abstract](#abstract) +- [Introduction](#introduction) +- [XFTP Model](#xftp-model) +- [Persistence model](#persistence-model) +- [XFTP procedure](#xftp-procedure) +- [File description](#file-description) +- [URIs syntax](#uris-syntax) + - [XFTP server URI](#xftp-server-uri) + - [File description URI](#file-description-URI) +- [XFTP qualities and features](#xftp-qualities-and-features) +- [Cryptographic algorithms](#cryptographic-algorithms) +- [File chunk IDs](#file-chunk-ids) +- [Server security requirements](#server-security-requirements) +- [Transport protocol](#transport-protocol) + - [TLS ALPN](#tls-alpn) + - [Connection handshake](#connection-handshake) + - [Requests and responses](#requests-and-responses) +- [XFTP commands](#xftp-commands) + - [Correlating responses with commands](#correlating-responses-with-commands) + - [Command authentication](#command-authentication) + - [Keep-alive command](#keep-alive-command) + - [File sender commands](#file-sender-commands) + - [Register new file chunk](#register-new-file-chunk) + - [Add file chunk recipients](#add-file-chunk-recipients) + - [Upload file chunk](#upload-file-chunk) + - [Delete file chunk](#delete-file-chunk) + - [File recipient commands](#file-recipient-commands) + - [Download file chunk](#download-file-chunk) + - [Acknowledge file chunk download](#acknowledge-file-chunk-download) +- [Threat model](#threat-model) + +## Abstract + +SimpleX File Transfer Protocol is a client-server protocol for asynchronous unidirectional file transmission. + +It's designed with the focus on communication security, integrity and meta-data privacy, under the assumption that any part of the message transmission network can be compromised. + +It is designed as a application level protocol to solve the problem of secure and private file transmission, making [MITM attacks][1] very difficult at any part of the file transmission system, and preserving meta-data privacy of the sent files. + +## Introduction + +The objective of SimpleX File Transfer Protocol (XFTP) is to facilitate the secure and private unidirectional transfer of files from senders to recipients via persistent file chunks stored by the xftp server. + +XFTP is implemented as an application level protocol on top of HTTP2 and TLS. + +The protocol describes the set of commands that senders and recipients can send to XFTP servers to create, upload, download and delete file chunks of several pre-defined sizes. XFTP servers SHOULD support chunks of 4 sizes: 64KB, 256KB, 1MB and 4MB (1KB = 1024 bytes, 1MB = 1024KB). + +The protocol is designed with the focus on meta-data privacy and security. While using TLS, the protocol does not rely on TLS security by using additional encryption to achieve that there are no identifiers or ciphertext in common in received and sent server traffic, frustrating traffic correlation even if TLS is compromised. + +XFTP does not use any form of participants' identities. It relies on out-of-band passing of "file description" - a human-readable YAML document with the list of file chunk locations, hashes and necessary cryptographic keys. + +## XFTP Model + +The XFTP model has three communication participants: the recipient, the file server (XFTP server) that is chosen and, possibly, controlled by the sender, and the sender. + +XFTP server allows uploading fixed size file chunks, with or without basic authentication. The same party that can be the sender of one file chunk can be the recipient of another, without exposing it to the server. + +Each file chunk allows multiple recipients, each recipient can download the same chunk multiple times. It allows depending on the threat model use the same recipient credentials for multiple parties, thus reducing server ability to understand the number of intended recipients (but server can still track IP addresses to determine it), or use one unique set of credentials for each recipient, frustrating traffic correlation on the assumption of compromised TLS. In the latter case, senders can create a larger number of recipient credentials to hide the actual number of intended recipients from the servers (which is what SimpleX clients do). + +``` + Sender Internet XFTP relays Internet Recipient +---------------------------- | ----------------- | ------------------- | ------------ | ---------- + | | | | + | | (can be self-hosted) | | + | | +---------+ | | + chunk 1 ----- HTTP2 over TLS ------ | XFTP | ---- HTTP2 / TLS ----- chunk 1 + |---> SimpleX File Transfer Protocol (XFTP) --> | Relay | ---> XFTP ------------->| + | --------------------------- +---------+ ---------------------- | + | | | | | | + | | | | | v + +----------+ | | +---------+ | | +-------------+ + | Sending | ch. 2 ------- HTTP2 / TLS ------- | XFTP | ---- HTTP2 / TLS ---- ch. 2 | Receiving | +file ---> | XFTP | ------> XFTP ----> | Relay | ---> XFTP ------> | XFTP | ---> file + | Client | --------------------------- +---------+ ---------------------- | Client | + +----------+ | | | | +-------------+ + | | | | | ^ + | | | +---------+ | | | + | ------- HTTP2 / TLS ------- | XFTP | ---- HTTP2 / TLS ---- | + |-------------> XFTP ----> | Relay | ---> XFTP ------------->| + chunk N --------------------------- +---------+ --------------------- chunk N + | | (store file chunks) | | + | | | | + | | | | +``` + +When sender client uploads a file chunk, it has to register it first with one sender ID and multiple recipient IDs, and one random unique key per ID to authenticate sender and recipients, and also provide its size and hash that will be validated when chunk is uploaded. + +To send the actual file, the sender client MUST pad it and encrypt it with a random symmetric key and distribute chunks of fixed sized across multiple XFTP servers. Information about chunk locations, keys, hashes and required keys is passed to the recipients as "[file description](#file-description)" out-of-band. + +Creating, uploading, downloading and deleting file chunks requires sending commands to the XFTP server - they are described in detail in [XFTP commands](#xftp-commands) section. + +## Persistence model + +Server stores file chunk records in memory, with optional adding to append-only log, to allow restoring them on server restart. File chunk bodies can be stored as files or as objects in any object store (e.g. S3). + +## XFTP procedure + +1. Sending the file. + +To send the file, the sender will: + +1) Prepare file + - compute its SHA512 digest. + - prepend header with the name and pad the file to match the whole number of chunks in size. It is RECOMMENDED to use 2 of 4 allowed chunk sizes, to balance upload size and metadata privacy. + - encrypt it with a randomly chosen symmetric key and IV (e.g., using NaCL secret_box). + - split into allowed size chunks. + - generate per-recipient keys. It is recommended that the sending client generates more per-recipient keys than the actual number of recipients, rounding up to a power of 2, to conceal the actual number of intended recipients. + +2) Upload file chunks + - register each chunk record with randomly chosen one or more (for redundancy) XFTP server(s). + - optionally request additional recipient IDs, if required number of recipient keys didn't fit into register request. + - upload each chunk to chosen server(s). + +3) Prepare file descriptions, one per recipient. + +The sending client combines addresses of all chunks and other information into "file description", different for each file recipient, that will include: + +- an encryption key used to encrypt/decrypt the full file (the same for all recipients). +- file SHA512 digest to validate download. +- list of chunk descriptions; information for each chunk: + - private Ed25519 key to sign commands for file transfer server. + - chunk address (server host and chunk ID). + - chunk sha512 digest. + +To reduce the size of file description, chunks are grouped by the server host. + +4) Send file description(s) to the recipient(s) out-of-band, via pre-existing secure and authenticated channel. E.g., SimpleX clients send it as messages via SMP protocol, but it can be done via any other channel. + +![Sending file](./diagrams/xftp/xftp-sending-file.svg) + +2. Receiving the file. + +Having received the description, the recipient will: + +1) Download all chunks. + +The receiving client can fall back to secondary servers, if necessary: +- if the server is not available. +- if the chunk is not present on the server (ERR AUTH response). +- if the hash of the downloaded file chunk does not match the description. + +Optionally recipient can acknowledge file chunk reception to delete file ID from server for this recipient. + +2) Combine the chunks into a file. + +3) Decrypt the file using the key in file description. + +4) Extract file name and unpad the file. + +5) Validate file digest with the file description. + +![Receiving file](./diagrams/xftp/xftp-receiving-file.svg) + +## File description + +"File description" is a human-readable YAML document that is sent via secure and authenticated channel. + +It includes these fields: +- `party` - "sender" or "recipient". Sender's file description is required to delete the file. +- `size` - padded file size equal to total size of all chunks, see `fileSize` syntax below. +- `digest` - SHA512 hash of encrypted file, base64url encoded string. +- `key` - symmetric encryption key to decrypt the file, base64url encoded string. +- `nonce` - nonce to decrypt the file, base64url encoded string. +- `chunkSize` - default chunk size, see `fileSize` syntax below. +- `replicas` - the array of file chunk replicas descriptions. +- `redirect` - optional property for redirect information indicating that the file is itself a description to another file, allowing to use file description as a short URI. + +Each replica description is an object with 2 fields: + +- `chunks` - and array of chunk replica descriptions stored on one server. +- `server` - [server address](#xftp-server-uri) where the chunks can be downloaded from. + +Each server replica description is a string with this syntax: + +```abnf +chunkReplica = chunkNo ":" replicaId ":" replicaKey [":" chunkDigest [":" chunkSize]] +chunkNo = 1*DIGIT + ; a sequential 1-based chunk number in the original file. +replicaId = base64url + ; server-assigned random chunk replica ID. +replicaKey = base64url + ; sender-generated random key to receive (or to delete, in case of sender's file description) the chunk replica. +chunkDigest = base64url + ; chunk digest that MUST be specified for the first replica of each chunk, + ; and SHOULD be omitted (or be the same) on the subsequent replicas +chunkSize = fileSize +fileSize = sizeInBytes / sizeInUnits + ; chunk size SHOULD only be specified on the first replica and only if it is different from default chunk size +sizeInBytes = 1*DIGIT +sizeInUnits = 1*DIGIT sizeUnit +sizeUnit = %s"kb" / %s"mb" / %s"gb" +base64url = ; RFC4648, section 5 +``` + +Optional redirect information has two fields: +- `size` - the size of the original encrypted file to which file description downloaded via the current file description will lead to, see `fileSize` syntax below. +- `digest` - SHA512 hash of the original file, base64url encoded string. + +## URIs syntax + +### XFTP server URI + +The XFTP server address is a URI with the following syntax: + +```abnf +xftpServerURI = %s"xftp://" xftpServer +xftpServer = serverIdentity [":" basicAuth] "@" srvHost [":" port] +srvHost = ; RFC1123, RFC5891 +port = 1*DIGIT +serverIdentity = base64url +basicAuth = base64url +``` + +### File description URI + +This file description URI can be generated by the client application to share a small file description as a QR code or as a link. Practically, to be able to scan a QR code it should be under 1000 characters, so only file descriptions with 1-2 chunks can be used in this case. This is supported with `redirect` property when file description leads to a file which in itself is a larger file description to another file - akin to URL shortener. + +File description URI syntax: + +```abnf +fileDescriptionURI = serviceScheme "/file" "#/?desc=" description [ "&data=" userData ] +serviceScheme = (%s"https://" clientAppServer) | %s"simplex:" +clientAppServer = hostname [ ":" port ] +; client app server, e.g. simplex.chat +description = +userData = +``` + +clientAppServer is not a server the client connects to - it is a server that shows the instruction on how to download the client app that will connect using this connection request. This server can also host a mobile or desktop app manifest so that this link is opened directly in the app if it is installed on the device. + +"simplex" URI scheme in serviceScheme can be used instead of client app server. Client apps MUST support this URI scheme. + +## XFTP qualities and features + +XFTP stands for SimpleX File Transfer Protocol. Its design is based on the same ideas and has some of the qualities of SimpleX Messaging Protocol: + +- recipient cannot see sender's IP address, as the file fragments (chunks) are temporarily stored on multiple XFTP relays. +- file can be sent asynchronously, without requiring the sender to be online for file to be received. +- there is no network of peers that can observe this transfer - sender chooses which XFTP relays to use, and can self-host their own. +- XFTP relays do not have any file metadata - they only see individual chunks, with access to each chunk authorized with anonymous credentials (using Edwards curve cryptographic signature) that are random per chunk. +- chunks have one of the sizes allowed by the servers - 64KB, 256KB, 1MB and 4MB chunks, so sending a large file looks indistinguishable from sending many small files to XFTP server. If the same transport connection is reused, server would only know that chunks are sent by the same user. +- each chunk can be downloaded by multiple recipients, but each recipient uses their own key and chunk ID to authorize access, and the chunk is encrypted by a different key agreed via ephemeral DH keys (NaCl crypto_box (SalsaX20Poly1305 authenticated encryption scheme ) with shared secret derived from Curve25519 key exchange) on the way from the server to each recipient. XFTP protocol as a result has the same quality as SMP protocol - there are no identifiers and ciphertext in common between sent and received traffic inside TLS connection, so even if TLS is compromised, it complicates traffic correlation attacks. +- XFTP protocol supports redundancy - each file chunk can be sent via multiple relays, and the recipient can choose the one that is available. Current implementation of XFTP protocol in SimpleX Chat does not support redundancy though. +- the file as a whole is encrypted with a random symmetric key using NaCl secret_box. + +## Cryptographic algorithms + +Clients must cryptographically authorize XFTP commands, see [Command authentication](#command-authentication). + +To authorize/verify transmissions clients and servers MUST use either signature algorithm Ed25519 algorithm defined in RFC8709 or using deniable authentication scheme based on NaCL crypto_box (see Simplex Messaging Protocol). + +To encrypt/decrypt file chunk bodies delivered to the recipients, servers/clients MUST use NaCL crypto_box. + +Clients MUST encrypt file chunk bodies sent via XFTP servers using use NaCL crypto_box. + +## File chunk IDs + +XFTP servers MUST generate a separate new set of IDs for each new chunk - for the sender (that uploads the chunk) and for each intended recipient. It is REQUIRED that: + +- These IDs are different and unique within the server. +- Based on random bytes generated with cryptographically strong pseudo-random number generator. + +## Server security requirements + +XFTP server implementations MUST NOT create, store or send to any other servers: + +- Logs of the client commands and transport connections in the production environment. + +- History of retrieved files. + +- Snapshots of the database they use to store file chunks (instead clients can manage redundancy by creating chunk replicas using more than one XFTP server). In-memory persistence is recommended for file chunks records. + +- Any other information that may compromise privacy or [forward secrecy][4] of communication between clients using XFTP servers. + +## Transport protocol + +- binary-encoded commands sent as fixed-size padded block in the body of HTTP2 POST request, similar to SMP and notifications server protocol transmission encodings. +- HTTP2 POST with a fixed size padded block body for file upload and download. + +Block size - 4096 bytes (it would fit ~120 Ed25519 recipient keys). + +The reasons to use HTTP2: + +- avoid the need to have two hostnames (or two different ports) for commands and file uploads. +- compatibility with the existing HTTP2 client libraries. + +The reason not to use JSON bodies: + +- bigger request size, so fewer recipient keys would fit in a single request +- signature over command has to be outside of JSON anyway. + +The reason not to use URI segments / HTTP verbs / REST semantics is to have consistent request size. + +### ALPN to agree handshake version + +Client and server use [ALPN extension][18] of TLS to agree handshake version. + +Server SHOULD send `xftp/1` protocol name and the client should confirm this name in order to use the current protocol version. This is added to allow support of older clients without breaking backward compatibility and to extend or modify handshake syntax. + +If the client does not confirm this protocol name, the server would fall back to v1 of XFTP protocol. + +### Transport handshake + +When a client and a server agree on handshake version using ALPN extension, they should proceed with XFTP handshake. + +As with SMP, a client doesn't reveal its version range to avoid version fingerprinting. Unlike SMP, XFTP runs a HTTP2 protocol over TLS and the server can't just send its handshake right away. So a session handshake is driven by client-sent requests: + +1. To pass initiative to the server, the client sends a request with empty body. +2. Server responds with its `paddedServerHello` block. +3. Clients sends a request containing `paddedClientHello` block, +4. Server sends an empty response, finalizing the handshake. + +Once TLS handshake is complete, client and server will exchange blocks of fixed size (16384 bytes). + +```abnf +paddedServerHello = +serverHello = xftpVersionRange sessionIdentifier serverCert signedServerKey ignoredPart +xftpVersionRange = minXftpVersion maxXftpVersion +minXftpVersion = xftpVersion +maxXftpVersion = xftpVersion +sessionIdentifier = shortString +; unique session identifier derived from transport connection handshake +serverCert = originalLength +signedServerKey = originalLength ; signed by server certificate + +paddedClientHello = +clientHello = xftpVersion keyHash ignoredPart +; chosen XFTP protocol version - must be the maximum supported version +; within the range offered by the server + +xftpVersion = 2*2OCTET ; Word16 version number +keyHash = shortString +shortString = length length*OCTET +length = 1*1OCTET +originalLength = 2*2OCTET +ignoredPart = *OCTET +``` + +In XFTP v2 the handshake is only used for version negotiation, but `serverCert` and `signedServerKey` must be validated by the client. + +`keyHash` is the CA fingerprint used by client to validate TLS certificate chain and is checked by a server against its own key. + +`ignoredPart` in handshake allows to add additional parameters in handshake without changing protocol version - the client and servers must ignore any extra bytes within the original block length. + +For TLS transport client should assert that `sessionIdentifier` is equal to `tls-unique` channel binding defined in [RFC 5929][14] (TLS Finished message struct); we pass it in `serverHello` block to allow communication over some other transport protocol (possibly, with another channel binding). + +### Requests and responses + +- File sender: + - create file chunk record. + - Parameters: + - Ed25519 key for subsequent sender commands and Ed25519 keys for commands of each recipient. + - chunk size. + - Response: + - chunk ID for the sender and different IDs for all recipients. + - add recipients to file chunk + - Parameters: + - sender's chunk ID + - Ed25519 keys for commands of each recipient. + - Response: + - chunk IDs for new recipients. + - upload file chunk. + - delete file chunk (invalidates all recipient IDs). +- File recipient: + - download file chunk: + - chunk ID + - DH key for additional encryption of the chunk. + - command should be signed with the key passed by the sender when creating chunk record. + - delete file chunk ID (only for one recipient): signed with the same key. + +## XFTP commands + +Commands syntax below is provided using ABNF with case-sensitive strings extension. + +```abnf +xftpCommand = ping / senderCommand / recipientCmd / serverMsg +senderCommand = register / add / put / delete +recipientCmd = get / ack +serverMsg = pong / sndIds / rcvIds / ok / file +``` + +The syntax of specific commands and responses is defined below. + +### Correlating responses with commands + +Commands are made via HTTP2 requests, responses to commands are correlated as HTTP2 responses. + +### Command authentication + +XFTP servers must authenticate all transmissions (excluding `ping`) by verifying the client signatures. Command signature should be generated by applying the algorithm specified for the file to the `signed` block of the transmission, using the key associated with the file chunk ID (recipient's or sender's depending on which file chunk ID is used). + +### Keep-alive command + +To keep the transport connection alive and to generate noise traffic the clients should use `ping` command to which the server responds with `pong` response. This command should be sent unsigned and without file chunk ID. + +```abnf +ping = %s"PING" +``` + +This command is always sent unsigned. + + data FileResponse = ... | FRPong | ... + +```abnf +pong = %s"PONG" +``` + +### File sender commands + +Sending any of the commands in this section (other than `register`, that is sent without file chunk ID) is only allowed with sender's ID. + +#### Register new file chunk + +This command is sent by the sender to the XFTP server to register a new file chunk. + +Servers SHOULD support basic auth with this command, to allow only server owners and trusted users to create file chunks on the servers. + +The syntax is: + +```abnf +register = %s"FNEW " fileInfo rcvPublicAuthKeys basicAuth +fileInfo = sndKey size digest +sndKey = length x509encoded +size = 1*DIGIT +digest = length *OCTET +rcvPublicAuthKeys = length 1*rcvPublicAuthKey +rcvPublicAuthKey = length x509encoded +basicAuth = "0" / "1" length *OCTET + +x509encoded = + +length = 1*1 OCTET +``` + +If the file chunk is registered successfully, the server must send `sndIds` response with the sender's and recipients' file chunk IDs: + +```abnf +sndIds = %s"SIDS " senderId recipientIds +senderId = length *OCTET +recipientIds = length 1*recipientId +recipientId = length *OCTET +``` + +#### Add file chunk recipients + +This command is sent by the sender to the XFTP server to add additional recipient keys to the file chunk record, in case number of keys requested by client didn't fit into `register` command. The syntax is: + +```abnf +add = %s"FADD " rcvPublicAuthKeys +rcvPublicAuthKeys = length 1*rcvPublicAuthKey +rcvPublicAuthKey = length x509encoded +``` + +If additional keys were added successfully, the server must send `rcvIds` response with the added recipients' file chunk IDs: + +```abnf +rcvIds = %s"RIDS " recipientIds +recipientIds = length 1*recipientId +recipientId = length *OCTET +``` + +#### Upload file chunk + +This command is sent by the sender to the XFTP server to upload file chunk body to server. The syntax is: + +```abnf +put = %s"FPUT" +``` + +Chunk body is streamed via HTTP2 request. + +If file chunk body was successfully received, the server must send `ok` response. + +```abnf +ok = %s"OK" +``` + +#### Delete file chunk + +This command is sent by the sender to the XFTP server to delete file chunk from the server. The syntax is: + +```abnf +delete = %s"FDEL" +``` + +Server should delete file chunk record, invalidating all recipient IDs, and delete file body from file storage. If file chunk was successfully deleted, the server must send `ok` response. + +### File recipient commands + +Sending any of the commands in this section is only allowed with recipient's ID. + +#### Download file chunk + +This command is sent by the recipient to the XFTP server to download file chunk body from the server. The syntax is: + +```abnf +get = %s"FGET " rDhKey +rDhKey = length x509encoded +``` + +If requested file is successfully located, the server must send `file` response. File chunk body is sent as HTTP2 response body. + +```abnf +file = %s"FILE " sDhKey cbNonce +sDhKey = length x509encoded +cbNonce = +``` + +Chunk is additionally encrypted on the way from the server to the recipient using a key agreed via ephemeral DH keys `rDhKey` and `sDhKey`, so there is no ciphertext in common between sent and received traffic inside TLS connection, in order to complicate traffic correlation attacks, if TLS is compromised. + +#### Acknowledge file chunk download + +This command is sent by the recipient to the XFTP server to acknowledge file reception, deleting file ID from server for this recipient. The syntax is: + +```abnf +ack = %s"FACK" +``` + +If file recipient ID is successfully deleted, the server must send `ok` response. + +In current implementation of XFTP protocol in SimpleX Chat clients don't use FACK command. Files are automatically expired on servers after configured time interval. + +## Threat model + +#### Global Assumptions + + - A user protects their local database and key material. + - The user's application is authentic, and no local malware is running. + - The cryptographic primitives in use are not broken. + - A user's choice of servers is not directly tied to their identity or otherwise represents distinguishing information about the user. + +#### A passive adversary able to monitor the traffic of one user + +*can:* + + - identify that and when a user is sending files over XFTP protocol. + + - determine which servers the user sends/receives files to/from. + + - observe how much traffic is being sent, and make guesses as to its purpose. + +*cannot:* + + - see who sends files to the user and who the user sends the files to. + +#### A passive adversary able to monitor a set of file senders and recipients + + *can:* + + - learn which XFTP servers are used to send and receive files for which users. + + - learn when files are sent and received. + + - perform traffic correlation attacks against senders and recipients and correlate senders and recipients within the monitored set, frustrated by the number of users on the servers. + + - observe how much traffic is being sent, and make guesses as to its purpose + +*cannot, even in case of a compromised transport protocol:* + + - perform traffic correlation attacks with any increase in efficiency over a non-compromised transport protocol + +#### XFTP server + +*can:* + +- learn when file senders and recipients are online. + +- know how many file chunks and chunk sizes are sent via the server. + +- perform the correlation of the file chunks as belonging to one file via either a re-used transport connection, user's IP address, or connection timing regularities. + +- learn file senders' and recipients' IP addresses, and infer information (e.g. employer) based on the IP addresses, as long as Tor is not used. + +- delete file chunks, preventing file delivery, as long as redundant delivery is not used. + +- lie about the state of a file chunk to the recipient and/or to the sender (e.g. deleted when it is not). + +- refuse deleting the file when instructed by the sender. + +*cannot:* + +- undetectably corrupt file chunks. + +- learn the contents, name or the exact size of sent files. + +- learn approximate size of sent files, as long as more than one server is used to send file chunks. + +- compromise the users' end-to-end encryption of files with an active attack. + +#### An attacker who obtained Alice's (decrypted) chat database + +*can:* + +- see the history of all files exchanged by Alice with her communication partners, as long as files were not deleted from the database. + +- receive all files sent and received by Alice that did not expire yet, as long as information about these files was not removed from the database. + +- prevent Alice's contacts from receiving the files she sent by deleting all or some of the file chunks from XFTP servers. + +#### A user's contact + +*can:* + +- spam the user with files. + +- forever retain files from the user. + +*cannot:* + +- cryptographically prove to a third-party that a file came from a user (assuming the user's device is not seized). + +- prove that two contacts they have is the same user. + +- cannot collaborate with another of the user's contacts to confirm they are communicating with the same user, even if they receive the same file. + +#### An attacker with Internet access + +*can:* + +- Denial of Service XFTP servers. + +*cannot:* + +- send files to a user who they are not connected with. + +- enumerate file chunks on an XFTP server. diff --git a/protocol/xrcp.md b/protocol/xrcp.md new file mode 100644 index 000000000..9f7187e66 --- /dev/null +++ b/protocol/xrcp.md @@ -0,0 +1,330 @@ +Version 1, 2024-06-22 + +# SimpleX Remote Control Protocol + +## Table of contents + +- [Abstract](#abstract) +- [XRCP model](#xrcp-model) +- [Transport protocol](#transport-protocol) + - [Session invitation](#session-invitation) + - [Establishing TLS connection](#establishing-tls-connection) + - [Session verification and protocol negotiation](#session-verification-and-protocol-negotiation) + - [Controller/host session operation](#Ņontrollerhost-session-operation) +- [Key agreement for announcement packet and for session](#key-agreement-for-announcement-packet-and-for-session) +- [Threat model](#threat-model) + +## Abstract + +The SimpleX Remote Control Protocol is a client-server protocol designed to transform application UIs into thin clients, enabling remote control from another device. This approach allows users to remotely access and utilize chat profiles without the complexities of master-master replication for end-to-end encryption states. + +Like SMP and XFTP, XRCP leverages out-of-band invitations to mitigate MITM attacks and employs multiple cryptographic layers to safeguard application data. + +## XRCP model + +XRCP assumes two application roles: host (that contain the application data) and controller that gains limited access to host data. +Applications are also split into two components: UI and core. + +When an XRCP session is established a host UI is locked out and a controller UI uses its core to proxy commands to the host core, getting back responses and events. + +``` + + +------+ +------+ xrcp +------+ +------+ + | Ctrl | commands | Ctrl | commands | Host | | Host | +user ---> | UI | -----------> | Core | -----------> | Core | | UI | + +------+ +------+ +------+ +------+ + ^ responses | ^ xrcp responses | ^ + |<------------------| |<-----------------| | +-------------+ + | events | | | Application |-+ + |<------------------| |----> | protocol | | + | servers | | + +-------------+ | + +--------------+ +``` + +## Transport protocol + +Protocol consists of four phases: +- controller session invitation +- establishing session TLS connection +- session verification and protocol negotiation +- session operation + +![Session sequence](./diagrams/xrcp/session.svg) + +### Session invitation + +The invitation to the first session between host and controller pair MUST be shared out-of-band, to establish a long term identity keys/certificates of the controller to host device. + +The subsequent sessions can be announced via an application-defined site-local multicast group, e.g. `224.0.0.251` (also used in mDNS/bonjour) and an application-defined port (SimpleX Chat uses 5227). + +The session invitation contains this data: +- supported version range for remote control protocol. +- application-specific information, e.g. device name, application name and supported version range, settings, etc. +- session start time in seconds since epoch. +- if multicast is used, counter of announce packets sent by controller. +- network address (ipv4 address and port) of the controller. +- CA TLS certificate fingerprint of the controller - this is part of long term identity of the controller established during the first session, and repeated in the subsequent session announcements. +- Session Ed25519 public key used to verify the announcement and commands - this mitigates the compromise of the long term signature key, as the controller will have to sign each command with this key first. +- Long-term Ed25519 public key used to verify the announcement and commands - this is part of the long term controller identity. +- Session X25519 DH key and SNTRUP761 KEM encapsulation key to agree session encryption (both for multicast announcement and for commands and responses in TLS), as described in https://datatracker.ietf.org/doc/draft-josefsson-ntruprime-hybrid/. The new keys are used for each session, and if client key is already available (from the previous session), the computed shared secret will be used to encrypt the announcement multicast packet. The out-of-band invitation is unencrypted. DH public key and KEM encapsulation key are sent unencrypted. NaCL crypto_box is used for encryption. + +Host application decrypts (except the first session) and validates the invitation: +- Session signature is valid. +- Timestamp is within some window from the current time. +- Long-term key signature is valid. +- Long-term CA and signature key are the same as in the first session. +- Some version in the offered range is supported. + +OOB session invitation is a URI with this syntax: + +```abnf +sessionAddressUri = "xrcp:/" encodedCAFingerprint "@" host ":" port "#/?" qsParams +encodedCAFingerprint = base64url +host = ; in textual form, RFC4001 +port = 1*DIGIT ; uint16 +qsParams = param *("&" param) +param = versionRangeParam / appInfoParam / sessionTsParam / + sessPubKeyParam / idPubKeyParam / dhPubKeyParam / + sessSignatureParam / idSignatureParam +versionRangeParam = "v=" (versionParam / (versionParam "-" versionParam)) +versionParam = 1*DIGIT +appInfoParam = "app=" escapedJSON +sessionTsParam = "ts=" 1*DIGIT +sessPubKeyParam = "skey=" base64url +idPubKeyParam = "idkey=" base64url +dhPubKeyParam = "dh=" base64url +sessSignatureParam = "ssig=" base64url ; signs the URI with this and idSignatureParam param removed +idSignatureParam = "idsig=" base64url ; signs the URI with this param removed +base64url = ; RFC4648, section 5 +``` + +Multicast session announcement is a binary encoded packet with this syntax: + +```abnf +sessionAddressPacket = dhPubKey nonce encrypted(unpaddedSize sessionAddress packetPad) +dhPubKey = length x509encoded ; same as announced +nonce = length *OCTET +sessionAddress = largeLength sessionAddressUri ; as above +length = 1*1 OCTET ; for binary data up to 255 bytes +largeLength = 2*2 OCTET ; for binary data up to 65535 bytes +packetPad = ; possibly, we may need to move KEM agreement one step later, +; with encapsulation key in HELLO block and KEM ciphertext in reply to HELLO. +``` + +### Establishing TLS connection + +Both controller and host use 2-element certificate chains with unique self-signed CA root representing long-term identities. Leaf certificates aren't stored and instead generated on each session start. + +A controller runs a TCP server to avoid opening listening socket on a host, which might create an attack vector. A controller keeps no sensitive data to be exposed this way. + +During TLS handshake, parties validate certificate chains against previously known (from invitation or storage) CA fingerprints. The fingerprints MUST be the same as in the invitation and in the subsequent connections. + +### Session verification and protocol negotiation + +Once TLS session is established, both the host and controller devices present a "session security code" to the user who must match them (e.g., visually or via QR code scan) and confirm on the host device. The session security code must be a digest of tlsunique channel binding. As it is computed as a digest of the TLS handshake for both the controller and the host, it will validate that the same TLS certificates are used on both sides, and that the same TLS session is established, mitigating the possibility of MITM attack in the connection. + +Once the session is confirmed by the user, the host sends HELLO block to the controller. + +XRCP blocks inside TLS are padded to 16384 bytes. + +Host HELLO block must contain: +- new session DH key - used to compute new shared secret with the controller keys from the announcement. +- encrypted part of HELLO block (JSON object), containing: + - chosen protocol version. + - host CA TLS certificate fingerprint - part of host long term identity - must match the one presented in TLS handshake and the previous sessions, otherwise the connection is terminated. + - KEM encapsulation key - used to compute new shared secret for the session. + - additional application specific parameters, e.g host device name, application version, host settings or JSON encoding format. + +Host HELLO block syntax: + +```abnf +hostHello = %s"HELLO " dhPubKey nonce encrypted(unpaddedSize hostHelloJSON helloPad) pad +unpaddedSize = largeLength +dhPubKey = length x509encoded +pad = +helloPad = +largeLength = 2*2 OCTET +``` + +The controller decrypts (including the first session) and validates the received HELLO block: +- Chosen versions are supported (must be within offered ranges). +- CA fingerprint matches the one presented in TLS handshake and the previous sessions - in subsequent sessions TLS connection should be rejected if the fingerprint is different. + +[JTD schema](https://www.rfc-editor.org/rfc/rfc8927) for the encrypted part of host HELLO block `hostHelloJSON`: + +```json +{ + "definitions": { + "version": { + "type": "string", + "metadata": { + "format": "[0-9]+" + } + }, + "base64url": { + "type": "string", + "metadata": { + "format": "base64url" + } + } + }, + "properties": { + "v": {"ref": "version"}, + "ca": {"ref": "base64url"}, + "kem": {"ref": "base64url"} + }, + "optionalProperties": { + "app": {"properties": {}, "additionalProperties": true} + }, + "additionalProperties": true +} +``` + +The controller should reply with with `ctrlHello` or `ctrlError` response: + +```abnf +ctrlHello = %s"HELLO " kemCiphertext nonce encrypted(unpaddedSize ctrlHelloJSON helloPad) pad +; ctrlHelloJSON is encrypted with the hybrid secret, +; including both previously agreed DH secret and KEM secret from kemCiphertext +unpaddedSize = largeLength +kemCiphertext = largeLength *OCTET +pad = +helloPad = +largeLength = 2*2 OCTET + +ctrlError = %s"ERROR " nonce encrypted(unpaddedSize ctrlErrorMessage helloPad) pad +ctrlErrorMessage = ; encrypted using previously agreed DH secret. +``` + +JTD schema for the encrypted part of controller HELLO block `ctrlHelloJSON`: + +```json +{ + "properties": {}, + "additionalProperties": true +} +``` + +Once the controller replies HELLO to the valid host HELLO block, it should stop accepting new TCP connections. + +### Controller/host session operation + +The protocol for communication during the session is out of scope of this protocol. + +SimpleX Chat uses HTTP2 encoding, where host device acts as a server and controller acts as a client (these roles are reversed compared with TLS connection, restoring client-server semantics in HTTP). + +Payloads in the protocol must be encrypted using NaCL secret_box using the hybrid shared secret agreed during session establishment. + +Commands of the controller must be signed after the encryption using the controller's session and long term Ed25519 keys. + +tlsunique channel binding from TLS session MUST be included in commands (included in the signed body). + +The syntax for encrypted command and response body encoding: + +```abnf +commandBody = encBody sessSignature idSignature [attachment] +responseBody = encBody [attachment] ; counter must match command +encBody = nonce encLength32 encrypted(tlsunique counter body) +attachment = %x01 nonce encLength32 encrypted(attachment) +noAttachment = %x00 +tlsunique = length 1*OCTET +counter = 8*8 OCTET ; int64 +encLength32 = 4*4 OCTET ; uint32, includes authTag +``` + +If the command or response includes attachment, its hash must be included in command/response and validated. + +## Key agreement for announcement packet and for session + +Initial announcement is shared out-of-band (URI with xrcp scheme), and it is not encrypted. + +This announcement contains only DH keys, as KEM key is too large to include in QR code, which are used to agree encryption key for host HELLO block. The host HELLO block will contain DH key in plaintext part and KEM encapsulation (public) key in encrypted part, that will be used to determine the shared secret (using SHA256 over concatenated DH shared secret and KEM encapsulated secret) both for controller HELLO response (that contains KEM ciphertext in plaintext part) and subsequent session commands and responses. + +During the next session the announcement is sent via encrypted multicast block. The shared key for this announcement and for host HELLO block is determined using the KEM shared secret from the previous session and DH shared secret computed using the host DH key from the previous session and the new controller DH key from the announcement. + +For the session, the shared secret is computed again using the KEM shared secret encapsulated by the controller using the new KEM key from the host HELLO block and DH shared secret computed using the host DH key from HELLO block and the new controller DH key from the announcement. + +In pseudo-code: + +``` +// session 1 +hostHelloSecret(1) = dhSecret(1) +sessionSecret(1) = sha256(dhSecret(1) || kemSecret(1)) // to encrypt session 1 data, incl. controller hello +dhSecret(1) = dh(hostHelloDhKey(1), controllerInvitationDhKey(1)) +kemCiphertext(1) = enc(kemSecret(1), kemEncKey(1)) +// kemEncKey is included in host HELLO, kemCiphertext - in controller HELLO +kemSecret(1) = dec(kemCiphertext(1), kemDecKey(1)) + +// multicast announcement for session n +announcementSecret(n) = sha256(dhSecret(n')) +dhSecret(n') = dh(hostHelloDhKey(n - 1), controllerDhKey(n)) + +// session n +hostHelloSecret(n) = dhSecret(n) +sessionSecret(n) = sha256(dhSecret(n) || kemSecret(n)) // to encrypt session n data, incl. controller hello +dhSecret(n) = dh(hostHelloDhKey(n), controllerDhKey(n)) +// controllerDhKey(n) is either from invitation or from multicast announcement +kemCiphertext(n) = enc(kemSecret(n), kemEncKey(n)) +kemSecret(n) = dec(kemCiphertext(n), kemDecKey(n)) +``` + +If controller fails to store the new host DH key after receiving HELLO block, the encryption will become out of sync and the host won't be able to decrypt the next announcement. To mitigate it, the host should keep the last session DH key and also previous session DH key to try to decrypt the next announcement computing shared secret using both keys (first the new one, and in case it fails - the previous). + +To decrypt a multicast announcement, the host should try to decrypt it using the keys of all known (paired) remote controllers. + +## Threat model + +#### A passive network adversary able to monitor the site-local traffic: + +*can:* +- observe session times, duration and volume of the transmitted data between host and controller. + +*cannot:* +- observe the content of the transmitted data. +- substitute the transmitted commands or responses. +- replay transmitted commands or events from the hosts. + +#### An active network adversary able to intercept and substitute the site-local traffic: + +*can:* +- prevent host and controller devices from establishing the session + +*cannot:* +- same as passive adversary, provided that user visually verified session code out-of-band. + +#### An active adversary with the access to the network: + +*can:* +- spam controller device. + +*cannot:* +- compromise host or controller devices. + +#### An active adversary with the access to the network who also observed OOB announcement: + +*can:* +- connect to controller instead of the host. +- present incorrect data to the controller. + +*cannot:* +- connect to the host or make host connect to itself. + +#### Compromised controller device: + +*can:* +- observe the content of the transmitted data. +- access any data of the controlled host application, within the capabilities of the provided API. + +*cannot:* +- access other data on the host device. +- compromise host device. + +#### Compromised host device: + +*can:* +- present incorrect data to the controller. +- incorrectly interpret controller commands. + +*cannot:* +- access controller data, even related to this host device. diff --git a/rfcs/2022-12-26-simplex-file-transfer.md b/rfcs/done/2022-12-26-simplex-file-transfer.md similarity index 100% rename from rfcs/2022-12-26-simplex-file-transfer.md rename to rfcs/done/2022-12-26-simplex-file-transfer.md diff --git a/rfcs/2022-12-27-queue-quota.md b/rfcs/done/2022-12-27-queue-quota.md similarity index 100% rename from rfcs/2022-12-27-queue-quota.md rename to rfcs/done/2022-12-27-queue-quota.md diff --git a/rfcs/2023-05-02-resync-ratchets.md b/rfcs/done/2023-05-02-resync-ratchets.md similarity index 100% rename from rfcs/2023-05-02-resync-ratchets.md rename to rfcs/done/2023-05-02-resync-ratchets.md diff --git a/rfcs/2023-05-03-delivery-receipts.md b/rfcs/done/2023-05-03-delivery-receipts.md similarity index 100% rename from rfcs/2023-05-03-delivery-receipts.md rename to rfcs/done/2023-05-03-delivery-receipts.md diff --git a/rfcs/2023-05-24-smp-delivery-proxy.md b/rfcs/done/2023-05-24-smp-delivery-proxy.md similarity index 100% rename from rfcs/2023-05-24-smp-delivery-proxy.md rename to rfcs/done/2023-05-24-smp-delivery-proxy.md diff --git a/rfcs/2023-06-08-resync-ratchets.md b/rfcs/done/2023-06-08-resync-ratchets.md similarity index 100% rename from rfcs/2023-06-08-resync-ratchets.md rename to rfcs/done/2023-06-08-resync-ratchets.md diff --git a/rfcs/2023-09-12-second-relays.md b/rfcs/done/2023-09-12-second-relays.md similarity index 100% rename from rfcs/2023-09-12-second-relays.md rename to rfcs/done/2023-09-12-second-relays.md diff --git a/rfcs/2023-10-25-remote-control.md b/rfcs/done/2023-10-25-remote-control.md similarity index 100% rename from rfcs/2023-10-25-remote-control.md rename to rfcs/done/2023-10-25-remote-control.md diff --git a/rfcs/2024-01-26-file-links.md b/rfcs/done/2024-01-26-file-links.md similarity index 100% rename from rfcs/2024-01-26-file-links.md rename to rfcs/done/2024-01-26-file-links.md diff --git a/rfcs/2024-02-03-deniability.md b/rfcs/done/2024-02-03-deniability.md similarity index 100% rename from rfcs/2024-02-03-deniability.md rename to rfcs/done/2024-02-03-deniability.md diff --git a/rfcs/2024-03-28-xftp-version.md b/rfcs/done/2024-03-28-xftp-version.md similarity index 100% rename from rfcs/2024-03-28-xftp-version.md rename to rfcs/done/2024-03-28-xftp-version.md diff --git a/rfcs/2021-02-28-streams.md b/rfcs/rejected/2021-02-28-streams.md similarity index 100% rename from rfcs/2021-02-28-streams.md rename to rfcs/rejected/2021-02-28-streams.md diff --git a/simplexmq.cabal b/simplexmq.cabal index 3db522c0c..38c44b229 100644 --- a/simplexmq.cabal +++ b/simplexmq.cabal @@ -5,7 +5,7 @@ cabal-version: 1.12 -- see: https://github.com/sol/hpack name: simplexmq -version: 5.8.0.10 +version: 5.8.1.0 synopsis: SimpleXMQ message broker description: This package includes <./docs/Simplex-Messaging-Server.html server>, <./docs/Simplex-Messaging-Client.html client> and diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs index c550ba04a..63f74b5b2 100644 --- a/src/Simplex/Messaging/Agent.hs +++ b/src/Simplex/Messaging/Agent.hs @@ -82,6 +82,7 @@ module Simplex.Messaging.Agent setNetworkConfig, setUserNetworkInfo, reconnectAllServers, + reconnectSMPServer, registerNtfToken, verifyNtfToken, checkNtfToken, diff --git a/src/Simplex/Messaging/Agent/Client.hs b/src/Simplex/Messaging/Agent/Client.hs index 303bb55be..2ffd6e3a4 100644 --- a/src/Simplex/Messaging/Agent/Client.hs +++ b/src/Simplex/Messaging/Agent/Client.hs @@ -32,6 +32,7 @@ module Simplex.Messaging.Agent.Client closeAgentClient, closeProtocolServerClients, reconnectServerClients, + reconnectSMPServer, closeXFTPServerClient, runSMPServerTest, runXFTPServerTest, @@ -925,6 +926,16 @@ reconnectServerClients :: ProtocolServerClient v err msg => AgentClient -> (Agen reconnectServerClients c clientsSel = readTVarIO (clientsSel c) >>= mapM_ (forkIO . closeClient_ c) +reconnectSMPServer :: AgentClient -> UserId -> SMPServer -> IO () +reconnectSMPServer c userId srv = do + cs <- readTVarIO $ smpClients c + let vs = M.foldrWithKey srvClient [] cs + mapM_ (forkIO . closeClient_ c) vs + where + srvClient (userId', srv', _) v + | userId == userId' && srv == srv' = (v :) + | otherwise = id + closeClient :: ProtocolServerClient v err msg => AgentClient -> (AgentClient -> TMap (TransportSession msg) (ClientVar msg)) -> TransportSession msg -> IO () closeClient c clientSel tSess = atomically (TM.lookupDelete tSess $ clientSel c) >>= mapM_ (closeClient_ c) diff --git a/src/Simplex/Messaging/Agent/Protocol.hs b/src/Simplex/Messaging/Agent/Protocol.hs index 0067d4ada..d6bbc13ca 100644 --- a/src/Simplex/Messaging/Agent/Protocol.hs +++ b/src/Simplex/Messaging/Agent/Protocol.hs @@ -332,7 +332,6 @@ data AEvent (e :: AEntity) where UP :: SMPServer -> [ConnId] -> AEvent AENone SWITCH :: QueueDirection -> SwitchPhase -> ConnectionStats -> AEvent AEConn RSYNC :: RatchetSyncState -> Maybe AgentCryptoError -> ConnectionStats -> AEvent AEConn - MID :: AgentMsgId -> PQEncryption -> AEvent AEConn SENT :: AgentMsgId -> Maybe SMPServer -> AEvent AEConn MWARN :: AgentMsgId -> AgentErrorType -> AEvent AEConn MERR :: AgentMsgId -> AgentErrorType -> AEvent AEConn @@ -401,7 +400,6 @@ data AEventTag (e :: AEntity) where UP_ :: AEventTag AENone SWITCH_ :: AEventTag AEConn RSYNC_ :: AEventTag AEConn - MID_ :: AEventTag AEConn SENT_ :: AEventTag AEConn MWARN_ :: AEventTag AEConn MERR_ :: AEventTag AEConn @@ -454,7 +452,6 @@ aEventTag = \case UP {} -> UP_ SWITCH {} -> SWITCH_ RSYNC {} -> RSYNC_ - MID {} -> MID_ SENT {} -> SENT_ MWARN {} -> MWARN_ MERR {} -> MERR_ diff --git a/src/Simplex/Messaging/Server.hs b/src/Simplex/Messaging/Server.hs index 88e3bf4a3..dfb4973ea 100644 --- a/src/Simplex/Messaging/Server.hs +++ b/src/Simplex/Messaging/Server.hs @@ -229,7 +229,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg} = do initialDelay <- (startAt -) . fromIntegral . (`div` 1000000_000000) . diffTimeToPicoseconds . utctDayTime <$> liftIO getCurrentTime liftIO $ putStrLn $ "server stats log enabled: " <> statsFilePath liftIO $ threadDelay' $ 1000000 * (initialDelay + if initialDelay < 0 then 86400 else 0) - ServerStats {fromTime, qCreated, qSecured, qDeletedAll, qDeletedNew, qDeletedSecured, qSub, qSubAuth, qSubDuplicate, qSubProhibited, msgSent, msgSentAuth, msgSentQuota, msgSentLarge, msgRecv, msgExpired, activeQueues, msgSentNtf, msgRecvNtf, activeQueuesNtf, qCount, msgCount, pRelays, pRelaysOwn, pMsgFwds, pMsgFwdsOwn, pMsgFwdsRecv} <- asks serverStats + ss@ServerStats {fromTime, qCreated, qSecured, qDeletedAll, qDeletedNew, qDeletedSecured, qSub, qSubAuth, qSubDuplicate, qSubProhibited, msgSent, msgSentAuth, msgSentQuota, msgSentLarge, msgRecv, msgExpired, activeQueues, msgSentNtf, msgRecvNtf, activeQueuesNtf, qCount, msgCount, pRelays, pRelaysOwn, pMsgFwds, pMsgFwdsOwn, pMsgFwdsRecv} <- asks serverStats let interval = 1000000 * logInterval forever $ do withFile statsFilePath AppendMode $ \h -> liftIO $ do @@ -255,6 +255,9 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg} = do msgSentNtf' <- atomically $ swapTVar msgSentNtf 0 msgRecvNtf' <- atomically $ swapTVar msgRecvNtf 0 psNtf <- atomically $ periodStatCounts activeQueuesNtf ts + msgNtfs' <- atomically $ swapTVar (msgNtfs ss) 0 + msgNtfNoSub' <- atomically $ swapTVar (msgNtfNoSub ss) 0 + msgNtfLost' <- atomically $ swapTVar (msgNtfLost ss) 0 pRelays' <- atomically $ getResetProxyStatsData pRelays pRelaysOwn' <- atomically $ getResetProxyStatsData pRelaysOwn pMsgFwds' <- atomically $ getResetProxyStatsData pMsgFwds @@ -296,7 +299,10 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg} = do show qSubProhibited', show msgSentAuth', show msgSentQuota', - show msgSentLarge' + show msgSentLarge', + show msgNtfs', + show msgNtfNoSub', + show msgNtfLost' ] ) liftIO $ threadDelay' interval @@ -714,7 +720,7 @@ client thParams' clnt@Client {subscriptions, ntfSubscriptions, rcvQ, sndQ, sessi let own = isOwnServer a srv inc own pRequests inc own $ if temporaryClientError e then pErrorsConnect else pErrorsOther - logError $ "Error connecting: " <> decodeLatin1 (strEncode $ host srv) <> " " <> tshow e + logWarn $ "Error connecting: " <> decodeLatin1 (strEncode $ host srv) <> " " <> tshow e pure . ERR $ smpProxyError e where proxyResp smp = @@ -1010,7 +1016,15 @@ client thParams' clnt@Client {subscriptions, ntfSubscriptions, rcvQ, sndQ, sessi pure $ err QUOTA Just msg -> time "SEND ok" $ do when (notification msgFlags) $ do - atomically . trySendNotification msg =<< asks random + forM_ (notifier qr) $ \ntf -> do + asks random >>= atomically . trySendNotification ntf msg >>= \case + Nothing -> do + atomically $ modifyTVar' (msgNtfNoSub stats) (+ 1) + logWarn "No notification subscription" + Just False -> do + atomically $ modifyTVar' (msgNtfLost stats) (+ 1) + logWarn "Dropped message notification" + Just True -> atomically $ modifyTVar' (msgNtfs stats) (+ 1) atomically $ modifyTVar' (msgSentNtf stats) (+ 1) atomically $ updatePeriodStats (activeQueuesNtf stats) (recipientId qr) atomically $ modifyTVar' (msgSent stats) (+ 1) @@ -1033,18 +1047,19 @@ client thParams' clnt@Client {subscriptions, ntfSubscriptions, rcvQ, sndQ, sessi deleted <- atomically $ sum <$> mapM (deleteExpiredMsgs q) old atomically $ modifyTVar' (msgExpired stats) (+ deleted) - trySendNotification :: Message -> TVar ChaChaDRG -> STM () - trySendNotification msg ntfNonceDrg = - forM_ (notifier qr) $ \NtfCreds {notifierId, rcvNtfDhSecret} -> - mapM_ (writeNtf notifierId msg rcvNtfDhSecret ntfNonceDrg) =<< TM.lookup notifierId notifiers + trySendNotification :: NtfCreds -> Message -> TVar ChaChaDRG -> STM (Maybe Bool) + trySendNotification NtfCreds {notifierId, rcvNtfDhSecret} msg ntfNonceDrg = + mapM (writeNtf notifierId msg rcvNtfDhSecret ntfNonceDrg) =<< TM.lookup notifierId notifiers - writeNtf :: NotifierId -> Message -> RcvNtfDhSecret -> TVar ChaChaDRG -> Client -> STM () + writeNtf :: NotifierId -> Message -> RcvNtfDhSecret -> TVar ChaChaDRG -> Client -> STM Bool writeNtf nId msg rcvNtfDhSecret ntfNonceDrg Client {sndQ = q} = - unlessM (isFullTBQueue q) $ case msg of - Message {msgId, msgTs} -> do - (nmsgNonce, encNMsgMeta) <- mkMessageNotification msgId msgTs rcvNtfDhSecret ntfNonceDrg - writeTBQueue q [(CorrId "", nId, NMSG nmsgNonce encNMsgMeta)] - _ -> pure () + ifM (isFullTBQueue q) (pure False) (sendNtf $> True) + where + sendNtf = case msg of + Message {msgId, msgTs} -> do + (nmsgNonce, encNMsgMeta) <- mkMessageNotification msgId msgTs rcvNtfDhSecret ntfNonceDrg + writeTBQueue q [(CorrId "", nId, NMSG nmsgNonce encNMsgMeta)] + _ -> pure () mkMessageNotification :: ByteString -> SystemTime -> RcvNtfDhSecret -> TVar ChaChaDRG -> STM (C.CbNonce, EncNMsgMeta) mkMessageNotification msgId msgTs rcvNtfDhSecret ntfNonceDrg = do @@ -1184,7 +1199,7 @@ client thParams' clnt@Client {subscriptions, ntfSubscriptions, rcvQ, sndQ, sessi ProhibitSub -> QProhibitSub qDelivered <- decodeLatin1 . encode <$$> tryReadTMVar delivered pure QSub {qSubThread, qDelivered} - + ok :: Transmission BrokerMsg ok = (corrId, queueId, OK) diff --git a/src/Simplex/Messaging/Server/Stats.hs b/src/Simplex/Messaging/Server/Stats.hs index 880791c3d..f2716c9c3 100644 --- a/src/Simplex/Messaging/Server/Stats.hs +++ b/src/Simplex/Messaging/Server/Stats.hs @@ -37,9 +37,12 @@ data ServerStats = ServerStats msgRecv :: TVar Int, msgExpired :: TVar Int, activeQueues :: PeriodStats RecipientId, - msgSentNtf :: TVar Int, - msgRecvNtf :: TVar Int, + msgSentNtf :: TVar Int, -- sent messages with NTF flag + msgRecvNtf :: TVar Int, -- received messages with NTF flag activeQueuesNtf :: PeriodStats RecipientId, + msgNtfs :: TVar Int, -- messages notications delivered to NTF server (<= msgSentNtf) + msgNtfNoSub :: TVar Int, -- no subscriber to notifications (e.g., NTF server not connected) + msgNtfLost :: TVar Int, -- notification is lost because NTF delivery queue is full pRelays :: ProxyStats, pRelaysOwn :: ProxyStats, pMsgFwds :: ProxyStats, @@ -70,6 +73,9 @@ data ServerStatsData = ServerStatsData _msgSentNtf :: Int, _msgRecvNtf :: Int, _activeQueuesNtf :: PeriodStatsData RecipientId, + _msgNtfs :: Int, + _msgNtfNoSub :: Int, + _msgNtfLost :: Int, _pRelays :: ProxyStatsData, _pRelaysOwn :: ProxyStatsData, _pMsgFwds :: ProxyStatsData, @@ -102,6 +108,9 @@ newServerStats ts = do msgSentNtf <- newTVar 0 msgRecvNtf <- newTVar 0 activeQueuesNtf <- newPeriodStats + msgNtfs <- newTVar 0 + msgNtfNoSub <- newTVar 0 + msgNtfLost <- newTVar 0 pRelays <- newProxyStats pRelaysOwn <- newProxyStats pMsgFwds <- newProxyStats @@ -109,7 +118,39 @@ newServerStats ts = do pMsgFwdsRecv <- newTVar 0 qCount <- newTVar 0 msgCount <- newTVar 0 - pure ServerStats {fromTime, qCreated, qSecured, qDeletedAll, qDeletedNew, qDeletedSecured, qSub, qSubAuth, qSubDuplicate, qSubProhibited, msgSent, msgSentAuth, msgSentQuota, msgSentLarge, msgRecv, msgExpired, activeQueues, msgSentNtf, msgRecvNtf, activeQueuesNtf, pRelays, pRelaysOwn, pMsgFwds, pMsgFwdsOwn, pMsgFwdsRecv, qCount, msgCount} + pure + ServerStats + { fromTime, + qCreated, + qSecured, + qDeletedAll, + qDeletedNew, + qDeletedSecured, + qSub, + qSubAuth, + qSubDuplicate, + qSubProhibited, + msgSent, + msgSentAuth, + msgSentQuota, + msgSentLarge, + msgRecv, + msgExpired, + activeQueues, + msgSentNtf, + msgRecvNtf, + activeQueuesNtf, + msgNtfs, + msgNtfNoSub, + msgNtfLost, + pRelays, + pRelaysOwn, + pMsgFwds, + pMsgFwdsOwn, + pMsgFwdsRecv, + qCount, + msgCount + } getServerStatsData :: ServerStats -> STM ServerStatsData getServerStatsData s = do @@ -121,7 +162,7 @@ getServerStatsData s = do _qDeletedSecured <- readTVar $ qDeletedSecured s _qSub <- readTVar $ qSub s _qSubAuth <- readTVar $ qSubAuth s - _qSubDuplicate <- readTVar $ qSubDuplicate s + _qSubDuplicate <- readTVar $ qSubDuplicate s _qSubProhibited <- readTVar $ qSubProhibited s _msgSent <- readTVar $ msgSent s _msgSentAuth <- readTVar $ msgSentAuth s @@ -133,6 +174,9 @@ getServerStatsData s = do _msgSentNtf <- readTVar $ msgSentNtf s _msgRecvNtf <- readTVar $ msgRecvNtf s _activeQueuesNtf <- getPeriodStatsData $ activeQueuesNtf s + _msgNtfs <- readTVar $ msgNtfs s + _msgNtfNoSub <- readTVar $ msgNtfNoSub s + _msgNtfLost <- readTVar $ msgNtfLost s _pRelays <- getProxyStatsData $ pRelays s _pRelaysOwn <- getProxyStatsData $ pRelaysOwn s _pMsgFwds <- getProxyStatsData $ pMsgFwds s @@ -140,7 +184,39 @@ getServerStatsData s = do _pMsgFwdsRecv <- readTVar $ pMsgFwdsRecv s _qCount <- readTVar $ qCount s _msgCount <- readTVar $ msgCount s - pure ServerStatsData {_fromTime, _qCreated, _qSecured, _qDeletedAll, _qDeletedNew, _qDeletedSecured, _qSub, _qSubAuth, _qSubDuplicate, _qSubProhibited, _msgSent, _msgSentAuth, _msgSentQuota, _msgSentLarge, _msgRecv, _msgExpired, _activeQueues, _msgSentNtf, _msgRecvNtf, _activeQueuesNtf, _pRelays, _pRelaysOwn, _pMsgFwds, _pMsgFwdsOwn, _pMsgFwdsRecv, _qCount, _msgCount} + pure + ServerStatsData + { _fromTime, + _qCreated, + _qSecured, + _qDeletedAll, + _qDeletedNew, + _qDeletedSecured, + _qSub, + _qSubAuth, + _qSubDuplicate, + _qSubProhibited, + _msgSent, + _msgSentAuth, + _msgSentQuota, + _msgSentLarge, + _msgRecv, + _msgExpired, + _activeQueues, + _msgSentNtf, + _msgRecvNtf, + _activeQueuesNtf, + _msgNtfs, + _msgNtfNoSub, + _msgNtfLost, + _pRelays, + _pRelaysOwn, + _pMsgFwds, + _pMsgFwdsOwn, + _pMsgFwdsRecv, + _qCount, + _msgCount + } setServerStats :: ServerStats -> ServerStatsData -> STM () setServerStats s d = do @@ -151,7 +227,7 @@ setServerStats s d = do writeTVar (qDeletedNew s) $! _qDeletedNew d writeTVar (qDeletedSecured s) $! _qDeletedSecured d writeTVar (qSub s) $! _qSub d - writeTVar (qSubAuth s) $! _qSubAuth d + writeTVar (qSubAuth s) $! _qSubAuth d writeTVar (qSubDuplicate s) $! _qSubDuplicate d writeTVar (qSubProhibited s) $! _qSubProhibited d writeTVar (msgSent s) $! _msgSent d @@ -164,6 +240,9 @@ setServerStats s d = do writeTVar (msgSentNtf s) $! _msgSentNtf d writeTVar (msgRecvNtf s) $! _msgRecvNtf d setPeriodStats (activeQueuesNtf s) (_activeQueuesNtf d) + writeTVar (msgNtfs s) $! _msgNtfs d + writeTVar (msgNtfNoSub s) $! _msgNtfNoSub d + writeTVar (msgNtfLost s) $! _msgNtfLost d setProxyStats (pRelays s) $! _pRelays d setProxyStats (pRelaysOwn s) $! _pRelaysOwn d setProxyStats (pMsgFwds s) $! _pMsgFwds d @@ -194,6 +273,9 @@ instance StrEncoding ServerStatsData where "msgExpired=" <> strEncode (_msgExpired d), "msgSentNtf=" <> strEncode (_msgSentNtf d), "msgRecvNtf=" <> strEncode (_msgRecvNtf d), + "msgNtfs=" <> strEncode (_msgNtfs d), + "msgNtfNoSub=" <> strEncode (_msgNtfNoSub d), + "msgNtfLost=" <> strEncode (_msgNtfLost d), "activeQueues:", strEncode (_activeQueues d), "activeQueuesNtf:", @@ -228,6 +310,9 @@ instance StrEncoding ServerStatsData where _msgExpired <- opt "msgExpired=" _msgSentNtf <- opt "msgSentNtf=" _msgRecvNtf <- opt "msgRecvNtf=" + _msgNtfs <- opt "msgNtfs=" + _msgNtfNoSub <- opt "msgNtfNoSub=" + _msgNtfLost <- opt "msgNtfLost=" _activeQueues <- optional ("activeQueues:" <* A.endOfLine) >>= \case Just _ -> strP <* optional A.endOfLine @@ -245,7 +330,39 @@ instance StrEncoding ServerStatsData where _pMsgFwds <- proxyStatsP "pMsgFwds:" _pMsgFwdsOwn <- proxyStatsP "pMsgFwdsOwn:" _pMsgFwdsRecv <- opt "pMsgFwdsRecv=" - pure ServerStatsData {_fromTime, _qCreated, _qSecured, _qDeletedAll, _qDeletedNew, _qDeletedSecured, _qSub, _qSubAuth, _qSubDuplicate, _qSubProhibited, _msgSent, _msgSentAuth, _msgSentQuota, _msgSentLarge, _msgRecv, _msgExpired, _msgSentNtf, _msgRecvNtf, _activeQueues, _activeQueuesNtf, _pRelays, _pRelaysOwn, _pMsgFwds, _pMsgFwdsOwn, _pMsgFwdsRecv, _qCount, _msgCount = 0} + pure + ServerStatsData + { _fromTime, + _qCreated, + _qSecured, + _qDeletedAll, + _qDeletedNew, + _qDeletedSecured, + _qSub, + _qSubAuth, + _qSubDuplicate, + _qSubProhibited, + _msgSent, + _msgSentAuth, + _msgSentQuota, + _msgSentLarge, + _msgRecv, + _msgExpired, + _msgSentNtf, + _msgRecvNtf, + _msgNtfs, + _msgNtfNoSub, + _msgNtfLost, + _activeQueues, + _activeQueuesNtf, + _pRelays, + _pRelaysOwn, + _pMsgFwds, + _pMsgFwdsOwn, + _pMsgFwdsRecv, + _qCount, + _msgCount = 0 + } where opt s = A.string s *> strP <* A.endOfLine <|> pure 0 proxyStatsP key = diff --git a/tests/SMPProxyTests.hs b/tests/SMPProxyTests.hs index 036c8b203..d71208db9 100644 --- a/tests/SMPProxyTests.hs +++ b/tests/SMPProxyTests.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE BangPatterns #-} {-# LANGUAGE DataKinds #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE GADTs #-} @@ -14,9 +15,9 @@ module SMPProxyTests where import AgentTests.EqInstances () import AgentTests.FunctionalAPITests -import Control.Concurrent (ThreadId) +import Control.Concurrent (ThreadId, threadDelay) import Control.Logger.Simple -import Control.Monad (forM, forM_, forever) +import Control.Monad (forM, forM_, forever, replicateM_) import Control.Monad.Trans.Except (ExceptT, runExceptT) import Data.ByteString.Char8 (ByteString) import Data.List.NonEmpty (NonEmpty) @@ -39,6 +40,7 @@ import Simplex.Messaging.Transport import Simplex.Messaging.Util (bshow, tshow) import Simplex.Messaging.Version (mkVersionRange) import System.FilePath (splitExtensions) +import System.Random (randomRIO) import Test.Hspec import UnliftIO import Util @@ -82,6 +84,10 @@ smpProxyTests = do it "1x1000" . twoServersFirstProxy $ deliver 1000 it "5x200" . twoServersFirstProxy $ 5 `inParrallel` deliver 200 it "10x100" . twoServersFirstProxy $ 10 `inParrallel` deliver 100 + describe "stress test - no host" $ do + it "1x1000, no delay" . oneServer $ proxyConnectDeadRelay 1000 0 srv1 + xit "1x1000, 100ms" . oneServer $ proxyConnectDeadRelay 1000 100000 srv1 + xit "100x1000, 100ms" . oneServer $ 100 `inParrallel` (randomRIO (0, 1000000) >>= threadDelay >> proxyConnectDeadRelay 1000 100000 srv1) xdescribe "stress test 10k" $ do let deliver n = deliverMessagesViaProxy srv1 srv2 C.SEd448 [] (map bshow [1 :: Int .. n]) it "1x10000" . twoServersFirstProxy $ deliver 10000 @@ -179,6 +185,20 @@ deliverMessagesViaProxy proxyServ relayServ alg unsecuredMsgs securedMsgs = do runExceptT' $ ackSMPMessage rc rPriv rcvId msgId' ) +proxyConnectDeadRelay :: Int -> Int -> SMPServer -> IO () +proxyConnectDeadRelay n d proxyServ = do + g <- C.newRandom + -- set up proxy + pc' <- getProtocolClient g (1, proxyServ, Nothing) defaultSMPClientConfig {serverVRange = mkVersionRange batchCmdsSMPVersion sendingProxySMPVersion} Nothing (\_ -> pure ()) + pc <- either (fail . show) pure pc' + THAuthClient {} <- maybe (fail "getProtocolClient returned no thAuth") pure $ thAuth $ thParams pc + -- get proxy session + replicateM_ n $ do + sess0 <- runExceptT $ connectSMPProxiedRelay pc (SMPServer testHost "45678" testKeyHash) (Just "correct") + case sess0 of + Right !_noWay -> error "got unexpected client" + Left !_err -> threadDelay d + agentDeliverMessageViaProxy :: (C.AlgorithmI a, C.AuthAlgorithm a) => (NonEmpty SMPServer, SMPProxyMode, Bool) -> (NonEmpty SMPServer, SMPProxyMode, Bool) -> C.SAlgorithm a -> ByteString -> ByteString -> IO () agentDeliverMessageViaProxy aTestCfg@(aSrvs, _, aViaProxy) bTestCfg@(bSrvs, _, bViaProxy) alg msg1 msg2 = withAgent 1 aCfg (servers aTestCfg) testDB $ \alice -> diff --git a/tests/ServerTests.hs b/tests/ServerTests.hs index e2ca278d9..a124a42e4 100644 --- a/tests/ServerTests.hs +++ b/tests/ServerTests.hs @@ -608,7 +608,7 @@ testRestoreMessages at@(ATransport t) = logSize testStoreLogFile `shouldReturn` 2 logSize testStoreMsgsFile `shouldReturn` 5 - logSize testServerStatsBackupFile `shouldReturn` 52 + logSize testServerStatsBackupFile `shouldReturn` 55 Right stats1 <- strDecode <$> B.readFile testServerStatsBackupFile checkStats stats1 [rId] 5 1 @@ -626,7 +626,7 @@ testRestoreMessages at@(ATransport t) = logSize testStoreLogFile `shouldReturn` 1 -- the last message is not removed because it was not ACK'd logSize testStoreMsgsFile `shouldReturn` 3 - logSize testServerStatsBackupFile `shouldReturn` 52 + logSize testServerStatsBackupFile `shouldReturn` 55 Right stats2 <- strDecode <$> B.readFile testServerStatsBackupFile checkStats stats2 [rId] 5 3 @@ -645,7 +645,7 @@ testRestoreMessages at@(ATransport t) = logSize testStoreLogFile `shouldReturn` 1 logSize testStoreMsgsFile `shouldReturn` 0 - logSize testServerStatsBackupFile `shouldReturn` 52 + logSize testServerStatsBackupFile `shouldReturn` 55 Right stats3 <- strDecode <$> B.readFile testServerStatsBackupFile checkStats stats3 [rId] 5 5