update rfc and protocol messages

rfcs/2022-08-14-queue-rotation.md

@@ -1,4 +1,4 @@
-# SMP queue rotation and redundancy
+# SMP queue rotation
 
 ## Problem
 
@@ -6,28 +6,22 @@
 
 2. If the user changes the configured SMP server(s), the previously created contacts do not migrate to the new server(s).
 
-3. Server can lose messages.
-
 ## Solution
 
-Additional messages exchanged by SMP agents to negotiate addition and removal of queues to the connection.
+Additional messages exchanged by SMP agents to negotiate migration of the queues to the new server. I considered combining this with queue redundancy, but it increases scope and adds a lot of complexity before it is needed.
 
-This approach allows for both rotation and redundancy, as changing queue will be done be adding a queue and then removing the existing queue.
-
-The reason for this approach is that otherwise it's non-trivial to switch from one queue to another without losing messages or delivering them out of order, it's easier to have messages delivered via both queues during the switch, however short or long time it is.
+The proposed approach separates queue rotation from any queue redundancy that may be added in the future.
 
 ### Messages
 
 Additional agent messages required for the protocol:
 
-`ADD`: add the new queue address(es), the same format as `REPLY` message, encoded as `A`.
+`SWITCH`: notify the sender that the queue has to be rotated to the new one, includes the address (server, sender ID and DH key) of the new queue. Encoded as `S`.
 
-`KEY`: pass sender's key via existing connection (SMP confirmation message will not be used, to avoid the same "race" of the initial key exchange that would create the risk of intercepting the queue for the attacker), encoded as `K`.
+`KEYS`: pass sender's server key and DH key via existing connection (SMP confirmation message will not be used, to avoid the same "race" of the initial key exchange that would create the risk of intercepting the queue for the attacker), encoded as `K`.
 
 `USE`: instruct the sender to use the new queue with sender's queue ID as parameter, encoded as `U`.
 
-`DEL`: instruct the sender to stop using the previous queue, encoded as `X`
-
 ### Protocol
 
 ```
@@ -37,13 +31,20 @@ participant R as Server that has A's receive queue
 participant S as Server that has A's send queue (B's receive queue)
 participant R' as Server that hosts the new A's receive queue
 
-A ->> R': create new queue
-A ->> S ->> B: ADD (R'): address of the new queue
-B ->> R ->> A: KEY (R'): sender's key for the new queue (to avoid the race of SMP confirmation for the initial exchange)
+A ->> R': create new queue without subscription
+A ->> S ->> B: SWITCH (R'): address of the new queue
+B ->> R ->> A: KEYS (R'): sender's key for the new queue (to avoid the race of SMP confirmation for the initial exchange)
+B ->> R ->> A: continue sending new messages to the old queue
 A ->> R': secure queue
 A ->> S ->> B: USE (R'): instruction to use new queue
-B ->> R' ->> A: HELLO
-B ->> R,R' ->> A: send all new messages to both queues
-A ->> S: DEL (R): the previous queue deleted
-B ->> R' ->> A: send all new messages to the new queue only
+B ->> R' ->> A: HELLO: to validate that the sender can send messages to the new queue before switching to it
+A ->> R: suspend queue, receive all messages
+A ->> R: delete queue
+A ->> R': subscribe to the new queue
+B ->> R' ->> A: once sending fails with AUTH error, start sending new (and any undelivered) messages to the new queue
 ```
+
+It will also require extending SMP protocol:
+
+- allow creating the queue without subscribing to it (to avoid processing the messages that can arrive out of order).
+- add message flag / meta-data indicating that this is the last message and the server has no more messages available (so that the recipient knows when it's safe to delete the queue).

src/Simplex/Messaging/Agent.hs

@@ -636,10 +636,9 @@ runSmpQueueMsgDelivery c@AgentClient {subQ} cData@ConnData {connId, duplexHandsh
                         _ -> connError msgId NOT_ACCEPTED
                   AM_REPLY_ -> notifyDel msgId $ ERR e
                   AM_A_MSG_ -> notifyDel msgId $ MERR mId e
-                  AM_A_ADD_ -> pure ()
-                  AM_A_KEY_ -> pure ()
-                  AM_A_USE_ -> pure ()
-                  AM_A_DEL_ -> pure ()
+                  AM_SWITCH_ -> pure ()
+                  AM_KEYS_ -> pure ()
+                  AM_USE_ -> pure ()
                 _
                   -- for other operations BROKER HOST is treated as a permanent error (e.g., when connecting to the server),
                   -- the message sending would be retried
@@ -1028,10 +1027,9 @@ processSMPTransmission c@AgentClient {smpClients, subQ} (srv, v, sessId, rId, cm
                       A_MSG body -> do
                         logServer "<--" c srv rId "MSG <MSG>"
                         notify $ MSG msgMeta msgFlags body
-                      A_ADD _ -> pure ()
-                      A_KEY _ _ -> pure ()
-                      A_USE _ _ -> pure ()
-                      A_DEL _ _ -> pure ()
+                      SWITCH _ _ -> pure ()
+                      KEYS _ _ -> pure ()
+                      USE _ -> pure ()
                     Right _ -> prohibited >> ack
                     Left e@(AGENT A_DUPLICATE) -> do
                       withStore' c (\db -> getLastMsg db connId srvMsgId) >>= \case

src/Simplex/Messaging/Agent/Protocol.hs

@@ -433,10 +433,9 @@ data AgentMessageType
   | AM_HELLO_
   | AM_REPLY_
   | AM_A_MSG_
-  | AM_A_ADD_
-  | AM_A_KEY_
-  | AM_A_USE_
-  | AM_A_DEL_
+  | AM_SWITCH_
+  | AM_KEYS_
+  | AM_USE_
   deriving (Eq, Show)
 
 instance Encoding AgentMessageType where
@@ -446,10 +445,9 @@ instance Encoding AgentMessageType where
     AM_HELLO_ -> "H"
     AM_REPLY_ -> "R"
     AM_A_MSG_ -> "M"
-    AM_A_ADD_ -> "A"
-    AM_A_KEY_ -> "K"
-    AM_A_USE_ -> "U"
-    AM_A_DEL_ -> "X"
+    AM_SWITCH_ -> "S"
+    AM_KEYS_ -> "K"
+    AM_USE_ -> "U"
   smpP =
     A.anyChar >>= \case
       'C' -> pure AM_CONN_INFO
@@ -457,10 +455,9 @@ instance Encoding AgentMessageType where
       'H' -> pure AM_HELLO_
       'R' -> pure AM_REPLY_
       'M' -> pure AM_A_MSG_
-      'A' -> pure AM_A_ADD_
-      'K' -> pure AM_A_KEY_
-      'U' -> pure AM_A_USE_
-      'X' -> pure AM_A_DEL_
+      'S' -> pure AM_SWITCH_
+      'K' -> pure AM_KEYS_
+      'U' -> pure AM_USE_
       _ -> fail "bad AgentMessageType"
 
 agentMessageType :: AgentMessage -> AgentMessageType
@@ -476,10 +473,9 @@ agentMessageType = \case
     -- REPLY is only used in v1
     REPLY _ -> AM_REPLY_
     A_MSG _ -> AM_A_MSG_
-    A_ADD _ -> AM_A_ADD_
-    A_KEY {} -> AM_A_KEY_
-    A_USE {} -> AM_A_USE_
-    A_DEL {} -> AM_A_DEL_
+    SWITCH {} -> AM_SWITCH_
+    KEYS {} -> AM_KEYS_
+    USE {} -> AM_USE_
 
 data APrivHeader = APrivHeader
   { -- | sequential ID assigned by the sending agent
@@ -498,10 +494,9 @@ data AMsgType
   = HELLO_
   | REPLY_
   | A_MSG_
-  | A_ADD_
-  | A_KEY_
-  | A_USE_
-  | A_DEL_
+  | SWITCH_
+  | KEYS_
+  | USE_
   deriving (Eq)
 
 instance Encoding AMsgType where
@@ -509,19 +504,17 @@ instance Encoding AMsgType where
     HELLO_ -> "H"
     REPLY_ -> "R"
     A_MSG_ -> "M"
-    A_ADD_ -> "A"
-    A_KEY_ -> "K"
-    A_USE_ -> "U"
-    A_DEL_ -> "X"
+    SWITCH_ -> "S"
+    KEYS_ -> "K"
+    USE_ -> "U"
   smpP =
     smpP >>= \case
       'H' -> pure HELLO_
       'R' -> pure REPLY_
       'M' -> pure A_MSG_
-      'A' -> pure A_ADD_
-      'K' -> pure A_KEY_
-      'U' -> pure A_USE_
-      'X' -> pure A_DEL_
+      'S' -> pure SWITCH_
+      'K' -> pure KEYS_
+      'U' -> pure USE_
       _ -> fail "bad AMsgType"
 
 -- | Messages sent between SMP agents once SMP queue is secured.
@@ -534,14 +527,12 @@ data AMessage
     REPLY (L.NonEmpty SMPQueueInfo)
   | -- | agent envelope for the client message
     A_MSG MsgBody
-  | -- add queue to connection
-    A_ADD SMPQueueInfo
-  | -- key to secure the added queue and agree e2e encryption key
-    A_KEY SMPQueueInfo SndPublicVerifyKey
-  | -- inform that the queue is ready to use
-    A_USE SMPServer SMP.SenderId
-  | -- inform that the queue will be deleted
-    A_DEL SMPServer SMP.SenderId
+  | -- switch the queue to another
+    SWITCH (SMPServer, SMP.SenderId) SMPQueueInfo
+  | -- server key to secure the new queue and DH key to agree e2e encryption
+    KEYS SndPublicVerifyKey SMPQueueInfo
+  | -- inform the sender that the queue is ready to use
+    USE (SMPServer, SMP.SenderId)
   deriving (Show)
 
 instance Encoding AMessage where
@@ -549,20 +540,18 @@ instance Encoding AMessage where
     HELLO -> smpEncode HELLO_
     REPLY smpQueues -> smpEncode (REPLY_, smpQueues)
     A_MSG body -> smpEncode (A_MSG_, Tail body)
-    A_ADD qInfo -> smpEncode (A_ADD_, qInfo)
-    A_KEY qInfo sKey -> smpEncode (A_KEY_, qInfo, sKey)
-    A_USE srv sndId -> smpEncode (A_USE_, srv, sndId)
-    A_DEL srv sndId -> smpEncode (A_DEL_, srv, sndId)
+    SWITCH addr qInfo -> smpEncode (SWITCH_, addr, qInfo)
+    KEYS qInfo sKey -> smpEncode (KEYS_, qInfo, sKey)
+    USE addr -> smpEncode (USE_, addr)
   smpP =
     smpP
       >>= \case
         HELLO_ -> pure HELLO
         REPLY_ -> REPLY <$> smpP
         A_MSG_ -> A_MSG . unTail <$> smpP
-        A_ADD_ -> A_ADD <$> smpP
-        A_KEY_ -> A_KEY <$> smpP <*> smpP
-        A_USE_ -> A_USE <$> smpP <*> smpP
-        A_DEL_ -> A_DEL <$> smpP <*> smpP
+        SWITCH_ -> SWITCH <$> smpP <*> smpP
+        KEYS_ -> KEYS <$> smpP <*> smpP
+        USE_ -> USE <$> smpP
 
 instance forall m. ConnectionModeI m => StrEncoding (ConnectionRequestUri m) where
   strEncode = \case