fix encoding, most tests

src/Simplex/Messaging/Protocol.hs

@@ -270,17 +270,17 @@ isAuthNone = \case
   _ -> False
 
 -- this encoding is backwards compatible with v6 that used Maybe C.ASignature instead of TAuthorization
-instance Encoding TransmissionAuth where
-  smpEncode = \case
-    TAuthNone -> "0" -- same as Nothing
-    TAuthSignature s -> smpEncode ('1', C.signatureBytes s) -- same as Just
-    TAuthEncHash s -> smpEncode ('2', s)
-  smpP =
-    A.anyChar >>= \case
-      '0' -> pure TAuthNone
-      '1' -> fmap TAuthSignature . C.decodeSignature <$?> smpP
-      '2' -> TAuthEncHash <$> smpP
-      _ -> fail "bad TransmissionAuth tag"
+tAuthBytes :: TransmissionAuth -> ByteString
+tAuthBytes = \case
+  TAuthNone -> ""
+  TAuthSignature s -> C.signatureBytes s
+  TAuthEncHash s -> s
+
+decodeTAuthBytes :: ByteString -> Either String TransmissionAuth
+decodeTAuthBytes s
+  | B.null s = Right TAuthNone
+  | B.length s == 64 + 16 = Right $ TAuthEncHash s
+  | otherwise = TAuthSignature <$> C.decodeSignature s
 
 instance IsString TransmissionAuth where
   fromString = parseString $ B64.decode >=> C.decodeSignature >=> pure . maybe TAuthNone TAuthSignature
@@ -1375,7 +1375,7 @@ batchTransmissions' batch bSize
         b = B.concat $ B.singleton (lenEncode n) : ss
 
 tEncode :: SentRawTransmission -> ByteString
-tEncode (auth, t) = smpEncode auth <> t
+tEncode (auth, t) = smpEncode (tAuthBytes auth) <> t
 {-# INLINE tEncode #-}
 
 tEncodeForBatch :: SentRawTransmission -> ByteString
@@ -1415,7 +1415,7 @@ tDecodeParseValidate :: forall err cmd. ProtocolEncoding err cmd => SessionId ->
 tDecodeParseValidate sessionId v = \case
   Right RawTransmission {authorization, authorized, sessId, corrId, entityId, command}
     | sessId == sessionId ->
-        let decodedTransmission = (,corrId,entityId,command) <$> smpDecode authorization
+        let decodedTransmission = (,corrId,entityId,command) <$> decodeTAuthBytes authorization
          in either (const $ tError corrId) (tParseValidate authorized) decodedTransmission
     | otherwise -> (TAuthNone, "", (CorrId corrId, "", Left $ fromProtocolError @err @cmd PESession))
   Left _ -> tError ""

src/Simplex/Messaging/Transport.hs

@@ -71,6 +71,7 @@ import Control.Monad.Except
 import Control.Monad.Trans.Except (throwE)
 import qualified Data.Aeson.TH as J
 import Data.Attoparsec.ByteString.Char8 (Parser)
+import qualified Data.Attoparsec.ByteString.Char8 as A
 import Data.Bifunctor (first)
 import Data.Bitraversable (bimapM)
 import Data.ByteString.Char8 (ByteString)
@@ -78,7 +79,6 @@ import qualified Data.ByteString.Char8 as B
 import qualified Data.ByteString.Lazy.Char8 as LB
 import Data.Default (def)
 import Data.Functor (($>))
-import Data.Maybe (fromMaybe)
 import Data.Version (showVersion)
 import GHC.IO.Handle.Internals (ioe_EOF)
 import Network.Socket
@@ -303,7 +303,7 @@ data ClientHandshake = ClientHandshake
 
 instance Encoding ClientHandshake where
   smpEncode ClientHandshake {smpVersion, keyHash, authPubKey} =
-    smpEncode (smpVersion, keyHash, fromMaybe "" authPubKey)
+    smpEncode (smpVersion, keyHash) <> encodeAuthPubKey smpVersion authPubKey
   smpP = do
     (smpVersion, keyHash) <- smpP
     -- TODO drop SMP v6: remove special parser and make key non-optional
@@ -312,7 +312,7 @@ instance Encoding ClientHandshake where
 
 instance Encoding ServerHandshake where
   smpEncode ServerHandshake {smpVersionRange, sessionId, authPubKey} =
-    smpEncode (smpVersionRange, sessionId, fromMaybe "" authPubKey)
+    smpEncode (smpVersionRange, sessionId) <> encodeAuthPubKey (maxVersion smpVersionRange) authPubKey
   smpP = do
     (smpVersionRange, sessionId) <- smpP
     -- TODO drop SMP v6: remove special parser and make key non-optional
@@ -322,6 +322,11 @@ instance Encoding ServerHandshake where
 authPubKeyP :: Version -> Parser (Maybe C.PublicKeyX25519)
 authPubKeyP v = if v >= authEncryptCmdsSMPVersion then Just <$> smpP else pure Nothing
 
+encodeAuthPubKey :: Version -> Maybe C.PublicKeyX25519 -> ByteString
+encodeAuthPubKey v k
+  | v >= authEncryptCmdsSMPVersion = maybe "" smpEncode k
+  | otherwise = ""
+
 -- | Error of SMP encrypted transport over TCP.
 data TransportError
   = -- | error parsing transport block