support for unknown message content types (#395)

* android: parse/serialize unknown chat items

* ios: more resilient decoding of MsgContent

* core: preserve JSON of unknown message content type in MCUknown, so it can be parsed once it is supported by the client

apps/android/app/build.gradle

@@ -45,6 +45,7 @@ android {
         freeCompilerArgs += "-opt-in=androidx.compose.material.ExperimentalMaterialApi"
         freeCompilerArgs += "-opt-in=com.google.accompanist.insets.ExperimentalAnimatedInsets"
         freeCompilerArgs += "-opt-in=com.google.accompanist.permissions.ExperimentalPermissionsApi"
+        freeCompilerArgs += "-opt-in=kotlinx.serialization.InternalSerializationApi"
     }
     externalNativeBuild {
         cmake {

apps/android/app/src/main/java/chat/simplex/app/model/ChatModel.kt

@@ -10,8 +10,12 @@ import androidx.compose.ui.text.style.TextDecoration
 import chat.simplex.app.ui.theme.SecretColor
 import chat.simplex.app.ui.theme.SimplexBlue
 import kotlinx.datetime.*
-import kotlinx.serialization.SerialName
-import kotlinx.serialization.Serializable
+import kotlinx.serialization.*
+import kotlinx.serialization.builtins.IntArraySerializer
+import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.encoding.*
+import kotlinx.serialization.json.*
+import kotlinx.serialization.modules.SerializersModule
 
 class ChatModel(val controller: ChatController) {
   var currentUser = mutableStateOf<User?>(null)
@@ -587,14 +591,57 @@ sealed class CIContent {
   }
 }
 
-@Serializable
+@Serializable(with = MsgContentSerializer::class)
 sealed class MsgContent {
   abstract val text: String
-  abstract val cmdString: String
 
-  @Serializable @SerialName("text")
-  class MCText(override val text: String): MsgContent() {
-    override val cmdString get() = "text $text"
+  class MCText(override val text: String): MsgContent()
+  class MCUnknown(val type: String? = null, override val text: String, val json: JsonElement): MsgContent()
+
+  val cmdString: String get() = when (this) {
+    is MCText -> "text $text"
+    is MCUnknown -> "json $json"
+  }
+}
+
+object MsgContentSerializer : KSerializer<MsgContent> {
+  override val descriptor: SerialDescriptor = buildSerialDescriptor("MsgContent", PolymorphicKind.SEALED) {
+    element("MCText", buildClassSerialDescriptor("MCText") {
+      element<String>("text")
+    })
+    element("MCUnknown", buildClassSerialDescriptor("MCUnknown"))
+  }
+
+  override fun deserialize(decoder: Decoder): MsgContent {
+    require(decoder is JsonDecoder)
+    val json = decoder.decodeJsonElement()
+    return if (json is JsonObject) {
+      if ("type" in json) {
+        val t = json["type"]?.jsonPrimitive?.content ?: ""
+        val text = json["text"]?.jsonPrimitive?.content ?: "unknown message format"
+        when (t) {
+          "text" -> MsgContent.MCText(text)
+          else -> MsgContent.MCUnknown(t, text, json)
+        }
+      } else {
+        MsgContent.MCUnknown(text = "invalid message format", json = json)
+      }
+    } else {
+      MsgContent.MCUnknown(text = "invalid message format", json = json)
+    }
+  }
+
+  override fun serialize(encoder: Encoder, value: MsgContent) {
+    require(encoder is JsonEncoder)
+    val json = when (value) {
+      is MsgContent.MCText ->
+        buildJsonObject {
+          put("type", "text")
+          put("text", value.text)
+        }
+      is MsgContent.MCUnknown -> value.json
+    }
+    encoder.encodeJsonElement(json)
   }
 }
 

apps/ios/Shared/Model/ChatModel.swift

@@ -625,15 +625,14 @@ struct RcvFileTransfer: Decodable {
 
 enum MsgContent {
     case text(String)
+    // TODO include original JSON, possibly using https://github.com/zoul/generic-json-swift
     case unknown(type: String, text: String)
-    case invalid(error: String)
 
     var text: String {
         get {
             switch self {
             case let .text(text): return text
             case let .unknown(_, text): return text
-            case .invalid:  return "invalid"
             }
         }
     }
@@ -655,8 +654,8 @@ enum MsgContent {
 
 extension MsgContent: Decodable {
     init(from decoder: Decoder) throws {
-        let container = try decoder.container(keyedBy: CodingKeys.self)
         do {
+            let container = try decoder.container(keyedBy: CodingKeys.self)
             let type = try container.decode(String.self, forKey: CodingKeys.type)
             switch type {
             case "text":
@@ -667,7 +666,7 @@ extension MsgContent: Decodable {
                 self = .unknown(type: type, text: text ?? "unknown message format")
             }
         } catch {
-            self = .invalid(error: String(describing: error))
+            self = .unknown(type: "unknown", text: "invalid message format")
         }
     }
 }

src/Simplex/Chat/Protocol.hs

@@ -15,10 +15,12 @@ module Simplex.Chat.Protocol where
 import Control.Monad ((<=<))
 import Data.Aeson (FromJSON, ToJSON, (.:), (.:?), (.=))
 import qualified Data.Aeson as J
+import qualified Data.Aeson.Encoding as JE
 import qualified Data.Aeson.KeyMap as JM
 import qualified Data.Aeson.Types as JT
 import qualified Data.Attoparsec.ByteString.Char8 as A
 import qualified Data.ByteString.Lazy.Char8 as LB
+import Data.Maybe (fromMaybe)
 import Data.Text (Text)
 import Data.Text.Encoding (decodeLatin1, encodeUtf8)
 import Database.SQLite.Simple.FromField (FromField (..))
@@ -107,26 +109,24 @@ instance ToJSON MsgContentType where
   toJSON = strToJSON
   toEncoding = strToJEncoding
 
--- TODO - include tag and original JSON into MCUnknown so that information is not lost
--- so when it serializes back it is the same as it was and chat upgrade makes it readable
-data MsgContent = MCText Text | MCUnknown
+data MsgContent = MCText Text | MCUnknown J.Value Text
   deriving (Eq, Show)
 
 msgContentText :: MsgContent -> Text
 msgContentText = \case
   MCText t -> t
-  MCUnknown -> unknownMsgType
+  MCUnknown _ t -> t
 
 toMsgContentType :: MsgContent -> MsgContentType
 toMsgContentType = \case
   MCText _ -> MCText_
-  MCUnknown -> MCUnknown_
+  MCUnknown {} -> MCUnknown_
 
 instance FromJSON MsgContent where
-  parseJSON (J.Object v) = do
+  parseJSON jv@(J.Object v) = do
     v .: "type" >>= \case
       MCText_ -> MCText <$> v .: "text"
-      MCUnknown_ -> pure MCUnknown
+      MCUnknown_ -> MCUnknown jv . fromMaybe unknownMsgType <$> v .:? "text"
   parseJSON invalid =
     JT.prependFailure "bad MsgContent, " (JT.typeMismatch "Object" invalid)
 
@@ -134,16 +134,12 @@ unknownMsgType :: Text
 unknownMsgType = "unknown message type"
 
 instance ToJSON MsgContent where
-  toJSON mc =
-    J.object $
-      ("type" .= toMsgContentType mc) : case mc of
-        MCText t -> ["text" .= t]
-        MCUnknown -> ["text" .= unknownMsgType]
-  toEncoding mc =
-    J.pairs $
-      ("type" .= toMsgContentType mc) <> case mc of
-        MCText t -> "text" .= t
-        MCUnknown -> "text" .= unknownMsgType
+  toJSON = \case
+    MCUnknown v _ -> v
+    MCText t -> J.object ["type" .= MCText_, "text" .= t]
+  toEncoding = \case
+    MCUnknown v _ -> JE.value v
+    MCText t -> J.pairs $ "type" .= MCText_ <> "text" .= t
 
 data CMEventTag
   = XMsgNew_

src/Simplex/Chat/View.hs

@@ -371,7 +371,7 @@ ttyMsgTime = styleTime . formatTime defaultTimeLocale "%H:%M"
 ttyMsgContent :: MsgContent -> [StyledString]
 ttyMsgContent = \case
   MCText t -> msgPlain t
-  MCUnknown -> ["unknown message type"]
+  MCUnknown _ t -> msgPlain t
 
 ttySentFile :: StyledString -> FileTransferId -> FilePath -> [StyledString]
 ttySentFile to fId fPath = ["/f " <> to <> ttyFilePath fPath, "use " <> highlight ("/fc " <> show fId) <> " to cancel sending"]