core: support batch sending in groups, batch introductions; send recent message history to new members (#3519)

* core: batch send stubs, comments

* multiple events in ChatMessage and supporting types

* Revert "multiple events in ChatMessage and supporting types"

This reverts commit 9b239b26ba.

* schema, refactor group processing for batched messages

* encoding, refactor processing

* refactor code to work with updated schema

* encoding, remove instances

* wip

* implement batching

* batch introductions

* wip

* collect and send message history

* missing new line

* rename

* test

* rework to build history via chat items

* refactor, tests

* correctly set member version range, dont include deleted items

* tests

* fix disappearing messages

* check number of errors

* comment

* check size in encodeChatMessage

* fix - don't check msg size for binary

* use builder

* rename

* rename

* rework batching

* lazy msg body

* use withStoreBatch

* refactor

* reverse batches

* comment

* possibly fix builder for single msg

* refactor batcher

* refactor

* dont repopulate msg_deliveries on down migration

* EncodedChatMessage type

* remove type

* batcher tests

* add tests

* group history preference

* test group link

* fix tests

* fix for random update

* add test testImageFitsSingleBatch

* refactor

* rename function

* refactor

* mconcat

* rename feature

* catch error on each batch

* refactor file inv retrieval

* refactor gathering item forward events

* refactor message batching

* unite migrations

* move files

* refactor

* Revert "unite migrations"

This reverts commit 0be7a3117a.

* refactor splitFileDescr

* improve tests

* Revert "dont repopulate msg_deliveries on down migration"

This reverts commit 2944c1cc28.

* fix down migration

---------

Co-authored-by: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com>

src/Simplex/Chat/Protocol.hs

@@ -29,7 +29,9 @@ import qualified Data.Attoparsec.ByteString.Char8 as A
 import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
 import Data.ByteString.Internal (c2w, w2c)
+import qualified Data.ByteString.Lazy as L
 import qualified Data.ByteString.Lazy.Char8 as LB
+import Data.Int (Int64)
 import Data.Maybe (fromMaybe)
 import Data.String
 import Data.Text (Text)
@@ -51,7 +53,7 @@ import Simplex.Messaging.Util (eitherToMaybe, safeDecodeUtf8, (<$?>))
 import Simplex.Messaging.Version hiding (version)
 
 currentChatVersion :: Version
-currentChatVersion = 4
+currentChatVersion = 5
 
 supportedChatVRange :: VersionRange
 supportedChatVRange = mkVersionRange 1 currentChatVersion
@@ -72,6 +74,10 @@ groupLinkNoContactVRange = mkVersionRange 3 currentChatVersion
 groupForwardVRange :: VersionRange
 groupForwardVRange = mkVersionRange 4 currentChatVersion
 
+-- version range that supports batch sending in groups
+batchSendVRange :: VersionRange
+batchSendVRange = mkVersionRange 5 currentChatVersion
+
 data ConnectionEntity
   = RcvDirectMsgConnection {entityConnection :: Connection, contact :: Maybe Contact}
   | RcvGroupMsgConnection {entityConnection :: Connection, groupInfo :: GroupInfo, groupMember :: GroupMember}
@@ -447,6 +453,18 @@ durationText duration =
       | n <= 9 = '0' : show n
       | otherwise = show n
 
+msgContentHasText :: MsgContent -> Bool
+msgContentHasText = \case
+  MCText t -> hasText t
+  MCLink {text} -> hasText text
+  MCImage {text} -> hasText text
+  MCVideo {text} -> hasText text
+  MCVoice {text} -> hasText text
+  MCFile t -> hasText t
+  MCUnknown {text} -> hasText text
+  where
+    hasText = not . T.null
+
 isVoice :: MsgContent -> Bool
 isVoice = \case
   MCVoice {} -> True
@@ -467,18 +485,34 @@ data ExtMsgContent = ExtMsgContent {content :: MsgContent, file :: Maybe FileInv
 
 $(JQ.deriveJSON defaultJSON ''QuotedMsg)
 
-instance MsgEncodingI e => StrEncoding (ChatMessage e) where
-  strEncode msg = case chatToAppMessage msg of
-    AMJson m -> LB.toStrict $ J.encode m
-    AMBinary m -> strEncode m
-  strP = (\(ACMsg _ m) -> checkEncoding m) <$?> strP
+-- this limit reserves space for metadata in forwarded messages
+-- 15780 (limit used for fileChunkSize) - 161 (x.grp.msg.forward overhead) = 15619, round to 15610
+maxChatMsgSize :: Int64
+maxChatMsgSize = 15610
 
-instance StrEncoding AChatMessage where
-  strEncode (ACMsg _ m) = strEncode m
-  strP =
-    A.peekChar' >>= \case
-      '{' -> ACMsg SJson <$> ((appJsonToCM <=< J.eitherDecodeStrict') <$?> A.takeByteString)
-      _ -> ACMsg SBinary <$> (appBinaryToCM <$?> strP)
+data EncodedChatMessage = ECMEncoded L.ByteString | ECMLarge
+
+encodeChatMessage :: MsgEncodingI e => ChatMessage e -> EncodedChatMessage
+encodeChatMessage msg = do
+  case chatToAppMessage msg of
+    AMJson m -> do
+      let body = J.encode m
+      if LB.length body > maxChatMsgSize
+        then ECMLarge
+        else ECMEncoded body
+    AMBinary m -> ECMEncoded . LB.fromStrict $ strEncode m
+
+parseChatMessages :: ByteString -> [Either String AChatMessage]
+parseChatMessages "" = [Left "empty string"]
+parseChatMessages s = case B.head s of
+  '{' -> [ACMsg SJson <$> J.eitherDecodeStrict' s]
+  '[' -> case J.eitherDecodeStrict' s of
+    Right v -> map parseItem v
+    Left e -> [Left e]
+  _ -> [ACMsg SBinary <$> (appBinaryToCM =<< strDecode s)]
+  where
+    parseItem :: J.Value -> Either String AChatMessage
+    parseItem v = ACMsg SJson <$> JT.parseEither parseJSON v
 
 parseMsgContainer :: J.Object -> JT.Parser MsgContainer
 parseMsgContainer v =