schema, refactor group processing for batched messages

src/Simplex/Chat/Migrations/M20231206_recreate_msg_deliveries.hs

@@ -0,0 +1,85 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Chat.Migrations.M20231206_recreate_msg_deliveries where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20231206_recreate_msg_deliveries :: Query
+m20231206_recreate_msg_deliveries =
+  [sql|
+DROP INDEX msg_delivery_events_msg_delivery_id;
+DROP TABLE msg_delivery_events;
+
+DROP INDEX idx_msg_deliveries_message_id;
+DROP INDEX idx_msg_deliveries_agent_ack_cmd_id;
+
+CREATE TABLE new_msg_deliveries(
+  msg_delivery_id INTEGER PRIMARY KEY,
+  message_id INTEGER NOT NULL REFERENCES messages ON DELETE CASCADE, -- non UNIQUE for group messages
+  connection_id INTEGER NOT NULL REFERENCES connections ON DELETE CASCADE,
+  agent_msg_id INTEGER, -- internal agent message ID (NULL while pending), non UNIQUE for batched messages
+  agent_msg_meta TEXT, -- JSON with timestamps etc. sent in MSG, NULL for sent
+  chat_ts TEXT NOT NULL DEFAULT(datetime('now')),
+  created_at TEXT CHECK(created_at NOT NULL),
+  updated_at TEXT CHECK(updated_at NOT NULL),
+  agent_ack_cmd_id INTEGER, -- broker_ts for received, created_at for sent
+  delivery_status TEXT -- MsgDeliveryStatus
+);
+
+INSERT INTO new_msg_deliveries
+  (msg_delivery_id, message_id, connection_id, agent_msg_id, agent_msg_meta, chat_ts, created_at, updated_at, agent_ack_cmd_id)
+SELECT
+  msg_delivery_id, message_id, connection_id, agent_msg_id, agent_msg_meta, chat_ts, created_at, updated_at, agent_ack_cmd_id
+  FROM msg_deliveries;
+
+DROP TABLE msg_deliveries;
+ALTER TABLE new_msg_deliveries RENAME TO msg_deliveries;
+
+CREATE INDEX idx_msg_deliveries_message_id ON "msg_deliveries"(message_id);
+CREATE INDEX idx_msg_deliveries_agent_ack_cmd_id ON "msg_deliveries"(connection_id, agent_ack_cmd_id);
+CREATE INDEX idx_msg_deliveries_agent_msg_id ON "msg_deliveries"(connection_id, agent_msg_id);
+|]
+
+down_m20231206_recreate_msg_deliveries :: Query
+down_m20231206_recreate_msg_deliveries =
+  [sql|
+DROP INDEX idx_msg_deliveries_message_id;
+DROP INDEX idx_msg_deliveries_agent_ack_cmd_id;
+DROP INDEX idx_msg_deliveries_agent_msg_id;
+
+CREATE TABLE old_msg_deliveries(
+  msg_delivery_id INTEGER PRIMARY KEY,
+  message_id INTEGER NOT NULL REFERENCES messages ON DELETE CASCADE, -- non UNIQUE for group messages
+  connection_id INTEGER NOT NULL REFERENCES connections ON DELETE CASCADE,
+  agent_msg_id INTEGER, -- internal agent message ID(NULL while pending)
+  agent_msg_meta TEXT, -- JSON with timestamps etc. sent in MSG, NULL for sent
+  chat_ts TEXT NOT NULL DEFAULT(datetime('now')),
+  created_at TEXT CHECK(created_at NOT NULL),
+  updated_at TEXT CHECK(updated_at NOT NULL),
+  agent_ack_cmd_id INTEGER, -- broker_ts for received, created_at for sent
+  UNIQUE(connection_id, agent_msg_id)
+);
+
+INSERT INTO old_msg_deliveries
+  (msg_delivery_id, message_id, connection_id, agent_msg_id, agent_msg_meta, chat_ts, created_at, updated_at, agent_ack_cmd_id)
+SELECT
+  msg_delivery_id, message_id, connection_id, agent_msg_id, agent_msg_meta, chat_ts, created_at, updated_at, agent_ack_cmd_id
+  FROM msg_deliveries;
+
+DROP TABLE msg_deliveries;
+ALTER TABLE old_msg_deliveries RENAME TO msg_deliveries;
+
+CREATE INDEX idx_msg_deliveries_message_id ON "msg_deliveries"(message_id);
+CREATE INDEX idx_msg_deliveries_agent_ack_cmd_id ON "msg_deliveries"(connection_id, agent_ack_cmd_id);
+
+CREATE TABLE msg_delivery_events (
+  msg_delivery_event_id INTEGER PRIMARY KEY,
+  msg_delivery_id INTEGER NOT NULL REFERENCES msg_deliveries ON DELETE CASCADE, -- non UNIQUE for multiple events per msg delivery
+  delivery_status TEXT NOT NULL, -- see MsgDeliveryStatus for allowed values
+  created_at TEXT NOT NULL DEFAULT (datetime('now')),
+  updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+  UNIQUE (msg_delivery_id, delivery_status)
+);
+CREATE INDEX msg_delivery_events_msg_delivery_id ON msg_delivery_events(msg_delivery_id);
+|]

src/Simplex/Chat/Protocol.hs

@@ -471,18 +471,21 @@ data ExtMsgContent = ExtMsgContent {content :: MsgContent, file :: Maybe FileInv
 
 $(JQ.deriveJSON defaultJSON ''QuotedMsg)
 
-instance MsgEncodingI e => StrEncoding (ChatMessage e) where
+instance MsgEncodingI e => StrEncoding [ChatMessage e] where
   strEncode msg = case chatToAppMessage msg of
     AMJson m -> LB.toStrict $ J.encode m
     AMBinary m -> strEncode m
   strP = (\(ACMsg _ m) -> checkEncoding m) <$?> strP
 
-instance StrEncoding AChatMessage where
-  strEncode (ACMsg _ m) = strEncode m
+instance StrEncoding [AChatMessage] where
+  strEncode [(ACMsg _ m)] = strEncode m
   strP =
     A.peekChar' >>= \case
-      '{' -> ACMsg SJson <$> ((appJsonToCM <=< J.eitherDecodeStrict') <$?> A.takeByteString)
-      _ -> ACMsg SBinary <$> (appBinaryToCM <$?> strP)
+      '{' -> (:[]) <$> parseItem
+      -- '[' ->
+      _ -> (:[]) . ACMsg SBinary <$> (appBinaryToCM <$?> strP)
+    where
+      parseItem = ACMsg SJson <$> ((appJsonToCM <=< J.eitherDecodeStrict') <$?> A.takeByteString)
 
 parseMsgContainer :: J.Object -> JT.Parser MsgContainer
 parseMsgContainer v =

src/Simplex/Chat/Store/Migrations.hs

@@ -91,6 +91,7 @@ import Simplex.Chat.Migrations.M20231107_indexes
 import Simplex.Chat.Migrations.M20231113_group_forward
 import Simplex.Chat.Migrations.M20231114_remote_control
 import Simplex.Chat.Migrations.M20231126_remote_ctrl_address
+import Simplex.Chat.Migrations.M20231206_recreate_msg_deliveries
 import Simplex.Messaging.Agent.Store.SQLite.Migrations (Migration (..))
 
 schemaMigrations :: [(String, Query, Maybe Query)]
@@ -181,7 +182,8 @@ schemaMigrations =
     ("20231107_indexes", m20231107_indexes, Just down_m20231107_indexes),
     ("20231113_group_forward", m20231113_group_forward, Just down_m20231113_group_forward),
     ("20231114_remote_control", m20231114_remote_control, Just down_m20231114_remote_control),
-    ("20231126_remote_ctrl_address", m20231126_remote_ctrl_address, Just down_m20231126_remote_ctrl_address)
+    ("20231126_remote_ctrl_address", m20231126_remote_ctrl_address, Just down_m20231126_remote_ctrl_address),
+    ("20231206_recreate_msg_deliveries", m20231206_recreate_msg_deliveries, Just down_m20231206_recreate_msg_deliveries)
   ]
 
 -- | The list of migrations in ascending order by date