core: autoincrement database IDs (#7766)

* core: autoincrement database IDs

* migration

* update simplexmq

---------

Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>

src/Simplex/Chat/Store/SQLite/Migrations.hs

@@ -180,6 +180,7 @@ import Simplex.Chat.Store.SQLite.Migrations.M20260925_badge_bindings
 import Simplex.Chat.Store.SQLite.Migrations.M20261010_invite_member_via_group
 import Simplex.Chat.Store.SQLite.Migrations.M20261012_delivery_job_notification
 import Simplex.Chat.Store.SQLite.Migrations.M20261016_badge_store_receipts
+import Simplex.Chat.Store.SQLite.Migrations.M20261018_autoincrement_ids
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))
 
 schemaMigrations :: [(String, Query, Maybe Query)]
@@ -359,7 +360,8 @@ schemaMigrations =
     ("20260925_badge_bindings", m20260925_badge_bindings, Just down_m20260925_badge_bindings),
     ("20261010_invite_member_via_group", m20261010_invite_member_via_group, Just down_m20261010_invite_member_via_group),
     ("20261012_delivery_job_notification", m20261012_delivery_job_notification, Just down_m20261012_delivery_job_notification),
-    ("20261016_badge_store_receipts", m20261016_badge_store_receipts, Just down_m20261016_badge_store_receipts)
+    ("20261016_badge_store_receipts", m20261016_badge_store_receipts, Just down_m20261016_badge_store_receipts),
+    ("20261018_autoincrement_ids", m20261018_autoincrement_ids, Just down_m20261018_autoincrement_ids)
   ]
 
 -- | The list of migrations in ascending order by date

src/Simplex/Chat/Store/SQLite/Migrations/M20261018_autoincrement_ids.hs

@@ -0,0 +1,110 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Chat.Store.SQLite.Migrations.M20261018_autoincrement_ids where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20261018_autoincrement_ids :: Query
+m20261018_autoincrement_ids =
+  [sql|
+INSERT INTO sqlite_sequence (name, seq)
+SELECT 'users', MAX(ROWID) FROM users;
+
+INSERT INTO sqlite_sequence (name, seq)
+SELECT 'contacts', MAX(ROWID) FROM contacts;
+
+INSERT INTO sqlite_sequence (name, seq)
+SELECT 'groups', MAX(ROWID) FROM groups;
+
+INSERT INTO sqlite_sequence (name, seq)
+SELECT 'group_members', MAX(ROWID) FROM group_members;
+
+INSERT INTO sqlite_sequence (name, seq)
+SELECT 'files', MAX(ROWID) FROM files;
+
+INSERT INTO sqlite_sequence (name, seq)
+SELECT 'connections', MAX(ROWID) FROM connections;
+
+INSERT INTO sqlite_sequence (name, seq)
+SELECT 'contact_requests', MAX(ROWID) FROM contact_requests;
+
+INSERT INTO sqlite_sequence (name, seq)
+SELECT 'rcv_roster_transfers', MAX(ROWID) FROM rcv_roster_transfers;
+
+PRAGMA writable_schema=1;
+
+UPDATE sqlite_master SET sql = replace(sql, 'user_id INTEGER PRIMARY KEY,', 'user_id INTEGER PRIMARY KEY AUTOINCREMENT,')
+WHERE name = 'users' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'contact_id INTEGER PRIMARY KEY,', 'contact_id INTEGER PRIMARY KEY AUTOINCREMENT,')
+WHERE name = 'contacts' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'group_id INTEGER PRIMARY KEY,', 'group_id INTEGER PRIMARY KEY AUTOINCREMENT,')
+WHERE name = 'groups' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'group_member_id INTEGER PRIMARY KEY,', 'group_member_id INTEGER PRIMARY KEY AUTOINCREMENT,')
+WHERE name = 'group_members' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'file_id INTEGER PRIMARY KEY,', 'file_id INTEGER PRIMARY KEY AUTOINCREMENT,')
+WHERE name = 'files' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'connection_id INTEGER PRIMARY KEY,', 'connection_id INTEGER PRIMARY KEY AUTOINCREMENT,')
+WHERE name = 'connections' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'contact_request_id INTEGER PRIMARY KEY,', 'contact_request_id INTEGER PRIMARY KEY AUTOINCREMENT,')
+WHERE name = 'contact_requests' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'roster_transfer_id INTEGER PRIMARY KEY,', 'roster_transfer_id INTEGER PRIMARY KEY AUTOINCREMENT,')
+WHERE name = 'rcv_roster_transfers' AND type = 'table';
+
+PRAGMA writable_schema=RESET;
+|]
+
+down_m20261018_autoincrement_ids :: Query
+down_m20261018_autoincrement_ids =
+  [sql|
+DELETE FROM sqlite_sequence WHERE name = 'users';
+
+DELETE FROM sqlite_sequence WHERE name = 'contacts';
+
+DELETE FROM sqlite_sequence WHERE name = 'groups';
+
+DELETE FROM sqlite_sequence WHERE name = 'group_members';
+
+DELETE FROM sqlite_sequence WHERE name = 'files';
+
+DELETE FROM sqlite_sequence WHERE name = 'connections';
+
+DELETE FROM sqlite_sequence WHERE name = 'contact_requests';
+
+DELETE FROM sqlite_sequence WHERE name = 'rcv_roster_transfers';
+
+PRAGMA writable_schema=1;
+
+UPDATE sqlite_master SET sql = replace(sql, 'user_id INTEGER PRIMARY KEY AUTOINCREMENT,', 'user_id INTEGER PRIMARY KEY,')
+WHERE name = 'users' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'contact_id INTEGER PRIMARY KEY AUTOINCREMENT,', 'contact_id INTEGER PRIMARY KEY,')
+WHERE name = 'contacts' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'group_id INTEGER PRIMARY KEY AUTOINCREMENT,', 'group_id INTEGER PRIMARY KEY,')
+WHERE name = 'groups' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'group_member_id INTEGER PRIMARY KEY AUTOINCREMENT,', 'group_member_id INTEGER PRIMARY KEY,')
+WHERE name = 'group_members' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'file_id INTEGER PRIMARY KEY AUTOINCREMENT,', 'file_id INTEGER PRIMARY KEY,')
+WHERE name = 'files' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'connection_id INTEGER PRIMARY KEY AUTOINCREMENT,', 'connection_id INTEGER PRIMARY KEY,')
+WHERE name = 'connections' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'contact_request_id INTEGER PRIMARY KEY AUTOINCREMENT,', 'contact_request_id INTEGER PRIMARY KEY,')
+WHERE name = 'contact_requests' AND type = 'table';
+
+UPDATE sqlite_master SET sql = replace(sql, 'roster_transfer_id INTEGER PRIMARY KEY AUTOINCREMENT,', 'roster_transfer_id INTEGER PRIMARY KEY,')
+WHERE name = 'rcv_roster_transfers' AND type = 'table';
+
+PRAGMA writable_schema=RESET;
+|]

src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt

@@ -614,6 +614,15 @@ Query:
 Plan:
 SEARCH address_ratchet_keys USING INDEX idx_address_ratchet_keys (conn_id=? AND ratchet_key_id=?)
 
+Query: 
+        UPDATE rcv_messages
+        SET receive_attempts = receive_attempts + 1
+        WHERE conn_id = ? AND internal_id = ?
+        RETURNING receive_attempts
+      
+Plan:
+SEARCH rcv_messages USING COVERING INDEX idx_rcv_messages_conn_id_internal_id (conn_id=? AND internal_id=?)
+
 Query: 
       DELETE FROM address_ratchet_keys
       WHERE conn_id = ?
@@ -1526,6 +1535,10 @@ Query: UPDATE ratchets SET ratchet_state = ? WHERE conn_id = ?
 Plan:
 SEARCH ratchets USING PRIMARY KEY (conn_id=?)
 
+Query: UPDATE rcv_file_chunk_replicas SET delay = ?, retries = retries + 1, updated_at = ? WHERE rcv_file_chunk_replica_id = ?
+Plan:
+SEARCH rcv_file_chunk_replicas USING INTEGER PRIMARY KEY (rowid=?)
+
 Query: UPDATE rcv_file_chunk_replicas SET received = 1, updated_at = ? WHERE rcv_file_chunk_replica_id = ?
 Plan:
 SEARCH rcv_file_chunk_replicas USING INTEGER PRIMARY KEY (rowid=?)

src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql

@@ -36,7 +36,7 @@ CREATE TABLE contact_profiles(
   preferences_json TEXT
 ) STRICT;
 CREATE TABLE users(
-  user_id INTEGER PRIMARY KEY,
+  user_id INTEGER PRIMARY KEY AUTOINCREMENT,
   contact_id INTEGER NOT NULL UNIQUE REFERENCES contacts ON DELETE RESTRICT
   DEFERRABLE INITIALLY DEFERRED,
   local_display_name TEXT NOT NULL UNIQUE,
@@ -63,6 +63,7 @@ CREATE TABLE users(
   ON UPDATE CASCADE
   DEFERRABLE INITIALLY DEFERRED
 ) STRICT;
+CREATE TABLE sqlite_sequence(name,seq);
 CREATE TABLE display_names(
   user_id INTEGER NOT NULL REFERENCES users ON DELETE CASCADE,
   local_display_name TEXT NOT NULL,
@@ -74,7 +75,7 @@ CREATE TABLE display_names(
   UNIQUE(user_id, ldn_base, ldn_suffix) ON CONFLICT FAIL
 ) WITHOUT ROWID, STRICT;
 CREATE TABLE contacts(
-  contact_id INTEGER PRIMARY KEY,
+  contact_id INTEGER PRIMARY KEY AUTOINCREMENT,
   contact_profile_id INTEGER REFERENCES contact_profiles ON DELETE SET NULL,
   user_id INTEGER NOT NULL REFERENCES users ON DELETE CASCADE,
   local_display_name TEXT NOT NULL,
@@ -151,7 +152,7 @@ CREATE TABLE group_profiles(
   preferences_json TEXT
 ) STRICT;
 CREATE TABLE groups(
-  group_id INTEGER PRIMARY KEY, -- local group ID
+  group_id INTEGER PRIMARY KEY AUTOINCREMENT, -- local group ID
   user_id INTEGER NOT NULL REFERENCES users ON DELETE CASCADE,
   local_display_name TEXT NOT NULL, -- local group name without spaces
   group_profile_id INTEGER REFERENCES group_profiles ON DELETE SET NULL, -- shared group profile
@@ -222,7 +223,7 @@ CREATE TABLE groups(
 ) STRICT;
 CREATE TABLE group_members(
   -- group members, excluding the local user
-  group_member_id INTEGER PRIMARY KEY,
+  group_member_id INTEGER PRIMARY KEY AUTOINCREMENT,
   group_id INTEGER NOT NULL REFERENCES groups ON DELETE CASCADE,
   member_id BLOB NOT NULL, -- shared member ID, unique per group
   member_role TEXT NOT NULL, -- owner, admin, member
@@ -280,7 +281,7 @@ CREATE TABLE group_member_intros(
   UNIQUE(re_group_member_id, to_group_member_id)
 ) STRICT;
 CREATE TABLE files(
-  file_id INTEGER PRIMARY KEY,
+  file_id INTEGER PRIMARY KEY AUTOINCREMENT,
   contact_id INTEGER REFERENCES contacts ON DELETE CASCADE,
   group_id INTEGER REFERENCES groups ON DELETE CASCADE,
   file_name TEXT NOT NULL,
@@ -352,7 +353,7 @@ CREATE TABLE rcv_file_chunks(
 ) WITHOUT ROWID, STRICT;
 CREATE TABLE connections(
   -- all SMP agent connections
-  connection_id INTEGER PRIMARY KEY,
+  connection_id INTEGER PRIMARY KEY AUTOINCREMENT,
   agent_conn_id BLOB NOT NULL UNIQUE,
   conn_level INTEGER NOT NULL DEFAULT 0,
   via_contact INTEGER REFERENCES contacts(contact_id) ON DELETE SET NULL,
@@ -419,7 +420,7 @@ CREATE TABLE user_contact_links(
   UNIQUE(user_id, local_display_name)
 ) STRICT;
 CREATE TABLE contact_requests(
-  contact_request_id INTEGER PRIMARY KEY,
+  contact_request_id INTEGER PRIMARY KEY AUTOINCREMENT,
   user_contact_link_id INTEGER REFERENCES user_contact_links
   ON UPDATE CASCADE ON DELETE SET NULL,
   agent_invitation_id BLOB NOT NULL,
@@ -530,7 +531,6 @@ CREATE TABLE chat_items(
   fwd_from_member_id BLOB,
   fwd_from_shared_msg_id BLOB
 ) STRICT;
-CREATE TABLE sqlite_sequence(name,seq);
 CREATE TABLE chat_item_messages(
   chat_item_id INTEGER NOT NULL REFERENCES chat_items ON DELETE CASCADE,
   message_id INTEGER NOT NULL UNIQUE REFERENCES messages ON DELETE CASCADE,
@@ -848,7 +848,7 @@ CREATE TABLE group_relays(
   base_web_url TEXT
 ) STRICT;
 CREATE TABLE rcv_roster_transfers(
-  roster_transfer_id INTEGER PRIMARY KEY,
+  roster_transfer_id INTEGER PRIMARY KEY AUTOINCREMENT,
   group_id INTEGER NOT NULL REFERENCES groups ON DELETE CASCADE,
   from_member_id INTEGER NOT NULL REFERENCES group_members ON DELETE CASCADE,
   roster_version INTEGER NOT NULL,