Merge branch 'master' into chat-relays

src/Simplex/Chat/Store/Postgres/Migrations.hs

@@ -21,7 +21,9 @@ import Simplex.Chat.Store.Postgres.Migrations.M20250919_group_summary
 import Simplex.Chat.Store.Postgres.Migrations.M20250922_remove_unused_connections
 import Simplex.Chat.Store.Postgres.Migrations.M20251007_connections_sync
 import Simplex.Chat.Store.Postgres.Migrations.M20251017_chat_tags_cascade
-import Simplex.Chat.Store.Postgres.Migrations.M20251018_chat_relays
+import Simplex.Chat.Store.Postgres.Migrations.M20251117_member_relations_vector
+-- import Simplex.Chat.Store.Postgres.Migrations.M20251128_member_relations_vector_stage_2
+import Simplex.Chat.Store.Postgres.Migrations.M20251212_chat_relays
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))
 
 schemaMigrations :: [(String, Text, Maybe Text)]
@@ -43,7 +45,9 @@ schemaMigrations =
     ("20250922_remove_unused_connections", m20250922_remove_unused_connections, Just down_m20250922_remove_unused_connections),
     ("20251007_connections_sync", m20251007_connections_sync, Just down_m20251007_connections_sync),
     ("20251017_chat_tags_cascade", m20251017_chat_tags_cascade, Just down_m20251017_chat_tags_cascade),
-    ("20251018_chat_relays", m20251018_chat_relays, Just down_m20251018_chat_relays)
+    ("20251117_member_relations_vector", m20251117_member_relations_vector, Just down_m20251117_member_relations_vector),
+    -- ("20251128_member_relations_vector_stage_2", m20251128_member_relations_vector_stage_2, Just down_m20251128_member_relations_vector_stage_2)
+    ("20251212_chat_relays", m20251212_chat_relays, Just down_m20251212_chat_relays)
   ]
 
 -- | The list of migrations in ascending order by date

src/Simplex/Chat/Store/Postgres/Migrations/M20251117_member_relations_vector.hs

@@ -0,0 +1,159 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Chat.Store.Postgres.Migrations.M20251117_member_relations_vector where
+
+import Data.Text (Text)
+import qualified Data.Text as T
+import Text.RawString.QQ (r)
+
+-- This migration creates custom aggregate function migrate_relations_vector(idx, direction, intro_status).
+-- Used in live migration and stage 2 migration (M20251128_member_relations_vector_stage_2).
+--
+-- Vector byte encoding: 4 reserved | 1 direction | 3 status
+-- Direction: 0 = IDSubjectIntroduced, 1 = IDReferencedIntroduced
+-- Status values: 0 = MRNew, 1 = MRIntroduced, 2 = MRSubjectConnected, 3 = MRReferencedConnected, 4 = MRConnected
+--
+-- The aggregate transforms intro_status into relation status:
+-- - intro_status 'new'/'sent'/'rcv'/'fwd': MRIntroduced (1)
+-- - intro_status 're-con': if direction=0 then MRSubjectConnected (2), else MRReferencedConnected (3)
+-- - intro_status 'to-con': if direction=0 then MRReferencedConnected (3), else MRSubjectConnected (2)
+-- - intro_status 'con': MRConnected (4)
+--
+-- Final byte combines direction and status: byte = (direction << 3) | status
+
+m20251117_member_relations_vector :: Text
+m20251117_member_relations_vector =
+  T.pack
+    [r|
+CREATE FUNCTION set_member_vector_new_relation(v BYTEA, idx BIGINT, direction INT, status INT)
+RETURNS BYTEA AS $$
+DECLARE
+  new_len INT;
+  result BYTEA;
+  byte_val INT;
+  old_byte INT;
+BEGIN
+  IF idx < 0 THEN
+    RETURN v;
+  END IF;
+  IF idx < length(v) THEN
+    old_byte := get_byte(v, idx::INT);
+  ELSE
+    old_byte := 0;
+  END IF;
+  byte_val := (old_byte & x'F0'::INT) | (direction * 8) | status;
+  new_len := GREATEST(length(v), idx + 1);
+  IF new_len > length(v) THEN
+    result := v || (SELECT string_agg('\x00'::BYTEA, ''::BYTEA) FROM generate_series(1, new_len - length(v)));
+  ELSE
+    result := v;
+  END IF;
+  result := set_byte(result, idx::INT, byte_val);
+  RETURN result;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+CREATE FUNCTION migrate_relations_vector_step(state BYTEA, idx BIGINT, direction INT, intro_status TEXT)
+RETURNS BYTEA AS $$
+DECLARE
+  new_len INT;
+  result BYTEA;
+  status INT;
+  byte_val INT;
+BEGIN
+  IF idx < 0 THEN
+    RETURN state;
+  END IF;
+  IF intro_status = 're-con' THEN
+    IF direction = 0 THEN status := 2; ELSE status := 3; END IF;
+  ELSIF intro_status = 'to-con' THEN
+    IF direction = 0 THEN status := 3; ELSE status := 2; END IF;
+  ELSIF intro_status = 'con' THEN
+    status := 4;
+  ELSE
+    status := 1;
+  END IF;
+  byte_val := (direction * 8) + status;
+  new_len := GREATEST(length(state), idx + 1);
+  IF new_len > length(state) THEN
+    result := state || (SELECT string_agg('\x00'::BYTEA, ''::BYTEA) FROM generate_series(1, new_len - length(state)));
+  ELSE
+    result := state;
+  END IF;
+  result := set_byte(result, idx::INT, byte_val);
+  RETURN result;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+CREATE AGGREGATE migrate_relations_vector(BIGINT, INT, TEXT) (
+  SFUNC = migrate_relations_vector_step,
+  STYPE = BYTEA,
+  INITCOND = ''
+);
+
+ALTER TABLE group_members ADD COLUMN index_in_group BIGINT NOT NULL DEFAULT 0;
+
+ALTER TABLE groups ADD COLUMN member_index BIGINT NOT NULL DEFAULT 0;
+
+ALTER TABLE group_members ADD COLUMN member_relations_vector BYTEA;
+
+CREATE INDEX tmp_idx_group_members_group_id_group_member_id ON group_members(group_id, group_member_id);
+
+CREATE TEMPORARY TABLE tmp_members_indexed AS
+SELECT
+  group_member_id,
+  ROW_NUMBER() OVER (
+    PARTITION BY group_id
+    ORDER BY group_member_id ASC
+  ) - 1 AS idx_in_group
+FROM group_members;
+
+CREATE INDEX tmp_idx_members_indexed ON tmp_members_indexed(group_member_id);
+
+UPDATE group_members AS gm
+SET index_in_group = tmi.idx_in_group
+FROM tmp_members_indexed tmi
+WHERE tmi.group_member_id = gm.group_member_id;
+
+DROP INDEX tmp_idx_group_members_group_id_group_member_id;
+DROP INDEX tmp_idx_members_indexed;
+DROP TABLE tmp_members_indexed;
+
+CREATE UNIQUE INDEX idx_group_members_group_id_index_in_group ON group_members(group_id, index_in_group);
+
+UPDATE groups g
+SET member_index = COALESCE((
+  SELECT MAX(index_in_group) + 1
+  FROM group_members
+  WHERE group_members.group_id = g.group_id
+), 0);
+
+UPDATE group_members
+SET member_relations_vector = ''::BYTEA
+WHERE group_id IN (
+  SELECT mu.group_id
+  FROM group_members mu
+  WHERE mu.member_category = 'user'
+    AND (
+      mu.member_role NOT IN ('admin', 'owner')
+      OR mu.member_status IN ('removed', 'left', 'deleted')
+    )
+);
+|]
+
+down_m20251117_member_relations_vector :: Text
+down_m20251117_member_relations_vector =
+  T.pack
+    [r|
+DROP AGGREGATE migrate_relations_vector(BIGINT, INT, TEXT);
+DROP FUNCTION migrate_relations_vector_step(BYTEA, BIGINT, INT, TEXT);
+DROP FUNCTION set_member_vector_new_relation(BYTEA, BIGINT, INT, INT);
+
+DROP INDEX idx_group_members_group_id_index_in_group;
+
+ALTER TABLE group_members DROP COLUMN index_in_group;
+
+ALTER TABLE groups DROP COLUMN member_index;
+
+ALTER TABLE group_members DROP COLUMN member_relations_vector;
+|]

src/Simplex/Chat/Store/Postgres/Migrations/M20251128_member_relations_vector_stage_2.hs

@@ -0,0 +1,45 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Chat.Store.Postgres.Migrations.M20251128_member_relations_vector_stage_2 where
+
+import Data.Text (Text)
+import qualified Data.Text as T
+import Text.RawString.QQ (r)
+
+-- Build member_relations_vector for all members that don't have it yet.
+-- Uses custom aggregate function migrate_relations_vector defined in M20251117_member_relations_vector.
+--
+-- Query returns (idx, direction, intro_status) for each introduction:
+-- - direction 0 (IDSubjectIntroduced): current member (subject) is re_group_member_id, was introduced to referenced member
+-- - direction 1 (IDReferencedIntroduced): current member (subject) is to_group_member_id, referenced member was introduced to it
+
+-- TODO [relations vector] drop group_member_intros in the end of migration
+m20251128_member_relations_vector_stage_2 :: Text
+m20251128_member_relations_vector_stage_2 =
+  T.pack
+    [r|
+UPDATE group_members
+SET member_relations_vector = (
+  SELECT migrate_relations_vector(idx, direction, intro_status)
+  FROM (
+    SELECT m.index_in_group AS idx, 0 AS direction, i.intro_status
+    FROM group_member_intros i
+    JOIN group_members m ON m.group_member_id = i.to_group_member_id
+    WHERE i.re_group_member_id = group_members.group_member_id
+    UNION ALL
+    SELECT m.index_in_group AS idx, 1 AS direction, i.intro_status
+    FROM group_member_intros i
+    JOIN group_members m ON m.group_member_id = i.re_group_member_id
+    WHERE i.to_group_member_id = group_members.group_member_id
+  ) AS relations
+)
+WHERE member_relations_vector IS NULL;
+|]
+
+-- TODO [relations vector] re-create group_member_intros
+down_m20251128_member_relations_vector_stage_2 :: Text
+down_m20251128_member_relations_vector_stage_2 =
+  T.pack
+    [r|
+
+|]

src/Simplex/Chat/Store/Postgres/Migrations/M20251212_chat_relays.hs

@@ -1,13 +1,13 @@
 {-# LANGUAGE QuasiQuotes #-}
 
-module Simplex.Chat.Store.Postgres.Migrations.M20251018_chat_relays where
+module Simplex.Chat.Store.Postgres.Migrations.M20251212_chat_relays where
 
 import Data.Text (Text)
 import qualified Data.Text as T
 import Text.RawString.QQ (r)
 
-m20251018_chat_relays :: Text
-m20251018_chat_relays =
+m20251212_chat_relays :: Text
+m20251212_chat_relays =
   T.pack
     [r|
 CREATE TABLE chat_relays(
@@ -32,8 +32,8 @@ ALTER TABLE users ADD COLUMN is_user_chat_relay SMALLINT NOT NULL DEFAULT 0;
 ALTER TABLE group_members ADD COLUMN is_chat_relay SMALLINT NOT NULL DEFAULT 0;
 |]
 
-down_m20251018_chat_relays :: Text
-down_m20251018_chat_relays =
+down_m20251212_chat_relays :: Text
+down_m20251212_chat_relays =
   T.pack
     [r|
 ALTER TABLE group_members DROP COLUMN is_chat_relay;

src/Simplex/Chat/Store/Postgres/Migrations/chat_schema.sql

@@ -34,6 +34,41 @@ $$;
 
 
 
+CREATE FUNCTION test_chat_schema.migrate_relations_vector_step(state bytea, idx bigint, direction integer, intro_status text) RETURNS bytea
+    LANGUAGE plpgsql IMMUTABLE
+    AS $$
+DECLARE
+  new_len INT;
+  result BYTEA;
+  status INT;
+  byte_val INT;
+BEGIN
+  IF idx < 0 THEN
+    RETURN state;
+  END IF;
+  IF intro_status = 're-con' THEN
+    IF direction = 0 THEN status := 2; ELSE status := 3; END IF;
+  ELSIF intro_status = 'to-con' THEN
+    IF direction = 0 THEN status := 3; ELSE status := 2; END IF;
+  ELSIF intro_status = 'con' THEN
+    status := 4;
+  ELSE
+    status := 1;
+  END IF;
+  byte_val := (direction * 8) + status;
+  new_len := GREATEST(length(state), idx + 1);
+  IF new_len > length(state) THEN
+    result := state || (SELECT string_agg('\x00'::BYTEA, ''::BYTEA) FROM generate_series(1, new_len - length(state)));
+  ELSE
+    result := state;
+  END IF;
+  result := set_byte(result, idx::INT, byte_val);
+  RETURN result;
+END;
+$$;
+
+
+
 CREATE FUNCTION test_chat_schema.on_group_members_delete_update_summary() RETURNS trigger
     LANGUAGE plpgsql
     AS $$
@@ -85,6 +120,45 @@ END;
 $$;
 
 
+
+CREATE FUNCTION test_chat_schema.set_member_vector_new_relation(v bytea, idx bigint, direction integer, status integer) RETURNS bytea
+    LANGUAGE plpgsql IMMUTABLE
+    AS $$
+DECLARE
+  new_len INT;
+  result BYTEA;
+  byte_val INT;
+  old_byte INT;
+BEGIN
+  IF idx < 0 THEN
+    RETURN v;
+  END IF;
+  IF idx < length(v) THEN
+    old_byte := get_byte(v, idx::INT);
+  ELSE
+    old_byte := 0;
+  END IF;
+  byte_val := (old_byte & x'F0'::INT) | (direction * 8) | status;
+  new_len := GREATEST(length(v), idx + 1);
+  IF new_len > length(v) THEN
+    result := v || (SELECT string_agg('\x00'::BYTEA, ''::BYTEA) FROM generate_series(1, new_len - length(v)));
+  ELSE
+    result := v;
+  END IF;
+  result := set_byte(result, idx::INT, byte_val);
+  RETURN result;
+END;
+$$;
+
+
+
+CREATE AGGREGATE test_chat_schema.migrate_relations_vector(bigint, integer, text) (
+    SFUNC = test_chat_schema.migrate_relations_vector_step,
+    STYPE = bytea,
+    INITCOND = ''
+);
+
+
 SET default_table_access_method = heap;
 
 
@@ -284,6 +358,32 @@ ALTER TABLE test_chat_schema.chat_items ALTER COLUMN chat_item_id ADD GENERATED
 
 
 
+CREATE TABLE test_chat_schema.chat_relays (
+    chat_relay_id bigint NOT NULL,
+    address text NOT NULL,
+    name text NOT NULL,
+    domains text NOT NULL,
+    preset smallint DEFAULT 0 NOT NULL,
+    tested smallint,
+    enabled smallint DEFAULT 1 NOT NULL,
+    user_id bigint NOT NULL,
+    created_at text DEFAULT now() NOT NULL,
+    updated_at text DEFAULT now() NOT NULL
+);
+
+
+
+ALTER TABLE test_chat_schema.chat_relays ALTER COLUMN chat_relay_id ADD GENERATED ALWAYS AS IDENTITY (
+    SEQUENCE NAME test_chat_schema.chat_relays_chat_relay_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1
+);
+
+
+
 CREATE TABLE test_chat_schema.chat_tags (
     chat_tag_id bigint NOT NULL,
     user_id bigint,
@@ -706,7 +806,10 @@ CREATE TABLE test_chat_schema.group_members (
     support_chat_items_mentions bigint DEFAULT 0 NOT NULL,
     support_chat_last_msg_from_member_ts timestamp with time zone,
     member_xcontact_id bytea,
-    member_welcome_shared_msg_id bytea
+    member_welcome_shared_msg_id bytea,
+    index_in_group bigint DEFAULT 0 NOT NULL,
+    member_relations_vector bytea,
+    is_chat_relay smallint DEFAULT 0 NOT NULL
 );
 
 
@@ -805,7 +908,8 @@ CREATE TABLE test_chat_schema.groups (
     request_shared_msg_id bytea,
     conn_link_prepared_connection smallint DEFAULT 0 NOT NULL,
     via_group_link_uri bytea,
-    summary_current_members_count bigint DEFAULT 0 NOT NULL
+    summary_current_members_count bigint DEFAULT 0 NOT NULL,
+    member_index bigint DEFAULT 0 NOT NULL
 );
 
 
@@ -1273,7 +1377,8 @@ CREATE TABLE test_chat_schema.users (
     user_member_profile_updated_at timestamp with time zone,
     ui_themes text,
     active_order bigint DEFAULT 0 NOT NULL,
-    auto_accept_member_contacts smallint DEFAULT 0 NOT NULL
+    auto_accept_member_contacts smallint DEFAULT 0 NOT NULL,
+    is_user_chat_relay smallint DEFAULT 0 NOT NULL
 );
 
 
@@ -1357,6 +1462,21 @@ ALTER TABLE ONLY test_chat_schema.chat_items
 
 
 
+ALTER TABLE ONLY test_chat_schema.chat_relays
+    ADD CONSTRAINT chat_relays_pkey PRIMARY KEY (chat_relay_id);
+
+
+
+ALTER TABLE ONLY test_chat_schema.chat_relays
+    ADD CONSTRAINT chat_relays_user_id_address_key UNIQUE (user_id, address);
+
+
+
+ALTER TABLE ONLY test_chat_schema.chat_relays
+    ADD CONSTRAINT chat_relays_user_id_name_key UNIQUE (user_id, name);
+
+
+
 ALTER TABLE ONLY test_chat_schema.chat_tags
     ADD CONSTRAINT chat_tags_pkey PRIMARY KEY (chat_tag_id);
 
@@ -1837,6 +1957,10 @@ CREATE INDEX idx_chat_items_user_id_item_status ON test_chat_schema.chat_items U
 
 
 
+CREATE INDEX idx_chat_relays_user_id ON test_chat_schema.chat_relays USING btree (user_id);
+
+
+
 CREATE INDEX idx_chat_tags_chats_chat_tag_id ON test_chat_schema.chat_tags_chats USING btree (chat_tag_id);
 
 
@@ -2081,6 +2205,10 @@ CREATE INDEX idx_group_members_group_id ON test_chat_schema.group_members USING
 
 
 
+CREATE UNIQUE INDEX idx_group_members_group_id_index_in_group ON test_chat_schema.group_members USING btree (group_id, index_in_group);
+
+
+
 CREATE INDEX idx_group_members_invited_by ON test_chat_schema.group_members USING btree (invited_by);
 
 
@@ -2463,6 +2591,11 @@ ALTER TABLE ONLY test_chat_schema.chat_items
 
 
 
+ALTER TABLE ONLY test_chat_schema.chat_relays
+    ADD CONSTRAINT chat_relays_user_id_fkey FOREIGN KEY (user_id) REFERENCES test_chat_schema.users(user_id) ON DELETE CASCADE;
+
+
+
 ALTER TABLE ONLY test_chat_schema.chat_tags_chats
     ADD CONSTRAINT chat_tags_chats_chat_tag_id_fkey FOREIGN KEY (chat_tag_id) REFERENCES test_chat_schema.chat_tags(chat_tag_id) ON DELETE CASCADE;