core: filter all messages with links in text when link content filter is used (#6591)

* core: change msg_content_tag stored as blob to text

* track if items have links and use has_link when searching for links (MCLink_ MsgContentTag is passed)

* fix test (broken in master)

* merge migrations, update api and schema

* minor fixes

* tests

* update plans

* update migrations
This commit is contained in:
Evgeny
2026-01-23 12:31:42 +00:00
committed by GitHub
parent d08202ee2d
commit 5453d8875c
21 changed files with 355 additions and 82 deletions
+3 -1
View File
@@ -148,6 +148,7 @@ import Simplex.Chat.Store.SQLite.Migrations.M20251117_member_relations_vector
import Simplex.Chat.Store.SQLite.Migrations.M20251128_migrate_member_relations
import Simplex.Chat.Store.SQLite.Migrations.M20251230_strict_tables
import Simplex.Chat.Store.SQLite.Migrations.M20260108_chat_indices
import Simplex.Chat.Store.SQLite.Migrations.M20260122_has_link
import Simplex.Messaging.Agent.Store.Shared (Migration (..))
schemaMigrations :: [(String, Query, Maybe Query)]
@@ -295,7 +296,8 @@ schemaMigrations =
("20251117_member_relations_vector", m20251117_member_relations_vector, Just down_m20251117_member_relations_vector),
("20251128_migrate_member_relations", m20251128_migrate_member_relations, Just down_m20251128_migrate_member_relations),
("20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables),
("20260108_chat_indices", m20260108_chat_indices, Just down_m20260108_chat_indices)
("20260108_chat_indices", m20260108_chat_indices, Just down_m20260108_chat_indices),
("20260122_has_link", m20260122_has_link, Just down_m20260122_has_link)
]
-- | The list of migrations in ascending order by date
@@ -0,0 +1,33 @@
{-# LANGUAGE QuasiQuotes #-}
module Simplex.Chat.Store.SQLite.Migrations.M20260122_has_link where
import Database.SQLite.Simple (Query)
import Database.SQLite.Simple.QQ (sql)
m20260122_has_link :: Query
m20260122_has_link =
[sql|
UPDATE chat_items SET msg_content_tag = 'text' WHERE msg_content_tag = 'liveText';
UPDATE chat_items SET msg_content_tag = CAST(msg_content_tag as TEXT) WHERE typeof(msg_content_tag) = 'blob';
ALTER TABLE chat_items ADD COLUMN has_link INTEGER NOT NULL DEFAULT 0;
UPDATE chat_items SET has_link = 1
WHERE msg_content_tag = 'link' OR item_text LIKE '%https://%';
CREATE INDEX idx_chat_items_groups_has_link_item_ts ON chat_items(user_id, group_id, has_link, item_ts);
CREATE INDEX idx_chat_items_contacts_has_link_created_at ON chat_items(user_id, contact_id, has_link, created_at);
CREATE INDEX idx_chat_items_note_folder_has_link_created_at ON chat_items(user_id, note_folder_id, has_link, created_at);
|]
down_m20260122_has_link :: Query
down_m20260122_has_link =
[sql|
DROP INDEX idx_chat_items_note_folder_has_link_created_at;
DROP INDEX idx_chat_items_contacts_has_link_created_at;
DROP INDEX idx_chat_items_groups_has_link_item_ts;
ALTER TABLE chat_items DROP COLUMN has_link;
|]
@@ -765,7 +765,7 @@ Query:
LIMIT ?
Plan:
SEARCH chat_items USING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=?)
SEARCH chat_items USING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=?)
USE TEMP B-TREE FOR ORDER BY
Query:
@@ -776,7 +776,7 @@ Query:
LIMIT ?
Plan:
SEARCH chat_items USING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=?)
SEARCH chat_items USING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=?)
USE TEMP B-TREE FOR ORDER BY
Query:
@@ -1088,7 +1088,7 @@ Query:
i.chat_item_id, i.item_ts, i.item_sent, i.item_content, i.item_text, i.item_status, i.via_proxy, i.shared_msg_id,
i.item_deleted, i.item_deleted_ts, i.item_edited, i.created_at, i.updated_at,
i.fwd_from_tag, i.fwd_from_chat_name, i.fwd_from_msg_dir, i.fwd_from_contact_id, i.fwd_from_group_id, i.fwd_from_chat_item_id,
i.timed_ttl, i.timed_delete_at, i.item_live, i.user_mention,
i.timed_ttl, i.timed_delete_at, i.item_live, i.user_mention, i.has_link,
-- CIFile
f.file_id, f.file_name, f.file_size, f.file_path, f.file_crypto_key, f.file_crypto_nonce, f.ci_file_status, f.protocol
FROM chat_items i
@@ -1105,7 +1105,7 @@ Query:
i.chat_item_id, i.item_ts, i.item_sent, i.item_content, i.item_text, i.item_status, i.via_proxy, i.shared_msg_id,
i.item_deleted, i.item_deleted_ts, i.item_edited, i.created_at, i.updated_at,
i.fwd_from_tag, i.fwd_from_chat_name, i.fwd_from_msg_dir, i.fwd_from_contact_id, i.fwd_from_group_id, i.fwd_from_chat_item_id,
i.timed_ttl, i.timed_delete_at, i.item_live, i.user_mention,
i.timed_ttl, i.timed_delete_at, i.item_live, i.user_mention, i.has_link,
-- CIFile
f.file_id, f.file_name, f.file_size, f.file_path, f.file_crypto_key, f.file_crypto_nonce, f.ci_file_status, f.protocol,
-- CIMeta forwardedByMember, showGroupAsSender
@@ -1161,7 +1161,7 @@ Query:
i.chat_item_id, i.item_ts, i.item_sent, i.item_content, i.item_text, i.item_status, i.via_proxy, i.shared_msg_id,
i.item_deleted, i.item_deleted_ts, i.item_edited, i.created_at, i.updated_at,
i.fwd_from_tag, i.fwd_from_chat_name, i.fwd_from_msg_dir, i.fwd_from_contact_id, i.fwd_from_group_id, i.fwd_from_chat_item_id,
i.timed_ttl, i.timed_delete_at, i.item_live, i.user_mention,
i.timed_ttl, i.timed_delete_at, i.item_live, i.user_mention, i.has_link,
-- CIFile
f.file_id, f.file_name, f.file_size, f.file_path, f.file_crypto_key, f.file_crypto_nonce, f.ci_file_status, f.protocol,
-- DirectQuote
@@ -1292,7 +1292,7 @@ Query:
LIMIT ?
Plan:
SEARCH chat_items USING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=?)
SEARCH chat_items USING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=?)
USE TEMP B-TREE FOR ORDER BY
Query:
@@ -4166,7 +4166,7 @@ Query:
Plan:
SEARCH files USING INDEX idx_files_user_id (user_id=?)
LIST SUBQUERY 1
SEARCH chat_items USING COVERING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=?)
SEARCH chat_items USING COVERING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=? AND note_folder_id=?)
SEARCH extra_xftp_file_descriptions USING COVERING INDEX idx_extra_xftp_file_descriptions_file_id (file_id=?)
SEARCH rcv_files USING INTEGER PRIMARY KEY (rowid=?)
SEARCH snd_files USING COVERING INDEX idx_snd_files_file_id (file_id=?)
@@ -4212,12 +4212,12 @@ Query:
user_id, created_by_msg_id, contact_id, group_id, group_member_id, note_folder_id, group_scope_tag, group_scope_group_member_id,
-- meta
item_sent, item_ts, item_content, item_content_tag, item_text, item_status, msg_content_tag, shared_msg_id,
forwarded_by_group_member_id, include_in_history, created_at, updated_at, item_live, user_mention, show_group_as_sender, timed_ttl, timed_delete_at,
forwarded_by_group_member_id, include_in_history, created_at, updated_at, item_live, user_mention, has_link, show_group_as_sender, timed_ttl, timed_delete_at,
-- quote
quoted_shared_msg_id, quoted_sent_at, quoted_content, quoted_sent, quoted_member_id,
-- forwarded from
fwd_from_tag, fwd_from_chat_name, fwd_from_msg_dir, fwd_from_contact_id, fwd_from_group_id, fwd_from_chat_item_id
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
Plan:
@@ -5141,7 +5141,7 @@ Query:
JOIN files f ON f.chat_item_id = i.chat_item_id
WHERE i.user_id = ?
Plan:
SEARCH i USING COVERING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=?)
SEARCH i USING COVERING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=?)
SEARCH f USING INDEX idx_files_chat_item_id (chat_item_id=?)
Query:
@@ -5150,7 +5150,7 @@ Query:
JOIN files f ON f.chat_item_id = i.chat_item_id
WHERE i.user_id = ? AND i.contact_id = ?
Plan:
SEARCH i USING COVERING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?)
SEARCH i USING COVERING INDEX idx_chat_items_contacts_has_link_created_at (user_id=? AND contact_id=?)
SEARCH f USING INDEX idx_files_chat_item_id (chat_item_id=?)
Query:
@@ -5168,7 +5168,7 @@ Query:
JOIN files f ON f.chat_item_id = i.chat_item_id
WHERE i.user_id = ? AND i.group_id = ?
Plan:
SEARCH i USING COVERING INDEX idx_chat_items_groups_user_mention (user_id=? AND group_id=?)
SEARCH i USING COVERING INDEX idx_chat_items_groups_has_link_item_ts (user_id=? AND group_id=?)
SEARCH f USING INDEX idx_files_chat_item_id (chat_item_id=?)
Query:
@@ -5186,7 +5186,7 @@ Query:
JOIN files f ON f.chat_item_id = i.chat_item_id
WHERE i.user_id = ? AND i.note_folder_id = ?
Plan:
SEARCH i USING COVERING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=?)
SEARCH i USING COVERING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=? AND note_folder_id=?)
SEARCH f USING INDEX idx_files_chat_item_id (chat_item_id=?)
Query:
@@ -5469,6 +5469,10 @@ Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND contact_id =
Plan:
SEARCH chat_items USING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?)
Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND contact_id = ? AND has_link = 1 ORDER BY created_at DESC, chat_item_id DESC LIMIT ?
Plan:
SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_has_link_created_at (user_id=? AND contact_id=? AND has_link=?)
Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND contact_id = ? AND msg_content_tag = ? ORDER BY created_at DESC, chat_item_id DESC LIMIT ?
Plan:
SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_msg_content_tag_created_at (user_id=? AND contact_id=? AND msg_content_tag=?)
@@ -5489,6 +5493,10 @@ Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ?
Plan:
SEARCH chat_items USING COVERING INDEX idx_chat_items_group_scope_item_ts (user_id=? AND group_id=? AND group_scope_tag=? AND group_scope_group_member_id=?)
Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ? AND has_link = 1 ORDER BY item_ts DESC, chat_item_id DESC LIMIT ?
Plan:
SEARCH chat_items USING COVERING INDEX idx_chat_items_groups_has_link_item_ts (user_id=? AND group_id=? AND has_link=?)
Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ? AND msg_content_tag = ? ORDER BY item_ts DESC, chat_item_id DESC LIMIT ?
Plan:
SEARCH chat_items USING COVERING INDEX idx_chat_items_groups_msg_content_tag_item_ts (user_id=? AND group_id=? AND msg_content_tag=?)
@@ -5497,6 +5505,10 @@ Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND note_folder_i
Plan:
SEARCH chat_items USING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=?)
Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND note_folder_id = ? AND has_link = 1 ORDER BY created_at DESC, chat_item_id DESC LIMIT ?
Plan:
SEARCH chat_items USING COVERING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=? AND note_folder_id=? AND has_link=?)
Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND note_folder_id = ? AND msg_content_tag = ? ORDER BY created_at DESC, chat_item_id DESC LIMIT ?
Plan:
SEARCH chat_items USING COVERING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=? AND note_folder_id=? AND msg_content_tag=?)
@@ -5549,7 +5561,7 @@ SEARCH chat_item_versions USING COVERING INDEX idx_chat_item_versions_chat_item_
Query: DELETE FROM chat_items WHERE user_id = ? AND contact_id = ?
Plan:
SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?)
SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_has_link_created_at (user_id=? AND contact_id=?)
SEARCH chat_item_mentions USING COVERING INDEX idx_chat_item_mentions_chat_item_id (chat_item_id=?)
SEARCH group_snd_item_statuses USING COVERING INDEX idx_group_snd_item_statuses_chat_item_id (chat_item_id=?)
SEARCH chat_item_versions USING COVERING INDEX idx_chat_item_versions_chat_item_id (chat_item_id=?)
@@ -5573,7 +5585,7 @@ SEARCH groups USING COVERING INDEX idx_groups_chat_item_id (chat_item_id=?)
Query: DELETE FROM chat_items WHERE user_id = ? AND contact_id = ? AND item_content_tag != 'chatBanner'
Plan:
SEARCH chat_items USING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?)
SEARCH chat_items USING INDEX idx_chat_items_contacts_has_link_created_at (user_id=? AND contact_id=?)
SEARCH chat_item_mentions USING COVERING INDEX idx_chat_item_mentions_chat_item_id (chat_item_id=?)
SEARCH group_snd_item_statuses USING COVERING INDEX idx_group_snd_item_statuses_chat_item_id (chat_item_id=?)
SEARCH chat_item_versions USING COVERING INDEX idx_chat_item_versions_chat_item_id (chat_item_id=?)
@@ -5585,7 +5597,7 @@ SEARCH groups USING COVERING INDEX idx_groups_chat_item_id (chat_item_id=?)
Query: DELETE FROM chat_items WHERE user_id = ? AND group_id = ?
Plan:
SEARCH chat_items USING COVERING INDEX idx_chat_items_groups_user_mention (user_id=? AND group_id=?)
SEARCH chat_items USING COVERING INDEX idx_chat_items_groups_has_link_item_ts (user_id=? AND group_id=?)
SEARCH chat_item_mentions USING COVERING INDEX idx_chat_item_mentions_chat_item_id (chat_item_id=?)
SEARCH group_snd_item_statuses USING COVERING INDEX idx_group_snd_item_statuses_chat_item_id (chat_item_id=?)
SEARCH chat_item_versions USING COVERING INDEX idx_chat_item_versions_chat_item_id (chat_item_id=?)
@@ -5609,7 +5621,7 @@ SEARCH groups USING COVERING INDEX idx_groups_chat_item_id (chat_item_id=?)
Query: DELETE FROM chat_items WHERE user_id = ? AND group_id = ? AND item_content_tag != 'chatBanner'
Plan:
SEARCH chat_items USING INDEX idx_chat_items_groups_user_mention (user_id=? AND group_id=?)
SEARCH chat_items USING INDEX idx_chat_items_groups_has_link_item_ts (user_id=? AND group_id=?)
SEARCH chat_item_mentions USING COVERING INDEX idx_chat_item_mentions_chat_item_id (chat_item_id=?)
SEARCH group_snd_item_statuses USING COVERING INDEX idx_group_snd_item_statuses_chat_item_id (chat_item_id=?)
SEARCH chat_item_versions USING COVERING INDEX idx_chat_item_versions_chat_item_id (chat_item_id=?)
@@ -5621,7 +5633,7 @@ SEARCH groups USING COVERING INDEX idx_groups_chat_item_id (chat_item_id=?)
Query: DELETE FROM chat_items WHERE user_id = ? AND note_folder_id = ?
Plan:
SEARCH chat_items USING COVERING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=?)
SEARCH chat_items USING COVERING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=? AND note_folder_id=?)
SEARCH chat_item_mentions USING COVERING INDEX idx_chat_item_mentions_chat_item_id (chat_item_id=?)
SEARCH group_snd_item_statuses USING COVERING INDEX idx_group_snd_item_statuses_chat_item_id (chat_item_id=?)
SEARCH chat_item_versions USING COVERING INDEX idx_chat_item_versions_chat_item_id (chat_item_id=?)
@@ -5903,7 +5915,7 @@ SEARCH protocol_servers USING COVERING INDEX idx_smp_servers_user_id (user_id=?)
SEARCH settings USING COVERING INDEX idx_settings_user_id (user_id=?)
SEARCH commands USING COVERING INDEX idx_commands_user_id (user_id=?)
SEARCH calls USING COVERING INDEX idx_calls_user_id (user_id=?)
SEARCH chat_items USING COVERING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=?)
SEARCH chat_items USING COVERING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=?)
SEARCH contact_requests USING COVERING INDEX sqlite_autoindex_contact_requests_2 (user_id=?)
SEARCH user_contact_links USING COVERING INDEX sqlite_autoindex_user_contact_links_1 (user_id=?)
SEARCH connections USING COVERING INDEX idx_connections_to_subscribe (user_id=?)
@@ -6067,7 +6079,7 @@ Query: SELECT EXISTS (SELECT 1 FROM chat_items WHERE user_id = ? AND contact_id
Plan:
SCAN CONSTANT ROW
SCALAR SUBQUERY 1
SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?)
SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_has_link_created_at (user_id=? AND contact_id=?)
Query: SELECT accepted_at FROM operator_usage_conditions WHERE server_operator_id = ? AND conditions_commit = ?
Plan:
@@ -440,7 +440,8 @@ CREATE TABLE chat_items(
user_mention INTEGER NOT NULL DEFAULT 0,
group_scope_tag TEXT,
group_scope_group_member_id INTEGER REFERENCES group_members(group_member_id) ON DELETE CASCADE,
show_group_as_sender INTEGER NOT NULL DEFAULT 0
show_group_as_sender INTEGER NOT NULL DEFAULT 0,
has_link INTEGER NOT NULL DEFAULT 0
) STRICT;
CREATE TABLE sqlite_sequence(name,seq);
CREATE TABLE chat_item_messages(
@@ -1199,6 +1200,24 @@ CREATE INDEX idx_chat_items_note_folder_msg_content_tag_created_at ON chat_items
msg_content_tag,
created_at
);
CREATE INDEX idx_chat_items_groups_has_link_item_ts ON chat_items(
user_id,
group_id,
has_link,
item_ts
);
CREATE INDEX idx_chat_items_contacts_has_link_created_at ON chat_items(
user_id,
contact_id,
has_link,
created_at
);
CREATE INDEX idx_chat_items_note_folder_has_link_created_at ON chat_items(
user_id,
note_folder_id,
has_link,
created_at
);
CREATE TRIGGER on_group_members_insert_update_summary
AFTER INSERT ON group_members
FOR EACH ROW