From c07c17689606c27442360e114532e7cce60dd07d Mon Sep 17 00:00:00 2001 From: Evgeny Date: Tue, 16 Sep 2025 21:58:28 +0100 Subject: [PATCH] core: batch load queues for subscriptions with PostgreSQL client (#6272) * core: batch load queues for subscriptions with PostgreSQL client * update simplexmq * fix, update simplexmq * update query plans --- cabal.project | 2 +- scripts/db/README.md | 76 ++++--------------- scripts/desktop/build-lib-mac.sh | 2 +- scripts/nix/sha256map.nix | 2 +- src/Simplex/Chat/Options/Postgres.hs | 4 +- .../SQLite/Migrations/agent_query_plans.txt | 25 +++--- tests/ChatClient.hs | 2 +- tests/PostgresSchemaDump.hs | 6 +- 8 files changed, 35 insertions(+), 84 deletions(-) diff --git a/cabal.project b/cabal.project index df72f5da32..a414dbcd5b 100644 --- a/cabal.project +++ b/cabal.project @@ -12,7 +12,7 @@ constraints: zip +disable-bzip2 +disable-zstd source-repository-package type: git location: https://github.com/simplex-chat/simplexmq.git - tag: 112cd9d5f44055b6d9b12cb918a1f108f78726f8 + tag: 7b7616ce7e3ba40eb101d20c5dccd79a14de5580 source-repository-package type: git diff --git a/scripts/db/README.md b/scripts/db/README.md index 333a6f4acf..c204e036c4 100644 --- a/scripts/db/README.md +++ b/scripts/db/README.md @@ -10,33 +10,13 @@ sqlcipher simplex_v1_agent.db ``` - 2. Set your db password: + 2. Run in sqlcipher: ```sql - PRAGMA key = ''; - ``` - - 3. Check if db was successfully decrypted: - - ```sh - SELECT count(*) FROM sqlite_master; - ``` - - 4. Attach new empty db: - - ```sh - ATTACH DATABASE 'simplex_v1_agent_plaintext.db' AS plaintext KEY ''; - ``` - - 5. Export opened db to attached db as plaintext: - - ```sh - SELECT sqlcipher_export('plaintext'); - ``` - - 6. Deattach the plaintext db: - - ```sh + PRAGMA key = ''; -- Set your db password + SELECT count(*) FROM sqlite_master; -- Check if db was successfully decrypted + ATTACH DATABASE 'simplex_v1_agent_plaintext.db' AS plaintext KEY ''; -- Attach new empty db + SELECT sqlcipher_export('plaintext'); -- Export opened db to attached db as plaintext DETACH DATABASE plaintext; ``` @@ -48,33 +28,13 @@ sqlcipher simplex_v1_chat.db ``` - 2. Set your db password: + 2. Run in sqlcipher: ```sql PRAGMA key = ''; - ``` - - 3. Check if db was successfully decrypted: - - ```sh SELECT count(*) FROM sqlite_master; - ``` - - 4. Attach new empty db: - - ```sh ATTACH DATABASE 'simplex_v1_chat_plaintext.db' AS plaintext KEY ''; - ``` - - 5. Export opened db to attached db as plaintext: - - ```sh SELECT sqlcipher_export('plaintext'); - ``` - - 6. Deattach the plaintext db: - - ```sh DETACH DATABASE plaintext; ``` @@ -86,22 +46,14 @@ psql -U postgres -h localhost ``` - 2. Create user with password: + 2. Run in psql: - ```sh - CREATE USER simplex WITH ENCRYPTED PASSWORD '123123'; - ``` - - 3. Create database: - - ```sh - CREATE DATABASE simplex_v1; - ``` - - 4. Assign permissions: - - ```sh - GRANT ALL PRIVILEGES ON DATABASE simplex_v1 TO simplex; + ```sql + CREATE USER simplex WITH ENCRYPTED PASSWORD '123123'; -- Create user with password + -- or + -- CREATE USER simplex; + CREATE DATABASE simplex_v1; -- Create database + GRANT ALL PRIVILEGES ON DATABASE simplex_v1 TO simplex; -- Assign permissions ``` 3. Prepare database: @@ -294,7 +246,7 @@ ./gradlew packageDmg -Pdatabase.backend=postgres ``` -## Transfer data from SQLite to Postgres database +## Transfer data from Postgres to SQLite database 1. Prepare sqlite db: diff --git a/scripts/desktop/build-lib-mac.sh b/scripts/desktop/build-lib-mac.sh index a3a824f81b..6e3415bd80 100755 --- a/scripts/desktop/build-lib-mac.sh +++ b/scripts/desktop/build-lib-mac.sh @@ -28,7 +28,7 @@ rm -rf $BUILD_DIR if [[ "$DATABASE_BACKEND" == "postgres" ]]; then echo "Building with postgres backend..." - cabal build -f client_postgres lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library' + cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library +client_postgres' --constraint 'simplex-chat +client_library +client_postgres' else echo "Building with sqlite backend..." cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library' diff --git a/scripts/nix/sha256map.nix b/scripts/nix/sha256map.nix index c8404afde4..cccdc96785 100644 --- a/scripts/nix/sha256map.nix +++ b/scripts/nix/sha256map.nix @@ -1,5 +1,5 @@ { - "https://github.com/simplex-chat/simplexmq.git"."112cd9d5f44055b6d9b12cb918a1f108f78726f8" = "18jp2jrq7a4giqvqpvd3jjrbznwwl1dsh1ymmq3197xdd515wxr1"; + "https://github.com/simplex-chat/simplexmq.git"."7b7616ce7e3ba40eb101d20c5dccd79a14de5580" = "0ccscn8xx09fj8n7721cn8ynligjniwrpzgj2hi1348j1vm8h2j8"; "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38"; "https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "1ql13f4kfwkbaq7nygkxgw84213i0zm7c1a8hwvramayxl38dq5d"; "https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl"; diff --git a/src/Simplex/Chat/Options/Postgres.hs b/src/Simplex/Chat/Options/Postgres.hs index 46c1faa5e0..13af13b20a 100644 --- a/src/Simplex/Chat/Options/Postgres.hs +++ b/src/Simplex/Chat/Options/Postgres.hs @@ -42,7 +42,7 @@ chatDbOptsP _appDir defaultDbName = do ( long "pool-size" <> metavar "DB_POOL_SIZE" <> help "Database connection pool size" - <> value 1 + <> value 10 <> showDefault ) dbCreateSchema <- @@ -81,7 +81,7 @@ mobileDbOpts schemaPrefix connstr = do ChatDbOpts { dbConnstr, dbSchemaPrefix, - dbPoolSize = 1, + dbPoolSize = 10, dbCreateSchema = True } diff --git a/src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt b/src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt index 41f01b3b51..b2b4cd87ae 100644 --- a/src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt +++ b/src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt @@ -327,16 +327,6 @@ Plan: SEARCH r USING INTEGER PRIMARY KEY (rowid=?) SEARCH s USING INTEGER PRIMARY KEY (rowid=?) -Query: - SELECT - user_id, conn_id, conn_mode, smp_agent_version, enable_ntfs, - last_external_snd_msg_id, deleted, ratchet_sync_state, pq_support - FROM connections - WHERE conn_id = ? - -Plan: -SEARCH connections USING PRIMARY KEY (conn_id=?) - Query: SELECT DISTINCT s.xftp_host, s.xftp_port, s.xftp_key_hash @@ -384,13 +374,13 @@ SEARCH s USING INTEGER PRIMARY KEY (rowid=?) USE TEMP B-TREE FOR DISTINCT Query: - SELECT DISTINCT c.host, c.port, COALESCE(c.server_key_hash, s.key_hash) + SELECT DISTINCT c.conn_id, c.host, c.port, COALESCE(c.server_key_hash, s.key_hash) FROM commands c LEFT JOIN servers s ON s.host = c.host AND s.port = c.port - WHERE conn_id = ? + ORDER BY c.conn_id Plan: -SEARCH c USING INDEX idx_commands_conn_id (conn_id=?) +SCAN c USING INDEX idx_commands_conn_id SEARCH s USING PRIMARY KEY (host=? AND port=?) LEFT-JOIN USE TEMP B-TREE FOR DISTINCT @@ -469,6 +459,15 @@ Plan: SEARCH s USING PRIMARY KEY (conn_id=? AND internal_snd_id=?) SEARCH m USING PRIMARY KEY (conn_id=? AND internal_id=?) +Query: + SELECT user_id, conn_id, conn_mode, smp_agent_version, enable_ntfs, + last_external_snd_msg_id, deleted, ratchet_sync_state, pq_support + FROM connections + WHERE conn_id = ? AND deleted = ? + +Plan: +SEARCH connections USING PRIMARY KEY (conn_id=?) + Query: DELETE FROM conn_confirmations WHERE conn_id = ? diff --git a/tests/ChatClient.hs b/tests/ChatClient.hs index 5e5c34d516..3f005cb915 100644 --- a/tests/ChatClient.hs +++ b/tests/ChatClient.hs @@ -297,7 +297,7 @@ startTestChat ps cfg opts@ChatOpts {coreOptions} dbPrefix = do createDatabase :: TestParams -> CoreChatOpts -> String -> IO (Either MigrationError ChatDatabase) #if defined(dbPostgres) createDatabase _params CoreChatOpts {dbOptions} dbPrefix = do - createChatDatabase dbOptions {dbSchemaPrefix = "client_" <> dbPrefix} MCError + createChatDatabase dbOptions {dbSchemaPrefix = "client_" <> dbPrefix} (MigrationConfig MCError Nothing) insertUser :: DBStore -> IO () insertUser st = withTransaction st (`DB.execute_` "INSERT INTO users DEFAULT VALUES") diff --git a/tests/PostgresSchemaDump.hs b/tests/PostgresSchemaDump.hs index f1b509ac6f..468cdb19d5 100644 --- a/tests/PostgresSchemaDump.hs +++ b/tests/PostgresSchemaDump.hs @@ -14,7 +14,7 @@ import Data.Maybe (fromJust, isJust) import Simplex.Messaging.Agent.Store.Postgres (closeDBStore, createDBStore) import Simplex.Messaging.Agent.Store.Postgres.Common (DBOpts (..)) import qualified Simplex.Messaging.Agent.Store.Postgres.Migrations as Migrations -import Simplex.Messaging.Agent.Store.Shared (Migration (..), MigrationConfirmation (..), MigrationsToRun (..), toDownMigration) +import Simplex.Messaging.Agent.Store.Shared (Migration (..), MigrationConfig (..), MigrationConfirmation (..), MigrationsToRun (..), toDownMigration) import Simplex.Messaging.Util (ifM, whenM) import System.Directory (doesFileExist, removeFile) import System.Process (readCreateProcess, shell) @@ -32,12 +32,12 @@ postgresSchemaDumpTest migrations skipComparisonForDownMigrations testDBOpts@DBO testVerifySchemaDump = do savedSchema <- ifM (doesFileExist srcSchemaPath) (readFile srcSchemaPath) (pure "") savedSchema `deepseq` pure () - void $ createDBStore testDBOpts migrations MCConsole + void $ createDBStore testDBOpts migrations (MigrationConfig MCConsole Nothing) getSchema srcSchemaPath `shouldReturn` savedSchema testSchemaMigrations = do let noDownMigrations = dropWhileEnd (\Migration {down} -> isJust down) migrations - st <- createDBStore testDBOpts noDownMigrations MCYesUpDown >>= \case + st <- createDBStore testDBOpts noDownMigrations (MigrationConfig MCYesUpDown Nothing) >>= \case Right st -> pure st Left e -> error $ show e mapM_ (testDownMigration st) $ drop (length noDownMigrations) migrations