dandri/simplex-chat
1
0
Fork 0
mirror of https://github.com/simplex-chat/simplex-chat.git synced 2026-08-27 15:49:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

core: batch load queues for subscriptions with PostgreSQL client (#6272)

Browse Source 
* core: batch load queues for subscriptions with PostgreSQL client

* update simplexmq

* fix, update simplexmq

* update query plans
This commit is contained in:
Evgeny
2025-09-16 21:58:28 +01:00
committed by GitHub
parent 7c92c7666c
commit c07c176896
8 changed files with 35 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cabal.project
+1 -1
View File
@@ -12,7 +12,7 @@ constraints: zip +disable-bzip2 +disable-zstd
source-repository-package
type: git
location: https://github.com/simplex-chat/simplexmq.git
tag: 112cd9d5f44055b6d9b12cb918a1f108f78726f8
tag: 7b7616ce7e3ba40eb101d20c5dccd79a14de5580
source-repository-package
type: git
scripts/db/README.md
+14 -62
View File
@@ -10,33 +10,13 @@
sqlcipher simplex_v1_agent.db
```
2. Set your db password:
2. Run in sqlcipher:
```sql
PRAGMA key = '<your_password>';
```
3. Check if db was successfully decrypted:
```sh
SELECT count(*) FROM sqlite_master;
```
4. Attach new empty db:
```sh
ATTACH DATABASE 'simplex_v1_agent_plaintext.db' AS plaintext KEY '';
```
5. Export opened db to attached db as plaintext:
```sh
SELECT sqlcipher_export('plaintext');
```
6. Deattach the plaintext db:
```sh
PRAGMA key = '<your_password>'; -- Set your db password
SELECT count(*) FROM sqlite_master; -- Check if db was successfully decrypted
ATTACH DATABASE 'simplex_v1_agent_plaintext.db' AS plaintext KEY ''; -- Attach new empty db
SELECT sqlcipher_export('plaintext'); -- Export opened db to attached db as plaintext
DETACH DATABASE plaintext;
```
@@ -48,33 +28,13 @@
sqlcipher simplex_v1_chat.db
```
2. Set your db password:
2. Run in sqlcipher:
```sql
PRAGMA key = '<your_password>';
```
3. Check if db was successfully decrypted:
```sh
SELECT count(*) FROM sqlite_master;
```
4. Attach new empty db:
```sh
ATTACH DATABASE 'simplex_v1_chat_plaintext.db' AS plaintext KEY '';
```
5. Export opened db to attached db as plaintext:
```sh
SELECT sqlcipher_export('plaintext');
```
6. Deattach the plaintext db:
```sh
DETACH DATABASE plaintext;
```
@@ -86,22 +46,14 @@
psql -U postgres -h localhost
```
2. Create user with password:
2. Run in psql:
```sh
CREATE USER simplex WITH ENCRYPTED PASSWORD '123123';
```
3. Create database:
```sh
CREATE DATABASE simplex_v1;
```
4. Assign permissions:
```sh
GRANT ALL PRIVILEGES ON DATABASE simplex_v1 TO simplex;
```sql
CREATE USER simplex WITH ENCRYPTED PASSWORD '123123'; -- Create user with password
-- or
-- CREATE USER simplex;
CREATE DATABASE simplex_v1; -- Create database
GRANT ALL PRIVILEGES ON DATABASE simplex_v1 TO simplex; -- Assign permissions
```
3. Prepare database:
@@ -294,7 +246,7 @@
./gradlew packageDmg -Pdatabase.backend=postgres
```
## Transfer data from SQLite to Postgres database
## Transfer data from Postgres to SQLite database
1. Prepare sqlite db:
scripts/desktop/build-lib-mac.sh
+1 -1
View File
@@ -28,7 +28,7 @@ rm -rf $BUILD_DIR
if [[ "$DATABASE_BACKEND" == "postgres" ]]; then
echo "Building with postgres backend..."
cabal build -f client_postgres lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library'
cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library +client_postgres' --constraint 'simplex-chat +client_library +client_postgres'
else
echo "Building with sqlite backend..."
cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library'
scripts/nix/sha256map.nix
+1 -1
View File
@@ -1,5 +1,5 @@
{
"https://github.com/simplex-chat/simplexmq.git"."112cd9d5f44055b6d9b12cb918a1f108f78726f8" = "18jp2jrq7a4giqvqpvd3jjrbznwwl1dsh1ymmq3197xdd515wxr1";
"https://github.com/simplex-chat/simplexmq.git"."7b7616ce7e3ba40eb101d20c5dccd79a14de5580" = "0ccscn8xx09fj8n7721cn8ynligjniwrpzgj2hi1348j1vm8h2j8";
"https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
"https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "1ql13f4kfwkbaq7nygkxgw84213i0zm7c1a8hwvramayxl38dq5d";
"https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl";
src/Simplex/Chat/Options/Postgres.hs
+2 -2
View File
@@ -42,7 +42,7 @@ chatDbOptsP _appDir defaultDbName = do
( long "pool-size"
<> metavar "DB_POOL_SIZE"
<> help "Database connection pool size"
<> value 1
<> value 10
<> showDefault
)
dbCreateSchema <-
@@ -81,7 +81,7 @@ mobileDbOpts schemaPrefix connstr = do
ChatDbOpts
{ dbConnstr,
dbSchemaPrefix,
dbPoolSize = 1,
dbPoolSize = 10,
dbCreateSchema = True
}
src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt
+12 -13
View File
@@ -327,16 +327,6 @@ Plan:
SEARCH r USING INTEGER PRIMARY KEY (rowid=?)
SEARCH s USING INTEGER PRIMARY KEY (rowid=?)
Query:
SELECT
user_id, conn_id, conn_mode, smp_agent_version, enable_ntfs,
last_external_snd_msg_id, deleted, ratchet_sync_state, pq_support
FROM connections
WHERE conn_id = ?
Plan:
SEARCH connections USING PRIMARY KEY (conn_id=?)
Query:
SELECT DISTINCT
s.xftp_host, s.xftp_port, s.xftp_key_hash
@@ -384,13 +374,13 @@ SEARCH s USING INTEGER PRIMARY KEY (rowid=?)
USE TEMP B-TREE FOR DISTINCT
Query:
SELECT DISTINCT c.host, c.port, COALESCE(c.server_key_hash, s.key_hash)
SELECT DISTINCT c.conn_id, c.host, c.port, COALESCE(c.server_key_hash, s.key_hash)
FROM commands c
LEFT JOIN servers s ON s.host = c.host AND s.port = c.port
WHERE conn_id = ?
ORDER BY c.conn_id
Plan:
SEARCH c USING INDEX idx_commands_conn_id (conn_id=?)
SCAN c USING INDEX idx_commands_conn_id
SEARCH s USING PRIMARY KEY (host=? AND port=?) LEFT-JOIN
USE TEMP B-TREE FOR DISTINCT
@@ -469,6 +459,15 @@ Plan:
SEARCH s USING PRIMARY KEY (conn_id=? AND internal_snd_id=?)
SEARCH m USING PRIMARY KEY (conn_id=? AND internal_id=?)
Query:
SELECT user_id, conn_id, conn_mode, smp_agent_version, enable_ntfs,
last_external_snd_msg_id, deleted, ratchet_sync_state, pq_support
FROM connections
WHERE conn_id = ? AND deleted = ?
Plan:
SEARCH connections USING PRIMARY KEY (conn_id=?)
Query:
DELETE FROM conn_confirmations
WHERE conn_id = ?
tests/ChatClient.hs
+1 -1
View File
@@ -297,7 +297,7 @@ startTestChat ps cfg opts@ChatOpts {coreOptions} dbPrefix = do
createDatabase :: TestParams -> CoreChatOpts -> String -> IO (Either MigrationError ChatDatabase)
#if defined(dbPostgres)
createDatabase _params CoreChatOpts {dbOptions} dbPrefix = do
createChatDatabase dbOptions {dbSchemaPrefix = "client_" <> dbPrefix} MCError
createChatDatabase dbOptions {dbSchemaPrefix = "client_" <> dbPrefix} (MigrationConfig MCError Nothing)
insertUser :: DBStore -> IO ()
insertUser st = withTransaction st (`DB.execute_` "INSERT INTO users DEFAULT VALUES")
tests/PostgresSchemaDump.hs
+3 -3
View File
@@ -14,7 +14,7 @@ import Data.Maybe (fromJust, isJust)
import Simplex.Messaging.Agent.Store.Postgres (closeDBStore, createDBStore)
import Simplex.Messaging.Agent.Store.Postgres.Common (DBOpts (..))
import qualified Simplex.Messaging.Agent.Store.Postgres.Migrations as Migrations
import Simplex.Messaging.Agent.Store.Shared (Migration (..), MigrationConfirmation (..), MigrationsToRun (..), toDownMigration)
import Simplex.Messaging.Agent.Store.Shared (Migration (..), MigrationConfig (..), MigrationConfirmation (..), MigrationsToRun (..), toDownMigration)
import Simplex.Messaging.Util (ifM, whenM)
import System.Directory (doesFileExist, removeFile)
import System.Process (readCreateProcess, shell)
@@ -32,12 +32,12 @@ postgresSchemaDumpTest migrations skipComparisonForDownMigrations testDBOpts@DBO
testVerifySchemaDump = do
savedSchema <- ifM (doesFileExist srcSchemaPath) (readFile srcSchemaPath) (pure "")
savedSchema `deepseq` pure ()
void $ createDBStore testDBOpts migrations MCConsole
void $ createDBStore testDBOpts migrations (MigrationConfig MCConsole Nothing)
getSchema srcSchemaPath `shouldReturn` savedSchema
testSchemaMigrations = do
let noDownMigrations = dropWhileEnd (\Migration {down} -> isJust down) migrations
st <- createDBStore testDBOpts noDownMigrations MCYesUpDown >>= \case
st <- createDBStore testDBOpts noDownMigrations (MigrationConfig MCYesUpDown Nothing) >>= \case
Right st -> pure st
Left e -> error $ show e
mapM_ (testDownMigration st) $ drop (length noDownMigrations) migrations