mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-08-23 18:39:48 +00:00
Merge branch 'stable' into stable-android-new
# Conflicts: # .github/workflows/build.yml # flake.nix # scripts/desktop/build-lib-linux.sh
This commit is contained in:
shum
@@ -23,5 +23,5 @@ unzip -o "$tmp/libsimplex.zip" -d "$tmp/simplex-chat/apps/multiplatform/common/s
|
||||
curl -sSf "$libsup" -o "$tmp/libsupport.zip"
|
||||
unzip -o "$tmp/libsupport.zip" -d "$tmp/simplex-chat/apps/multiplatform/common/src/commonMain/cpp/android/libs/arm64-v8a"
|
||||
|
||||
gradle -p "$tmp/simplex-chat/apps/multiplatform/" clean build
|
||||
gradle -p "$tmp/simplex-chat/apps/multiplatform/" -Psimplex.assets.dir=../../assets clean build
|
||||
cp "$tmp/simplex-chat/apps/multiplatform/android/build/outputs/apk/release/android-release-unsigned.apk" "$PWD/simplex-chat.apk"
|
||||
|
||||
@@ -67,13 +67,13 @@ checks() {
|
||||
if ! command -v "$i" > /dev/null 2>&1; then
|
||||
commands_failed="$i $commands_failed"
|
||||
else
|
||||
gradle_ver_local="$(gradle -v | grep Gradle | awk '{print $2}')"
|
||||
gradle_ver_local_compare="$(printf ${gradle_ver_local:-0.0} | awk -F. '{print $1$2}')"
|
||||
gradle_ver_local="$(gradle --version | sed -n 's/^Gradle //p')"
|
||||
gradle_ver_local_compare="$(printf '%s' "$gradle_ver_local" | awk -F. '{print $1"."$2}')"
|
||||
gradle_ver_remote="$(grep distributionUrl ${folder}/apps/multiplatform/gradle/wrapper/gradle-wrapper.properties)"
|
||||
gradle_ver_remote="${gradle_ver_remote#*-}"
|
||||
gradle_ver_remote="${gradle_ver_remote%-*}"
|
||||
gradle_ver_remote_compare="$(printf ${gradle_ver_remote} | awk -F. '{print $1$2}')"
|
||||
|
||||
gradle_ver_remote_compare="$(printf '%s' "$gradle_ver_remote" | awk -F. '{print $1"."$2}')"
|
||||
|
||||
if [ "$gradle_ver_local_compare" != "$gradle_ver_remote_compare" ]; then
|
||||
commands_failed="$i[installed=${gradle_ver_local},required=${gradle_ver_remote}] $commands_failed"
|
||||
fi
|
||||
@@ -102,6 +102,7 @@ build() {
|
||||
sed -i.bak 's/jniLibs.useLegacyPackaging =.*/jniLibs.useLegacyPackaging = true/' "$folder/apps/multiplatform/android/build.gradle.kts"
|
||||
sed -i.bak '/android {/a lint {abortOnError = false}' "$folder/apps/multiplatform/android/build.gradle.kts"
|
||||
sed -i.bak '/tasks/Q' "$folder/apps/multiplatform/android/build.gradle.kts"
|
||||
sed -i.bak "s/android.version_code=.*/android.version_code=${vercode}/" "$folder/apps/multiplatform/gradle.properties"
|
||||
|
||||
for arch in $arches; do
|
||||
if [ "$arch" = "armv7a" ]; then
|
||||
@@ -133,15 +134,20 @@ build() {
|
||||
|
||||
# Build only one arch
|
||||
sed -i.bak "s/include(.*/include(\"${android_arch}\")/" "$folder/apps/multiplatform/android/build.gradle.kts"
|
||||
gradle -p "$folder/apps/multiplatform/" clean :android:assembleRelease
|
||||
gradle -p "$folder/apps/multiplatform/" -Psimplex.assets.dir=../../assets clean :android:assembleRelease
|
||||
|
||||
mkdir -p "$android_tmp_folder"
|
||||
unzip -oqd "$android_tmp_folder" "$android_apk_output"
|
||||
|
||||
# Determenistic build
|
||||
find "$android_tmp_folder" -type f -exec chmod 644 {} +
|
||||
find "$android_tmp_folder" -type d -exec chmod 755 {} +
|
||||
find "$android_tmp_folder" -exec touch -h -d '@1764547200' {} +
|
||||
|
||||
(
|
||||
cd "$android_tmp_folder" && \
|
||||
zip -rq5 "$tmp/$android_apk_output_final" . && \
|
||||
zip -rq0 "$tmp/$android_apk_output_final" resources.arsc res
|
||||
find . -type f -print0 | sort -z | xargs -0 zip -X -rq5 "$tmp/$android_apk_output_final" && \
|
||||
find res resources.arsc -type f -print0 | sort -z | xargs -0 zip -X -rq0 "$tmp/$android_apk_output_final"
|
||||
)
|
||||
|
||||
zipalign -p -f 4 "$tmp/$android_apk_output_final" "$PWD/$android_apk_output_final"
|
||||
@@ -164,8 +170,10 @@ pre() {
|
||||
done
|
||||
|
||||
shift $(( $OPTIND - 1 ))
|
||||
|
||||
commit="${1:-HEAD}"
|
||||
|
||||
vercode="${1}"
|
||||
|
||||
commit="${2:-HEAD}"
|
||||
}
|
||||
|
||||
main() {
|
||||
|
||||
@@ -29,20 +29,43 @@ for ORIG_NAME in "${ORIG_NAMES[@]}"; do
|
||||
ORIG_NAME_COPY=$ORIG_NAME-copy
|
||||
mv "$ORIG_NAME" "$ORIG_NAME_COPY"
|
||||
|
||||
(cd apk && zip -r -q -"$level" ../"$ORIG_NAME" .)
|
||||
# Shouldn't be compressed because of Android requirement
|
||||
(cd apk && zip -r -q -0 ../"$ORIG_NAME" resources.arsc)
|
||||
# Determenistic build
|
||||
find apk -type f -exec chmod 644 {} +
|
||||
find apk -type d -exec chmod 755 {} +
|
||||
find apk -exec touch -h -d '2025-12-01T00:00:00' {} +
|
||||
|
||||
(
|
||||
cd apk
|
||||
find . -not -path './res/*' -not -name 'resources.arsc' -type f -print0 | sort -z | xargs -0 zip -X -r -q -"$level" ../"$ORIG_NAME"
|
||||
)
|
||||
|
||||
if [ $case_insensitive -eq 1 ]; then
|
||||
# For case-insensitive file systems
|
||||
list_of_files=$(unzip -l "$ORIG_NAME_COPY" | grep res/ | sed -e "s|.*res/|res/|")
|
||||
for file in $list_of_files; do unzip -o -q -d apk "$ORIG_NAME_COPY" "$file" && (cd apk && zip -r -q -0 ../"$ORIG_NAME" "$file"); done
|
||||
list_of_files=$(unzip -l "$ORIG_NAME_COPY" | grep res/ | sed -e "s|.*res/|res/|" | sort)
|
||||
for file in $list_of_files; do
|
||||
unzip -o -q -d apk "$ORIG_NAME_COPY" "$file"
|
||||
(
|
||||
cd apk
|
||||
chmod 644 "$file"
|
||||
touch -h -d '2025-12-01T00:00:00' "$file"
|
||||
zip -X -r -q -0 ../"$ORIG_NAME" "$file"
|
||||
)
|
||||
done
|
||||
else
|
||||
# This method is not working correctly on case-insensitive file systems since Android AAPT produce the same names of files
|
||||
# but with different case like xX.png, Xx.png, xx.png, etc
|
||||
(cd apk && zip -r -q -0 ../"$ORIG_NAME" res)
|
||||
(
|
||||
cd apk
|
||||
find res -type f -print0 | sort -z | xargs -0 zip -X -r -q -0 ../"$ORIG_NAME"
|
||||
)
|
||||
fi
|
||||
|
||||
# Shouldn't be compressed because of Android requirement
|
||||
(
|
||||
cd apk
|
||||
find resources.arsc -type f -print0 | sort -z | xargs -0 zip -X -r -q -0 ../"$ORIG_NAME"
|
||||
)
|
||||
|
||||
#(cd apk && 7z a -r -mx=$level -tzip -x!resources.arsc ../$ORIG_NAME .)
|
||||
#(cd apk && 7z a -r -mx=0 -tzip ../$ORIG_NAME resources.arsc)
|
||||
|
||||
@@ -61,4 +84,4 @@ for ORIG_NAME in "${ORIG_NAMES[@]}"; do
|
||||
rm "$ORIG_NAME_COPY" 2> /dev/null || true
|
||||
rm -rf apk || true
|
||||
rm "${ORIG_NAME}".idsig 2> /dev/null || true
|
||||
done
|
||||
done
|
||||
|
||||
+248
-62
@@ -1,96 +1,238 @@
|
||||
# Transfer data from SQLite to Postgres database
|
||||
## Transfer data from SQLite to Postgres database
|
||||
|
||||
1. \* Decrypt SQLite database if it is encrypted.
|
||||
1. Decrypt SQLite database if it is encrypted.
|
||||
|
||||
```sh
|
||||
sqlcipher encrypted_simplex_v1_agent.db
|
||||
```
|
||||
1. Agent:
|
||||
|
||||
```sql
|
||||
PRAGMA key = 'password';
|
||||
ATTACH DATABASE 'simplex_v1_agent.db' AS plaintext KEY '';
|
||||
SELECT sqlcipher_export('plaintext');
|
||||
DETACH DATABASE plaintext;
|
||||
```
|
||||
1. Open sqlite db:
|
||||
|
||||
Repeat for `simplex_v1_chat.db`.
|
||||
```sh
|
||||
sqlcipher simplex_v1_agent.db
|
||||
```
|
||||
|
||||
2. Run in sqlcipher:
|
||||
|
||||
```sql
|
||||
PRAGMA key = '<your_password>'; -- Set your db password
|
||||
SELECT count(*) FROM sqlite_master; -- Check if db was successfully decrypted
|
||||
ATTACH DATABASE 'simplex_v1_agent_plaintext.db' AS plaintext KEY ''; -- Attach new empty db
|
||||
SELECT sqlcipher_export('plaintext'); -- Export opened db to attached db as plaintext
|
||||
DETACH DATABASE plaintext;
|
||||
```
|
||||
|
||||
2. Chat:
|
||||
|
||||
1. Open sqlite db:
|
||||
|
||||
```sh
|
||||
sqlcipher simplex_v1_chat.db
|
||||
```
|
||||
|
||||
2. Run in sqlcipher:
|
||||
|
||||
```sql
|
||||
PRAGMA key = '<your_password>';
|
||||
SELECT count(*) FROM sqlite_master;
|
||||
ATTACH DATABASE 'simplex_v1_chat_plaintext.db' AS plaintext KEY '';
|
||||
SELECT sqlcipher_export('plaintext');
|
||||
DETACH DATABASE plaintext;
|
||||
```
|
||||
|
||||
2. Prepare Postgres database.
|
||||
|
||||
- Create Postgres database. In shell:
|
||||
1. Connect to PostgreSQL databse:
|
||||
|
||||
```sh
|
||||
createdb -O simplex simplex_v1
|
||||
psql -U postgres -h localhost
|
||||
```
|
||||
|
||||
Or via query.
|
||||
2. Run in psql:
|
||||
|
||||
- Build `simplex-chat` executable with `client_postgres` flag and run it to initialize new chat database.
|
||||
```sql
|
||||
CREATE USER simplex WITH ENCRYPTED PASSWORD '123123'; -- Create user with password
|
||||
-- or
|
||||
-- CREATE USER simplex;
|
||||
CREATE DATABASE simplex_v1; -- Create database
|
||||
GRANT ALL PRIVILEGES ON DATABASE simplex_v1 TO simplex; -- Assign permissions
|
||||
```
|
||||
|
||||
This should create `simplex_v1_agent_schema` and `simplex_v1_chat_schema` schemas in `simplex_v1` database, with `migrations` tables populated. Some tables would have initialization data - it will be truncated via pgloader command in next step.
|
||||
3. Prepare database:
|
||||
|
||||
You should build the CLI binary from the same `TAG` as the desktop.
|
||||
|
||||
1. Build CLI with PostgreSQL support:
|
||||
|
||||
```sh
|
||||
cabal build -fclient_postgres exe:simplex-chat
|
||||
```
|
||||
|
||||
And rename it to:
|
||||
|
||||
```sh
|
||||
mv simplex-chat simplex-chat-pg
|
||||
```
|
||||
|
||||
2. Execute CLI:
|
||||
|
||||
```sh
|
||||
./simplex-chat-pg -d "postgresql://simplex:123123@localhost:5432/simplex_v1" --create-schema
|
||||
```
|
||||
|
||||
Press `Ctrl+C` when CLI ask for a display name.
|
||||
|
||||
This should create `simplex_v1_agent_schema` and `simplex_v1_chat_schema` schemas in `simplex_v1` database, with `migrations` tables populated. Some tables would have initialization data - it will be truncated via pgloader command in next step.
|
||||
|
||||
3. Load data from decrypted SQLite databases to Postgres database via pgloader.
|
||||
|
||||
Install pgloader and add it to PATH. Run in shell (substitute paths):
|
||||
|
||||
```sh
|
||||
SQLITE_DBPATH='simplex_v1_agent.db' POSTGRES_CONN='postgres://simplex@/simplex_v1' POSTGRES_SCHEMA='simplex_v1_agent_schema' pgloader --on-error-stop sqlite.load
|
||||
export POSTGRES_CONN='postgresql://simplex:123123@localhost:5432/simplex_v1'
|
||||
```
|
||||
|
||||
SQLITE_DBPATH='simplex_v1_chat.db' POSTGRES_CONN='postgres://simplex@/simplex_v1' POSTGRES_SCHEMA='simplex_v1_chat_schema' pgloader --on-error-stop sqlite.load
|
||||
And then:
|
||||
|
||||
```sh
|
||||
SQLITE_DBPATH='simplex_v1_agent_plaintext.db' \
|
||||
POSTGRES_SCHEMA='simplex_v1_agent_schema' \
|
||||
CPU_CORES=$(nproc) WORKERS=$((CPU_CORES - 1)) pgloader --dynamic-space-size 262144 --on-error-stop sqlite.load
|
||||
|
||||
SQLITE_DBPATH='simplex_v1_chat_plaintext.db' \
|
||||
POSTGRES_SCHEMA='simplex_v1_chat_schema' \
|
||||
CPU_CORES=$(nproc) WORKERS=$((CPU_CORES - 1)) pgloader --dynamic-space-size 262144 --on-error-stop sqlite.load
|
||||
```
|
||||
|
||||
4. Update sequences for Postgres tables.
|
||||
|
||||
```sql
|
||||
DO $$
|
||||
DECLARE
|
||||
rec RECORD;
|
||||
BEGIN
|
||||
EXECUTE 'SET SEARCH_PATH TO simplex_v1_agent_schema';
|
||||
Connect to db:
|
||||
|
||||
FOR rec IN
|
||||
SELECT
|
||||
table_name,
|
||||
column_name,
|
||||
pg_get_serial_sequence(table_name, column_name) AS seq_name
|
||||
FROM
|
||||
information_schema.columns
|
||||
WHERE
|
||||
table_schema = 'simplex_v1_agent_schema'
|
||||
AND identity_generation = 'ALWAYS'
|
||||
LOOP
|
||||
EXECUTE format(
|
||||
'SELECT setval(%L, (SELECT MAX(%I) FROM %I))',
|
||||
rec.seq_name, rec.column_name, rec.table_name
|
||||
);
|
||||
END LOOP;
|
||||
END $$;
|
||||
```sh
|
||||
PGPASSWORD=123123 psql -h localhost -U simplex -d simplex_v1
|
||||
```
|
||||
|
||||
Repeat for `simplex_v1_chat_schema`.
|
||||
Execute the following:
|
||||
|
||||
5. \* Compare number of rows between Postgres and SQLite tables.
|
||||
1. For `agent`:
|
||||
|
||||
To check number of rows for all tables in Postgres database schema run:
|
||||
```sql
|
||||
DO $$
|
||||
DECLARE
|
||||
rec RECORD;
|
||||
BEGIN
|
||||
EXECUTE 'SET SEARCH_PATH TO simplex_v1_agent_schema';
|
||||
|
||||
```sql
|
||||
WITH tbl AS (
|
||||
SELECT table_schema, table_name
|
||||
FROM information_schema.Tables
|
||||
WHERE table_name NOT LIKE 'pg_%'
|
||||
AND table_schema IN ('simplex_v1_agent_schema')
|
||||
)
|
||||
SELECT
|
||||
table_schema AS schema_name,
|
||||
table_name,
|
||||
(xpath('/row/c/text()', query_to_xml(
|
||||
format('SELECT count(*) AS c FROM %I.%I', table_schema, table_name), false, true, ''
|
||||
)))[1]::text::int AS records_count
|
||||
FROM tbl
|
||||
ORDER BY records_count DESC;
|
||||
```
|
||||
FOR rec IN
|
||||
SELECT
|
||||
table_name,
|
||||
column_name,
|
||||
pg_get_serial_sequence(table_name, column_name) AS seq_name
|
||||
FROM
|
||||
information_schema.columns
|
||||
WHERE
|
||||
table_schema = 'simplex_v1_agent_schema'
|
||||
AND identity_generation = 'ALWAYS'
|
||||
LOOP
|
||||
EXECUTE format(
|
||||
'SELECT setval(%L, (SELECT MAX(%I) FROM %I))',
|
||||
rec.seq_name, rec.column_name, rec.table_name
|
||||
);
|
||||
END LOOP;
|
||||
END $$;
|
||||
```
|
||||
|
||||
Repeat for `simplex_v1_chat_schema`.
|
||||
2. For `chat`:
|
||||
|
||||
```sql
|
||||
DO $$
|
||||
DECLARE
|
||||
rec RECORD;
|
||||
BEGIN
|
||||
EXECUTE 'SET SEARCH_PATH TO simplex_v1_chat_schema';
|
||||
|
||||
FOR rec IN
|
||||
SELECT
|
||||
table_name,
|
||||
column_name,
|
||||
pg_get_serial_sequence(table_name, column_name) AS seq_name
|
||||
FROM
|
||||
information_schema.columns
|
||||
WHERE
|
||||
table_schema = 'simplex_v1_chat_schema'
|
||||
AND identity_generation = 'ALWAYS'
|
||||
LOOP
|
||||
EXECUTE format(
|
||||
'SELECT setval(%L, (SELECT MAX(%I) FROM %I))',
|
||||
rec.seq_name, rec.column_name, rec.table_name
|
||||
);
|
||||
END LOOP;
|
||||
END $$;
|
||||
```
|
||||
|
||||
5. Compare number of rows between Postgres and SQLite tables.
|
||||
|
||||
**PostgreSQL**:
|
||||
|
||||
1. For `agent`:
|
||||
|
||||
```sql
|
||||
WITH tbl AS (
|
||||
SELECT table_schema, table_name
|
||||
FROM information_schema.Tables
|
||||
WHERE table_name NOT LIKE 'pg_%'
|
||||
AND table_schema IN ('simplex_v1_agent_schema')
|
||||
)
|
||||
SELECT
|
||||
table_schema AS schema_name,
|
||||
table_name,
|
||||
(xpath('/row/c/text()', query_to_xml(
|
||||
format('SELECT count(*) AS c FROM %I.%I', table_schema, table_name), false, true, ''
|
||||
)))[1]::text::int AS records_count
|
||||
FROM tbl
|
||||
ORDER BY records_count DESC;
|
||||
```
|
||||
|
||||
2. For `chat`:
|
||||
|
||||
```sql
|
||||
WITH tbl AS (
|
||||
SELECT table_schema, table_name
|
||||
FROM information_schema.Tables
|
||||
WHERE table_name NOT LIKE 'pg_%'
|
||||
AND table_schema IN ('simplex_v1_chat_schema')
|
||||
)
|
||||
SELECT
|
||||
table_schema AS schema_name,
|
||||
table_name,
|
||||
(xpath('/row/c/text()', query_to_xml(
|
||||
format('SELECT count(*) AS c FROM %I.%I', table_schema, table_name), false, true, ''
|
||||
)))[1]::text::int AS records_count
|
||||
FROM tbl
|
||||
ORDER BY records_count DESC;
|
||||
```
|
||||
|
||||
**SQLite**:
|
||||
|
||||
1. For `agent`:
|
||||
|
||||
```sh
|
||||
db="simplex_v1_agent_plaintext.db"
|
||||
sqlite3 "$db" "SELECT name FROM sqlite_master WHERE type='table';" |
|
||||
while read table; do
|
||||
count=$(sqlite3 "$db" "SELECT COUNT(*) FROM \"$table\";")
|
||||
echo "$table: $count"
|
||||
done | sort -k2 -nr | less
|
||||
```
|
||||
|
||||
2. For `chat`:
|
||||
|
||||
```sh
|
||||
db="simplex_v1_chat_plaintext.db"
|
||||
sqlite3 "$db" "SELECT name FROM sqlite_master WHERE type='table';" |
|
||||
while read table; do
|
||||
count=$(sqlite3 "$db" "SELECT COUNT(*) FROM \"$table\";")
|
||||
echo "$table: $count"
|
||||
done | sort -k2 -nr | less
|
||||
```
|
||||
|
||||
6. Build and run desktop app with Postgres backend.
|
||||
|
||||
@@ -103,3 +245,47 @@
|
||||
# or
|
||||
./gradlew packageDmg -Pdatabase.backend=postgres
|
||||
```
|
||||
|
||||
## Transfer data from Postgres to SQLite database
|
||||
|
||||
1. Prepare sqlite db:
|
||||
|
||||
1. Download simplex-chat CLI:
|
||||
|
||||
You should download the CLI binary from the same `TAG` as the desktop.
|
||||
|
||||
```sh
|
||||
export TAG='v6.4.3.1'
|
||||
curl -L "https://github.com/simplex-chat/simplex-chat/releases/download/${TAG}/simplex-chat-ubuntu-22_04-x86_64" -o 'simplex-chat'
|
||||
```
|
||||
|
||||
2. Run the CLI:
|
||||
|
||||
```sh
|
||||
./simplex-chat
|
||||
```
|
||||
|
||||
Press `Ctrl+C` when CLI ask for a display name.
|
||||
|
||||
3. Move database:
|
||||
|
||||
```sh
|
||||
mv ~/.simplex/simplex_v1_* ~/.local/share/simplex/
|
||||
```
|
||||
|
||||
2. Transfer data:
|
||||
|
||||
```sh
|
||||
./pg2sqlite.py --verbose 'postgresql://simplex:123123@localhost:5432/simplex_v1' ~/.local/share/simplex/
|
||||
```
|
||||
|
||||
4. Update BLOBs:
|
||||
|
||||
```sh
|
||||
sqlite3 simplex_v1_chat.db
|
||||
```
|
||||
|
||||
```sh
|
||||
UPDATE group_members SET member_role = CAST(member_role as BLOB);
|
||||
UPDATE user_contact_links SET group_link_member_role = CAST(group_link_member_role AS BLOB) WHERE group_link_member_role is not null;
|
||||
```
|
||||
|
||||
Executable
+899
@@ -0,0 +1,899 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
PostgreSQL -> per-schema SQLite migration with colored, clean logging.
|
||||
|
||||
Usage example:
|
||||
python db_migrate.py 'postgresql://user:pass@host:5432/db' /path/to/sqlite/dir --dry-run
|
||||
|
||||
Note: color output will be disabled automatically if stdout is not a TTY or if --no-color is passed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import re
|
||||
import datetime
|
||||
from pathlib import Path
|
||||
from typing import Set, List, Tuple, Dict, Optional, NamedTuple
|
||||
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
|
||||
# ----------------------
|
||||
# Small utilities
|
||||
# ----------------------
|
||||
|
||||
try:
|
||||
sqlite3.register_adapter(
|
||||
datetime.datetime,
|
||||
lambda v: v.isoformat(sep=" ", timespec="microseconds"),
|
||||
)
|
||||
sqlite3.register_adapter(datetime.date, lambda v: v.isoformat())
|
||||
sqlite3.register_adapter(
|
||||
datetime.time, lambda v: v.isoformat(timespec="microseconds")
|
||||
)
|
||||
except ValueError:
|
||||
pass # already registered
|
||||
|
||||
ANSI = {
|
||||
"reset": "\x1b[0m",
|
||||
"bold": "\x1b[1m",
|
||||
"dim": "\x1b[2m",
|
||||
"red": "\x1b[31m",
|
||||
"green": "\x1b[32m",
|
||||
"yellow": "\x1b[33m",
|
||||
"blue": "\x1b[34m",
|
||||
"magenta": "\x1b[35m",
|
||||
"cyan": "\x1b[36m",
|
||||
"gray": "\x1b[90m",
|
||||
}
|
||||
|
||||
DEFAULT_BATCH_SIZE = 10000
|
||||
|
||||
TYPE_COMPATIBILITY = {
|
||||
"bytea": ["BLOB", "CHAR", "CLOB", "TEXT", "JSON"],
|
||||
"int": ["INT", "NUMERIC"],
|
||||
"serial": ["INT", "NUMERIC"],
|
||||
"numeric": ["NUMERIC", "DECIMAL", "REAL", "FLOAT", "DOUBLE"],
|
||||
"decimal": ["NUMERIC", "DECIMAL", "REAL", "FLOAT", "DOUBLE"],
|
||||
"real": ["NUMERIC", "DECIMAL", "REAL", "FLOAT", "DOUBLE"],
|
||||
"double": ["NUMERIC", "DECIMAL", "REAL", "FLOAT", "DOUBLE"],
|
||||
"float": ["NUMERIC", "DECIMAL", "REAL", "FLOAT", "DOUBLE"],
|
||||
"money": ["NUMERIC", "DECIMAL", "REAL", "FLOAT", "DOUBLE"],
|
||||
"bool": ["BOOL", "INT", "NUMERIC"],
|
||||
"varchar": ["CHAR", "CLOB", "TEXT"],
|
||||
"char": ["CHAR", "CLOB", "TEXT"],
|
||||
"text": ["CHAR", "CLOB", "TEXT"],
|
||||
"citext": ["CHAR", "CLOB", "TEXT"],
|
||||
"timestamp": ["DATE", "TIME", "CHAR", "TEXT", "DATETIME"],
|
||||
"time": ["DATE", "TIME", "CHAR", "TEXT", "DATETIME"],
|
||||
"date": ["DATE", "TIME", "CHAR", "TEXT", "DATETIME"],
|
||||
"uuid": ["CHAR", "TEXT", "UUID", "CLOB"],
|
||||
"json": ["JSON", "TEXT", "CHAR", "CLOB"],
|
||||
"jsonb": ["JSON", "TEXT", "CHAR", "CLOB"],
|
||||
}
|
||||
|
||||
|
||||
def _sanitize_cursor_name(s: str) -> str:
|
||||
return re.sub(r"[^A-Za-z0-9_]+", "_", s)
|
||||
|
||||
|
||||
def supports_color(force_no: bool) -> bool:
|
||||
"""Return True when we should emit ANSI colors."""
|
||||
if force_no:
|
||||
return False
|
||||
if os.getenv("NO_COLOR"):
|
||||
return False
|
||||
term = os.getenv("TERM", "")
|
||||
if term == "" or term.lower() == "dumb":
|
||||
return False
|
||||
try:
|
||||
isatty = sys.stdout.isatty()
|
||||
except Exception:
|
||||
isatty = False
|
||||
return isatty
|
||||
|
||||
|
||||
class ColoredFormatter(logging.Formatter):
|
||||
LEVEL_COLORS = {
|
||||
logging.DEBUG: ANSI["gray"],
|
||||
logging.INFO: ANSI["green"],
|
||||
logging.WARNING: ANSI["yellow"],
|
||||
logging.ERROR: ANSI["red"],
|
||||
logging.CRITICAL: ANSI["red"] + ANSI["bold"],
|
||||
}
|
||||
|
||||
TAG_COLORS = {
|
||||
"SKIP": ANSI["yellow"],
|
||||
"SCHEMA": ANSI["blue"],
|
||||
"OK": ANSI["magenta"],
|
||||
}
|
||||
|
||||
def __init__(self, use_color: bool = True):
|
||||
super().__init__(fmt="%(message)s")
|
||||
self.use_color = use_color
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
msg = super().format(record)
|
||||
parts = msg.split(" ", 1)
|
||||
tag = parts[0]
|
||||
rest = parts[1] if len(parts) > 1 else ""
|
||||
plain_label = f"[{tag}]"
|
||||
|
||||
if not self.use_color:
|
||||
return f"{plain_label}{(' ' + rest) if rest else ''}"
|
||||
|
||||
color = self.TAG_COLORS.get(
|
||||
tag.upper(), self.LEVEL_COLORS.get(record.levelno, "")
|
||||
)
|
||||
reset = ANSI["reset"]
|
||||
return f"{color}{plain_label}{reset}{(' ' + rest) if rest else ''}"
|
||||
|
||||
|
||||
def setup_logger(verbose: bool, no_color: bool) -> logging.Logger:
|
||||
use_color = supports_color(no_color)
|
||||
logger = logging.getLogger("db_migrate")
|
||||
logger.setLevel(logging.DEBUG if verbose else logging.INFO)
|
||||
handler = logging.StreamHandler()
|
||||
handler.setLevel(logging.DEBUG if verbose else logging.INFO)
|
||||
handler.setFormatter(ColoredFormatter(use_color=use_color))
|
||||
logger.handlers.clear()
|
||||
logger.addHandler(handler)
|
||||
logging.getLogger("psycopg").setLevel(logging.WARNING)
|
||||
|
||||
if verbose:
|
||||
logger.debug(f"color_support: {use_color}")
|
||||
try:
|
||||
isatty = sys.stdout.isatty()
|
||||
except Exception:
|
||||
isatty = False
|
||||
logger.debug(
|
||||
f"TERM={os.getenv('TERM', '')!r} "
|
||||
f"NO_COLOR={os.getenv('NO_COLOR')!r} "
|
||||
f"isatty={isatty}"
|
||||
)
|
||||
return logger
|
||||
|
||||
|
||||
def quote_sqlite_identifier(name: str) -> str:
|
||||
return '"' + name.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def quote_pg_identifier(name: str) -> str:
|
||||
"""Simple PG identifier quoting for building safe SQL strings."""
|
||||
return '"' + name.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def sqlite_decl_satisfies(pg_type: str, sqlite_decl: str) -> bool:
|
||||
# Treat empty/blank SQLite declarations more permissively based on PG type.
|
||||
decl_raw = sqlite_decl or ""
|
||||
if not decl_raw.strip():
|
||||
pg = (pg_type or "").lower()
|
||||
# arrays -> textual affinity
|
||||
if pg.endswith("[]"):
|
||||
return True
|
||||
# integer-like
|
||||
if re.search(r"\b(?:int|serial|bigint)\b", pg):
|
||||
return True
|
||||
# numeric/float
|
||||
if re.search(r"\b(?:numeric|decimal|real|double|float|money)\b", pg):
|
||||
return True
|
||||
# boolean
|
||||
if re.search(r"\b(?:bool|boolean)\b", pg):
|
||||
return True
|
||||
# binary
|
||||
if re.search(r"\b(?:bytea)\b", pg):
|
||||
return True
|
||||
# textual/json/uuid/timestamps/dates/times
|
||||
if re.search(
|
||||
r"\b(?:varchar|char|text|citext|jsonb|json|uuid|timestamp|time|date)\b", pg
|
||||
):
|
||||
return True
|
||||
# conservative fallback: accept empty decl as permissive
|
||||
return True
|
||||
|
||||
decl = decl_raw.upper()
|
||||
pg = (pg_type or "").lower()
|
||||
|
||||
# array type
|
||||
if pg.endswith("[]"):
|
||||
return any(tok in decl for tok in ("TEXT", "CHAR", "CLOB", "JSON"))
|
||||
|
||||
for key, allowed_types in TYPE_COMPATIBILITY.items():
|
||||
if re.search(r"\b" + re.escape(key) + r"\b", pg):
|
||||
return any(tok in decl for tok in allowed_types)
|
||||
|
||||
return any(
|
||||
tok in decl for tok in ("TEXT", "CHAR", "CLOB", "NUMERIC", "BLOB", "INT")
|
||||
)
|
||||
|
||||
|
||||
# ----------------------
|
||||
# Postgres helpers
|
||||
# ----------------------
|
||||
|
||||
|
||||
def list_user_schemas(pg_cursor) -> List[str]:
|
||||
pg_cursor.execute(
|
||||
"""
|
||||
SELECT nspname
|
||||
FROM pg_namespace
|
||||
WHERE nspname NOT LIKE 'pg_%'
|
||||
AND nspname != 'information_schema'
|
||||
AND nspname != 'public'
|
||||
ORDER BY nspname;
|
||||
"""
|
||||
)
|
||||
return [r[0] for r in pg_cursor.fetchall()]
|
||||
|
||||
|
||||
def list_tables_in_schema(pg_cursor, schema: str) -> List[str]:
|
||||
pg_cursor.execute(
|
||||
sql.SQL(
|
||||
"""
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = %s
|
||||
ORDER BY table_name;
|
||||
"""
|
||||
),
|
||||
(schema,),
|
||||
)
|
||||
return [r[0] for r in pg_cursor.fetchall()]
|
||||
|
||||
|
||||
def get_pg_columns(pg_cursor, schema: str, table: str) -> List[Tuple[str, str]]:
|
||||
pg_cursor.execute(
|
||||
sql.SQL(
|
||||
"""
|
||||
SELECT column_name, data_type
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = %s AND table_name = %s
|
||||
ORDER BY ordinal_position;
|
||||
"""
|
||||
),
|
||||
(schema, table),
|
||||
)
|
||||
return [(r[0], r[1]) for r in pg_cursor.fetchall()]
|
||||
|
||||
|
||||
# ----------------------
|
||||
# SQLite helpers
|
||||
# ----------------------
|
||||
|
||||
|
||||
class SQLiteCol(NamedTuple):
|
||||
cid: int
|
||||
name: str
|
||||
type: str
|
||||
notnull: int
|
||||
dflt_value: str
|
||||
pk: int
|
||||
|
||||
|
||||
def get_sqlite_table_info(sqlite_cursor, table_name: str) -> list[SQLiteCol]:
|
||||
qname = quote_sqlite_identifier(table_name)
|
||||
sqlite_cursor.execute(f"PRAGMA table_info({qname});")
|
||||
return [SQLiteCol(*row) for row in sqlite_cursor.fetchall()]
|
||||
|
||||
|
||||
def sqlite_table_has_autoincrement(sqlite_cursor, table_name: str) -> bool:
|
||||
sqlite_cursor.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE type='table' AND name = ?;", (table_name,)
|
||||
)
|
||||
row = sqlite_cursor.fetchone()
|
||||
if not row or not row[0]:
|
||||
return False
|
||||
return "AUTOINCREMENT" in row[0].upper()
|
||||
|
||||
|
||||
def sqlite_sequence_table_exists(sqlite_cursor) -> bool:
|
||||
sqlite_cursor.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='sqlite_sequence';"
|
||||
)
|
||||
exists = sqlite_cursor.fetchone() is not None
|
||||
if not exists:
|
||||
logging.getLogger("db_migrate").debug("sqlite_sequence table not present")
|
||||
return exists
|
||||
|
||||
|
||||
# ----------------------
|
||||
# Core migration logic
|
||||
# ----------------------
|
||||
|
||||
|
||||
def build_select_for_sqlite_columns(
|
||||
schema: str,
|
||||
table: str,
|
||||
sqlite_cols: list[SQLiteCol],
|
||||
pg_columns_map: Dict[str, Tuple[str, str]],
|
||||
) -> str:
|
||||
"""Return a plain SQL string (no trailing semicolon) selecting PG columns in the order of sqlite_cols.
|
||||
Uses simple identifier quoting to avoid psycopg.sql objects which may vary by driver version.
|
||||
"""
|
||||
select_parts = []
|
||||
for col in sqlite_cols:
|
||||
lc = col.name.lower()
|
||||
if lc in pg_columns_map:
|
||||
pg_name, pg_type = pg_columns_map[lc]
|
||||
pg_type_l = (pg_type or "").lower()
|
||||
if any(tok in pg_type_l for tok in ("timestamp", "time")):
|
||||
expr = f"{quote_pg_identifier(pg_name)}::text"
|
||||
else:
|
||||
expr = f"{quote_pg_identifier(pg_name)}"
|
||||
select_parts.append(expr)
|
||||
else:
|
||||
select_parts.append("NULL")
|
||||
select_list = ", ".join(select_parts)
|
||||
# Quote schema and table names simply (this is not comprehensive for every corner case but avoids psycopg.sql use)
|
||||
return f"SELECT {select_list} FROM {quote_pg_identifier(schema)}.{quote_pg_identifier(table)}"
|
||||
|
||||
|
||||
def validate_column_compatibility(
|
||||
sqlite_cols: list[SQLiteCol],
|
||||
pg_columns_map: Dict[str, Tuple[str, str]],
|
||||
schema: str,
|
||||
table: str,
|
||||
) -> None:
|
||||
for col in sqlite_cols:
|
||||
lc = col.name.lower()
|
||||
if lc not in pg_columns_map:
|
||||
continue
|
||||
pg_type = pg_columns_map[lc][1]
|
||||
if not sqlite_decl_satisfies(pg_type, col.type):
|
||||
raise ValueError(
|
||||
f"Type mismatch for {schema}.{table}.{col.name}: "
|
||||
f"PG='{pg_type}' vs SQLite='{col.type}'"
|
||||
)
|
||||
|
||||
|
||||
def process_row_for_sqlite(row: Tuple, sqlite_cols: list[SQLiteCol]) -> Tuple:
|
||||
out = []
|
||||
for i, val in enumerate(row):
|
||||
col = sqlite_cols[i]
|
||||
decl_raw = col.type or ""
|
||||
decl = decl_raw.upper()
|
||||
decl_is_empty = decl_raw.strip() == ""
|
||||
|
||||
if isinstance(val, memoryview):
|
||||
b = val.tobytes()
|
||||
if "BLOB" in decl or decl_is_empty:
|
||||
out.append(sqlite3.Binary(b))
|
||||
else:
|
||||
try:
|
||||
out.append(b.decode("utf-8"))
|
||||
except UnicodeDecodeError as e:
|
||||
raise ValueError(
|
||||
f"UTF-8 decode failed for column '{col.name}': {e}"
|
||||
)
|
||||
elif isinstance(val, (bytes, bytearray)):
|
||||
b = bytes(val)
|
||||
if "BLOB" in decl or decl_is_empty:
|
||||
out.append(sqlite3.Binary(b))
|
||||
else:
|
||||
try:
|
||||
out.append(b.decode("utf-8"))
|
||||
except UnicodeDecodeError as e:
|
||||
raise ValueError(
|
||||
f"UTF-8 decode failed for column '{col.name}': {e}"
|
||||
)
|
||||
else:
|
||||
out.append(val)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def fetch_and_validate_row_batches(
|
||||
pg_cursor,
|
||||
select_sql: str,
|
||||
sqlite_cols: list[SQLiteCol],
|
||||
batch_size: int = DEFAULT_BATCH_SIZE,
|
||||
):
|
||||
pg_cursor.execute(select_sql)
|
||||
total_rows_seen = 0
|
||||
while True:
|
||||
rows = pg_cursor.fetchmany(batch_size)
|
||||
if not rows:
|
||||
break
|
||||
validated_batch = []
|
||||
for row in rows:
|
||||
total_rows_seen += 1
|
||||
try:
|
||||
validated_batch.append(process_row_for_sqlite(row, sqlite_cols))
|
||||
except ValueError as e:
|
||||
raise ValueError(f"Row {total_rows_seen} validation error: {e}")
|
||||
yield validated_batch
|
||||
|
||||
|
||||
def _get_postgres_sequence_info(
|
||||
pg_cursor, schema: str, table: str, pk_col: str
|
||||
) -> Optional[tuple[str, int]]:
|
||||
"""Get PostgreSQL sequence name and last_value for a table primary key, if any.
|
||||
|
||||
Returns (sequence_name_text, last_value) or None.
|
||||
"""
|
||||
pg_cursor.execute(
|
||||
"""
|
||||
SELECT data_type
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = %s AND table_name = %s AND column_name = %s;
|
||||
""",
|
||||
(schema, table, pk_col),
|
||||
)
|
||||
r = pg_cursor.fetchone()
|
||||
if not r:
|
||||
return None
|
||||
|
||||
pg_type = (r[0] or "").lower()
|
||||
# Match whole words to avoid matching 'interval' etc.
|
||||
if not re.search(r"\b(?:int|serial|bigint)\b", pg_type):
|
||||
return None
|
||||
|
||||
# Get sequence name text (NULL if none)
|
||||
pg_cursor.execute(
|
||||
"SELECT pg_get_serial_sequence(%s, %s);",
|
||||
(f"{schema}.{table}", pk_col),
|
||||
)
|
||||
row = pg_cursor.fetchone()
|
||||
seq_name = row[0] if row else None
|
||||
if not seq_name:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Read last_value by casting the pg_get_serial_sequence result to regclass.
|
||||
pg_cursor.execute(
|
||||
"SELECT last_value FROM pg_get_serial_sequence(%s, %s)::regclass;",
|
||||
(f"{schema}.{table}", pk_col),
|
||||
)
|
||||
rr = pg_cursor.fetchone()
|
||||
if rr and rr[0] is not None:
|
||||
return (seq_name, int(rr[0]))
|
||||
except Exception:
|
||||
# Swallow errors (best-effort); callers treat missing info as absent
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _get_sqlite_max_pk_value(sqlite_cursor, table: str, pk_col: str) -> Optional[int]:
|
||||
"""Get the maximum primary key value from SQLite table."""
|
||||
logger = logging.getLogger("db_migrate")
|
||||
try:
|
||||
sqlite_cursor.execute(
|
||||
f"SELECT MAX({quote_sqlite_identifier(pk_col)}) "
|
||||
f"FROM {quote_sqlite_identifier(table)};"
|
||||
)
|
||||
r3 = sqlite_cursor.fetchone()
|
||||
if r3 and r3[0] is not None:
|
||||
return int(r3[0])
|
||||
except Exception:
|
||||
logger.debug("Failed to read sqlite max pk", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def _update_sqlite_sequence(
|
||||
sqlite_conn: sqlite3.Connection, table: str, sequence_value: int
|
||||
) -> bool:
|
||||
"""Update SQLite sqlite_sequence with new value.
|
||||
|
||||
IMPORTANT: This function does not commit; caller must commit/rollback.
|
||||
"""
|
||||
s_cur = sqlite_conn.cursor()
|
||||
s_cur.execute(
|
||||
"UPDATE sqlite_sequence SET seq = ? WHERE name = ?;",
|
||||
(sequence_value, table),
|
||||
)
|
||||
if s_cur.rowcount == 0:
|
||||
s_cur.execute(
|
||||
"INSERT INTO sqlite_sequence(name, seq) VALUES (?, ?);",
|
||||
(table, sequence_value),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def try_update_sqlite_sequence(
|
||||
pg_cursor,
|
||||
sqlite_conn: sqlite3.Connection,
|
||||
schema: str,
|
||||
pg_table: str, # PostgreSQL table name
|
||||
sqlite_table: str, # SQLite table name
|
||||
sqlite_cols: list[SQLiteCol],
|
||||
) -> bool:
|
||||
"""Update SQLite sequence table based on PostgreSQL sequence values."""
|
||||
logger = logging.getLogger("db_migrate")
|
||||
# Check if table has a single primary key
|
||||
pks = [c for c in sqlite_cols if c.pk]
|
||||
if len(pks) != 1:
|
||||
return False
|
||||
pk_col = pks[0].name
|
||||
|
||||
# Check if SQLite has AUTOINCREMENT (use sqlite_table)
|
||||
s_cur = sqlite_conn.cursor()
|
||||
if not sqlite_table_has_autoincrement(s_cur, sqlite_table):
|
||||
return False
|
||||
if not sqlite_sequence_table_exists(s_cur):
|
||||
return False
|
||||
|
||||
# Get PostgreSQL sequence info (use pg_table)
|
||||
seq_info = _get_postgres_sequence_info(pg_cursor, schema, pg_table, pk_col)
|
||||
pg_last_value = seq_info[1] if seq_info else None
|
||||
|
||||
# Get SQLite max PK value (use sqlite_table)
|
||||
sqlite_max = _get_sqlite_max_pk_value(s_cur, sqlite_table, pk_col)
|
||||
|
||||
# Determine the value to use
|
||||
if pg_last_value is not None and sqlite_max is not None:
|
||||
candidate = max(pg_last_value, sqlite_max)
|
||||
elif pg_last_value is not None:
|
||||
candidate = pg_last_value
|
||||
elif sqlite_max is not None:
|
||||
candidate = sqlite_max
|
||||
else:
|
||||
return False
|
||||
|
||||
updated = _update_sqlite_sequence(sqlite_conn, sqlite_table, candidate)
|
||||
if updated:
|
||||
try:
|
||||
sqlite_conn.commit()
|
||||
except Exception:
|
||||
logger.debug("Failed to commit sqlite_sequence update", exc_info=True)
|
||||
# Let caller proceed; treat as best-effort
|
||||
return updated
|
||||
|
||||
|
||||
# ----------------------
|
||||
# Flow: per-schema migration
|
||||
# ----------------------
|
||||
|
||||
|
||||
def migrate_schema(
|
||||
pg_conn,
|
||||
sqlite_dir: str,
|
||||
schema: str,
|
||||
skipped_tables: Set[str],
|
||||
logger: logging.Logger,
|
||||
dry_run: bool = False,
|
||||
batch_size: int = DEFAULT_BATCH_SIZE,
|
||||
) -> Tuple[int, int]:
|
||||
"""Migrate a single schema. Returns (tables_migrated, rows_inserted_total)."""
|
||||
processed_schema = schema[:-7] if schema.endswith("_schema") else schema
|
||||
sqlite_db_path = Path(sqlite_dir) / f"{processed_schema}.db"
|
||||
if not sqlite_db_path.is_file():
|
||||
logger.error(f"Missing SQLite DB: {sqlite_db_path}")
|
||||
return 0, 0
|
||||
|
||||
tables_migrated = 0
|
||||
rows_inserted = 0
|
||||
|
||||
sqlite_path = sqlite_db_path.resolve()
|
||||
if dry_run:
|
||||
uri = sqlite_path.as_uri() + "?mode=ro"
|
||||
conn_args = {"database": uri, "uri": True}
|
||||
else:
|
||||
conn_args = {"database": str(sqlite_path)}
|
||||
|
||||
with sqlite3.connect(**conn_args) as sqlite_conn:
|
||||
sqlite_cur = sqlite_conn.cursor()
|
||||
sqlite_cur.execute("SELECT name FROM sqlite_master WHERE type='table';")
|
||||
sqlite_tables = [r[0] for r in sqlite_cur.fetchall()]
|
||||
# O(1) lookup map for matching by lowercase name
|
||||
sqlite_table_map = {t.lower(): t for t in sqlite_tables}
|
||||
|
||||
with pg_conn.cursor() as pg_cur:
|
||||
# Ensure we start in a clean state for this connection
|
||||
try:
|
||||
pg_conn.rollback()
|
||||
except Exception:
|
||||
# Ignore rollback failure; connection is newly opened most likely
|
||||
logger.debug(
|
||||
"pg_conn.rollback() at schema start ignored", exc_info=True
|
||||
)
|
||||
|
||||
# Get table list for this schema, but catch/rollback on failure
|
||||
try:
|
||||
tables = list_tables_in_schema(pg_cur, schema)
|
||||
except Exception as e:
|
||||
logger.error("ERROR %s (list tables): %s", schema, e)
|
||||
logger.debug("Traceback (list tables):", exc_info=True)
|
||||
try:
|
||||
pg_conn.rollback()
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"pg_conn.rollback() failed after list tables error",
|
||||
exc_info=True,
|
||||
)
|
||||
return 0, 0
|
||||
|
||||
for table in tables:
|
||||
# Defensive: ensure connection is in a clean state before any new PG work.
|
||||
# A prior error can leave the connection in an aborted transaction; calling
|
||||
# rollback() clears that and allows subsequent SELECTs to run.
|
||||
try:
|
||||
pg_conn.rollback()
|
||||
except Exception:
|
||||
# ignore: best-effort cleanup
|
||||
logger.debug(
|
||||
"pg_conn.rollback() ignored at start of table loop",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if table.lower() in skipped_tables:
|
||||
logger.info("SKIP %s.%s (explicit)", schema, table)
|
||||
continue
|
||||
if table.lower() not in sqlite_table_map:
|
||||
logger.info("SKIP %s.%s (no target table)", schema, table)
|
||||
continue
|
||||
|
||||
matched_table = sqlite_table_map[table.lower()]
|
||||
logger.info("OK %s.%s -> %s", schema, table, matched_table)
|
||||
|
||||
# Fetch PG columns, but defend against aborted transaction here
|
||||
try:
|
||||
pg_cols = get_pg_columns(pg_cur, schema, table)
|
||||
except Exception as e:
|
||||
logger.error("ERROR %s.%s (get columns): %s", schema, table, e)
|
||||
logger.debug("Traceback (get columns):", exc_info=True)
|
||||
try:
|
||||
pg_conn.rollback()
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"pg_conn.rollback() failed after get columns error",
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
|
||||
pg_map: Dict[str, Tuple[str, str]] = {
|
||||
name.lower(): (name, dtype) for name, dtype in pg_cols
|
||||
}
|
||||
|
||||
sqlite_info = get_sqlite_table_info(sqlite_cur, matched_table)
|
||||
if not sqlite_info:
|
||||
logger.warning(f"SKIP {schema}.{table} (no sqlite info)")
|
||||
continue
|
||||
|
||||
# Warn once about schema drift (extra columns)
|
||||
pg_col_names = {name.lower() for name, _ in pg_cols}
|
||||
extra_in_sqlite = {
|
||||
c.name for c in sqlite_info if c.name.lower() not in pg_col_names
|
||||
}
|
||||
if extra_in_sqlite:
|
||||
logger.warning(
|
||||
f"Schema drift: {schema}.{table} has extra SQLite columns {extra_in_sqlite}"
|
||||
)
|
||||
|
||||
try:
|
||||
validate_column_compatibility(sqlite_info, pg_map, schema, table)
|
||||
except ValueError as e:
|
||||
logger.error(f"ERROR {schema}.{table} (type mismatch): {e}")
|
||||
continue
|
||||
|
||||
select_sql = build_select_for_sqlite_columns(
|
||||
schema, table, sqlite_info, pg_map
|
||||
)
|
||||
|
||||
csr_name = _sanitize_cursor_name(f"csr_{schema}_{table}")
|
||||
try:
|
||||
with pg_conn.cursor(name=csr_name) as data_cur:
|
||||
batch_gen = fetch_and_validate_row_batches(
|
||||
data_cur, select_sql, sqlite_info, batch_size=batch_size
|
||||
)
|
||||
first_batch = next(batch_gen, None)
|
||||
|
||||
if not first_batch:
|
||||
logger.info(f"SKIP {schema}.{table} (no rows)")
|
||||
# data_cur will be closed automatically on leaving the 'with'
|
||||
continue
|
||||
|
||||
if dry_run:
|
||||
count = len(first_batch)
|
||||
for batch in batch_gen:
|
||||
count += len(batch)
|
||||
inserted = 0
|
||||
logger.info(
|
||||
f"DRY {schema}.{table} rows_validated={count}"
|
||||
)
|
||||
else:
|
||||
cur = sqlite_conn.cursor()
|
||||
quoted_table = quote_sqlite_identifier(matched_table)
|
||||
col_names = [c.name for c in sqlite_info]
|
||||
quoted_cols = ", ".join(
|
||||
quote_sqlite_identifier(c) for c in col_names
|
||||
)
|
||||
placeholders = ", ".join(["?"] * len(col_names))
|
||||
|
||||
cur.execute("BEGIN;")
|
||||
try:
|
||||
cur.execute(f"DELETE FROM {quoted_table};")
|
||||
inserted = 0
|
||||
if first_batch:
|
||||
cur.executemany(
|
||||
f"INSERT INTO {quoted_table} ({quoted_cols}) VALUES ({placeholders});",
|
||||
first_batch,
|
||||
)
|
||||
inserted += len(first_batch)
|
||||
for batch in batch_gen:
|
||||
if not batch:
|
||||
continue
|
||||
cur.executemany(
|
||||
f"INSERT INTO {quoted_table} ({quoted_cols}) VALUES ({placeholders});",
|
||||
batch,
|
||||
)
|
||||
inserted += len(batch)
|
||||
except Exception:
|
||||
cur.execute("ROLLBACK;")
|
||||
raise
|
||||
else:
|
||||
cur.execute("COMMIT;")
|
||||
except ValueError as e:
|
||||
logger.error("ERROR %s.%s (row validation): %s", schema, table, e)
|
||||
try:
|
||||
pg_conn.rollback()
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"pg_conn.rollback() failed after row validation error",
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error("ERROR %s.%s (select failed): %s", schema, table, e)
|
||||
logger.debug("Traceback (select failed):", exc_info=True)
|
||||
try:
|
||||
pg_conn.rollback()
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"pg_conn.rollback() failed after select failed",
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
|
||||
rows_inserted += inserted
|
||||
tables_migrated += 1
|
||||
logger.info(f"DONE {schema}.{table} rows={inserted}")
|
||||
|
||||
if not dry_run:
|
||||
try:
|
||||
if try_update_sqlite_sequence(
|
||||
pg_cur,
|
||||
sqlite_conn,
|
||||
schema,
|
||||
table,
|
||||
matched_table,
|
||||
sqlite_info,
|
||||
):
|
||||
logger.info(
|
||||
"SEQ %s.%s sqlite_sequence updated", schema, table
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"SEQ %s.%s update failed (ignored): %s", schema, table, e
|
||||
)
|
||||
|
||||
return tables_migrated, rows_inserted
|
||||
|
||||
|
||||
def migrate_data(
|
||||
pg_conn_str: str,
|
||||
sqlite_dir: str,
|
||||
schema_filter: Optional[str],
|
||||
dry_run: bool,
|
||||
skip_tables: str,
|
||||
logger: logging.Logger,
|
||||
batch_size: int,
|
||||
) -> None:
|
||||
skipped_tables = {t.strip().lower() for t in skip_tables.split(",") if t.strip()}
|
||||
total_tables = 0
|
||||
total_rows = 0
|
||||
total_errors = 0
|
||||
|
||||
# List schemas once with a short-lived connection
|
||||
with psycopg.connect(pg_conn_str) as tmp_conn:
|
||||
with tmp_conn.cursor() as cur:
|
||||
schemas = list_user_schemas(cur)
|
||||
|
||||
if schema_filter:
|
||||
schemas = [s for s in schemas if s == schema_filter]
|
||||
if not schemas:
|
||||
logger.error("No schemas to process")
|
||||
return
|
||||
|
||||
for schema in schemas:
|
||||
logger.info("SCHEMA %s", schema)
|
||||
if dry_run:
|
||||
logger.info("(dry-run) validating only — no writes will be performed")
|
||||
|
||||
# Use a fresh connection per-schema to isolate failures/aborted transactions
|
||||
try:
|
||||
with psycopg.connect(pg_conn_str) as pg_conn:
|
||||
migrated_tables, inserted_rows = migrate_schema(
|
||||
pg_conn,
|
||||
sqlite_dir,
|
||||
schema,
|
||||
skipped_tables,
|
||||
logger,
|
||||
dry_run=dry_run,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Schema %s failed: %s", schema, e)
|
||||
logger.debug("Traceback (schema failure):", exc_info=True)
|
||||
total_errors += 1
|
||||
continue
|
||||
|
||||
total_tables += migrated_tables
|
||||
total_rows += inserted_rows
|
||||
|
||||
logger.info("---")
|
||||
logger.info(
|
||||
f"SUMMARY: schemas={len(schemas)} "
|
||||
f"tables_migrated={total_tables} "
|
||||
f"rows_inserted={total_rows} "
|
||||
f"errors={total_errors}"
|
||||
)
|
||||
|
||||
|
||||
# ----------------------
|
||||
# CLI
|
||||
# ----------------------
|
||||
|
||||
|
||||
def parse_args(argv):
|
||||
p = argparse.ArgumentParser(
|
||||
description="Migrate Postgres data into per-schema SQLite DBs (non-invasive)."
|
||||
)
|
||||
p.add_argument("pg_conn", help="Postgres connection string")
|
||||
p.add_argument(
|
||||
"sqlite_dir", help="Directory containing per-schema sqlite .db files"
|
||||
)
|
||||
p.add_argument(
|
||||
"--schema", help="Only migrate this schema (exact match)", default=None
|
||||
)
|
||||
p.add_argument(
|
||||
"--dry-run",
|
||||
help="Validate and report only; do not write to sqlite",
|
||||
action="store_true",
|
||||
)
|
||||
p.add_argument("--verbose", help="Verbose logging (debug)", action="store_true")
|
||||
p.add_argument("--no-color", help="Disable ANSI color output", action="store_true")
|
||||
p.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=DEFAULT_BATCH_SIZE,
|
||||
help="Batch size for data fetching",
|
||||
)
|
||||
p.add_argument(
|
||||
"--skip-tables",
|
||||
help="Comma-separated list of tables to skip",
|
||||
default="migrations,servers_stats",
|
||||
)
|
||||
|
||||
return p.parse_args(argv[1:])
|
||||
|
||||
|
||||
def main(argv):
|
||||
args = parse_args(argv)
|
||||
logger = setup_logger(args.verbose, args.no_color)
|
||||
|
||||
sqlite_dir_path = Path(args.sqlite_dir)
|
||||
if not sqlite_dir_path.is_dir():
|
||||
logger.error("SQLite directory does not exist or is not a directory.")
|
||||
raise SystemExit(1)
|
||||
|
||||
try:
|
||||
migrate_data(
|
||||
args.pg_conn,
|
||||
str(sqlite_dir_path),
|
||||
args.schema,
|
||||
args.dry_run,
|
||||
args.skip_tables,
|
||||
logger,
|
||||
args.batch_size,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Fatal error: {e}")
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv)
|
||||
@@ -10,7 +10,8 @@ LOAD DATABASE
|
||||
-- pgloader implementation doesn't find "GENERATED ALWAYS AS IDENTITY" sequences,
|
||||
-- instead we reset sequences manually via custom query after load
|
||||
reset no sequences,
|
||||
data only
|
||||
data only,
|
||||
workers = {{WORKERS}}
|
||||
|
||||
EXCLUDING TABLE NAMES LIKE 'migrations', 'sqlite_sequence'
|
||||
|
||||
|
||||
@@ -12,5 +12,10 @@ echo "$APPLE_SIMPLEX_SIGNING_KEYCHAIN" | base64 --decode -o /tmp/simplex.keychai
|
||||
|
||||
scripts/desktop/build-lib-mac.sh
|
||||
cd apps/multiplatform
|
||||
./gradlew packageDmg
|
||||
./gradlew notarizeDmg
|
||||
if [ -n "${ASSETS_DIR:-}" ]; then
|
||||
set -- -Psimplex.assets.dir="$ASSETS_DIR"
|
||||
else
|
||||
set --
|
||||
fi
|
||||
./gradlew "$@" packageDmg
|
||||
./gradlew "$@" notarizeDmg
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
@@ -7,7 +7,8 @@ function readlink() {
|
||||
}
|
||||
|
||||
OS=linux
|
||||
ARCH=${1:-`uname -a | rev | cut -d' ' -f2 | rev`}
|
||||
ARCH="$(uname -m)"
|
||||
DATABASE_BACKEND="${1:-sqlite}"
|
||||
GHC_VERSION=8.10.7
|
||||
|
||||
if [ "$ARCH" == "aarch64" ]; then
|
||||
@@ -24,18 +25,25 @@ exports=( $(sed 's/foreign export ccall "chat_migrate_init_key"//' src/Simplex/C
|
||||
for elem in "${exports[@]}"; do count=$(grep -R "$elem$" libsimplex.dll.def | wc -l); if [ $count -ne 1 ]; then echo Wrong exports in libsimplex.dll.def. Add \"$elem\" to that file; exit 1; fi ; done
|
||||
for elem in "${exports[@]}"; do count=$(grep -R "\"$elem\"" flake.nix | wc -l); if [ $count -ne 2 ]; then echo Wrong exports in flake.nix. Add \"$elem\" in two places of the file; exit 1; fi ; done
|
||||
|
||||
rm -rf $BUILD_DIR
|
||||
cabal build lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN -flink-rts -threaded' --ghc-options="-optl-L$(ghc --print-libdir)/rts -optl-Wl,--as-needed,-lHSrts_thr-ghc$GHC_VERSION" --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library'
|
||||
#rm -rf $BUILD_DIR
|
||||
if [[ "$DATABASE_BACKEND" == "postgres" ]]; then
|
||||
echo "Building with postgres backend..."
|
||||
cabal build lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN -optl-Wl,-soname,libsimplex.so -flink-rts -threaded' --constraint 'simplexmq +client_library +client_postgres' --constraint 'simplex-chat +client_library +client_postgres'
|
||||
else
|
||||
echo "Building with sqlite backend..."
|
||||
cabal build lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN -optl-Wl,-soname,libsimplex.so -flink-rts -threaded' --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library'
|
||||
fi
|
||||
cd $BUILD_DIR/build
|
||||
#patchelf --add-needed libHSrts_thr-ghc${GHC_VERSION}.so libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so
|
||||
#patchelf --add-rpath '$ORIGIN' libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so
|
||||
mv libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so libsimplex.so 2> /dev/null || true
|
||||
#patchelf --add-needed libHSrts_thr-ghc${GHC_VERSION}.so libsimplex.so
|
||||
#patchelf --add-rpath '$ORIGIN' libsimplex.so
|
||||
|
||||
# GitHub's Ubuntu 20.04 runner started to set libffi.so.7 as a dependency while Ubuntu 20.04 on user's devices may not have it
|
||||
# but libffi.so.8 is shipped as an external library with other libs
|
||||
patchelf --replace-needed "libffi.so.7" "libffi.so.8" libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so
|
||||
patchelf --replace-needed "libffi.so.7" "libffi.so.8" libsimplex.so
|
||||
|
||||
mkdir deps 2> /dev/null || true
|
||||
ldd libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so | grep "ghc" | cut -d' ' -f 3 | xargs -I {} cp {} ./deps/
|
||||
ldd libsimplex.so | grep "ghc" | cut -d' ' -f 3 | xargs -I {} cp {} ./deps/
|
||||
|
||||
cd -
|
||||
|
||||
@@ -44,7 +52,7 @@ rm -rf apps/multiplatform/desktop/build/cmake
|
||||
|
||||
mkdir -p apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
|
||||
cp -r $BUILD_DIR/build/deps/* apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
|
||||
cp $BUILD_DIR/build/libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
|
||||
cp $BUILD_DIR/build/libsimplex.so apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
|
||||
scripts/desktop/prepare-vlc-linux.sh
|
||||
|
||||
links_dir=apps/multiplatform/build/links
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
set -e
|
||||
|
||||
OS=mac
|
||||
ARCH="${1:-`uname -a | rev | cut -d' ' -f1 | rev`}"
|
||||
ARCH="${1:-$(uname -m)}"
|
||||
COMPOSE_ARCH=$ARCH
|
||||
GHC_VERSION=9.6.3
|
||||
DATABASE_BACKEND="${2:-sqlite}"
|
||||
@@ -14,7 +14,7 @@ else
|
||||
COMPOSE_ARCH=x64
|
||||
fi
|
||||
LIB_EXT=dylib
|
||||
LIB=libHSsimplex-chat-*-inplace-ghc*.$LIB_EXT
|
||||
LIB=libsimplex.$LIB_EXT
|
||||
GHC_LIBS_DIR=$(ghc --print-libdir)
|
||||
|
||||
BUILD_DIR=dist-newstyle/build/$ARCH-*/ghc-*/simplex-chat-*
|
||||
@@ -27,13 +27,14 @@ rm -rf $BUILD_DIR
|
||||
|
||||
if [[ "$DATABASE_BACKEND" == "postgres" ]]; then
|
||||
echo "Building with postgres backend..."
|
||||
cabal build -f client_postgres lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library'
|
||||
cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-install_name,@rpath/$LIB -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library +client_postgres' --constraint 'simplex-chat +client_library +client_postgres'
|
||||
else
|
||||
echo "Building with sqlite backend..."
|
||||
cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library'
|
||||
cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-install_name,@rpath/$LIB -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library'
|
||||
fi
|
||||
|
||||
cd $BUILD_DIR/build
|
||||
mv libHSsimplex-chat-*-inplace-ghc*.$LIB_EXT libsimplex.dylib 2> /dev/null || true
|
||||
mkdir deps 2> /dev/null || true
|
||||
|
||||
# It's not included by default for some reason. Compiled lib tries to find system one but it's not always available
|
||||
@@ -87,7 +88,7 @@ rm -rf apps/multiplatform/desktop/build/cmake
|
||||
|
||||
mkdir -p apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
|
||||
cp -r $BUILD_DIR/build/deps/* apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
|
||||
cp $BUILD_DIR/build/libHSsimplex-chat-*-inplace-ghc*.$LIB_EXT apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
|
||||
cp $BUILD_DIR/build/$LIB apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
|
||||
|
||||
cd apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
@@ -18,7 +18,12 @@ libcrypto_path=$(ldd common/src/commonMain/cpp/desktop/libs/*/libHSdirect-sqlcip
|
||||
trap "rm common/src/commonMain/cpp/desktop/libs/*/`basename $libcrypto_path` 2> /dev/null || true" EXIT
|
||||
cp $libcrypto_path common/src/commonMain/cpp/desktop/libs/*
|
||||
|
||||
./gradlew createDistributable
|
||||
if [ -n "${ASSETS_DIR:-}" ]; then
|
||||
set -- -Psimplex.assets.dir="$ASSETS_DIR"
|
||||
else
|
||||
set --
|
||||
fi
|
||||
./gradlew "$@" createDistributable
|
||||
rm common/src/commonMain/cpp/desktop/libs/*/`basename $libcrypto_path`
|
||||
|
||||
rm -rf $release_app_dir/AppDir 2>/dev/null
|
||||
|
||||
@@ -4,7 +4,12 @@ ARCH="$(uname -m)"
|
||||
|
||||
scripts/desktop/build-lib-linux.sh
|
||||
cd apps/multiplatform
|
||||
./gradlew packageDeb
|
||||
if [ -n "${ASSETS_DIR:-}" ]; then
|
||||
set -- -Psimplex.assets.dir="$ASSETS_DIR"
|
||||
else
|
||||
set --
|
||||
fi
|
||||
./gradlew "$@" packageDeb
|
||||
|
||||
# Workaround for skiko library
|
||||
#
|
||||
@@ -37,8 +42,12 @@ export SOURCE_DATE_EPOCH=1704067200
|
||||
|
||||
dpkg-deb -R ./release/main/deb/simplex*.deb ./extracted
|
||||
|
||||
# Source the distribution variables (VERSION_CODENAME)
|
||||
. /etc/os-release
|
||||
|
||||
rm -f ./extracted/opt/*imple*/lib/app/*skiko-awt-runtime-linux*
|
||||
sed -i -e '/skiko-awt-runtime-linux/d' ./extracted/opt/*imple*/lib/app/simplex.cfg
|
||||
sed -i "/Version/ s/\$/~$VERSION_CODENAME/" ./extracted/DEBIAN/control
|
||||
find ./extracted/ -exec touch -d "@$SOURCE_DATE_EPOCH" {} +
|
||||
|
||||
dpkg-deb --build --root-owner-group --uniform-compression ./extracted ./release/main/deb/simplex_${ARCH}.deb
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
set -e
|
||||
|
||||
ARCH="${1:-`uname -a | rev | cut -d' ' -f1 | rev`}"
|
||||
ARCH="${1:-$(uname -m)}"
|
||||
if [ "$ARCH" == "arm64" ]; then
|
||||
ARCH=aarch64
|
||||
vlc_arch=arm64
|
||||
|
||||
@@ -38,6 +38,186 @@
|
||||
</description>
|
||||
|
||||
<releases>
|
||||
<release version="6.5.2" date="2026-05-15">
|
||||
<url type="details">https://simplex.chat/blog/20260430-simplex-channels-v6-5-consortium-crowdfunding-freedom-of-speech.html</url>
|
||||
<description>
|
||||
<p>New in v6.5.2:</p>
|
||||
<ul>
|
||||
<li>allow deleting messages from channel history without time limit.</li>
|
||||
</ul>
|
||||
<p>New in v6.5:</p>
|
||||
<p>Public channels - speak freely!</p>
|
||||
<ul>
|
||||
<li>Reliability: many relays per channel.</li>
|
||||
<li>Ownership: you can run your own relays.</li>
|
||||
<li>Security: owners hold channel keys.</li>
|
||||
<li>Privacy: for owners and subscribers.</li>
|
||||
</ul>
|
||||
<p>Easier to invite your friends: we made connecting simpler for new users.</p>
|
||||
<p>Safe web links:</p>
|
||||
<ul>
|
||||
<li>opt-in to send link previews.</li>
|
||||
<li>use SOCKS proxy for previews (if enabled).</li>
|
||||
<li>prevent hyperlink phishing.</li>
|
||||
<li>remove link tracking.</li>
|
||||
</ul>
|
||||
<p>Non-profit governance: to make SimpleX Network last.</p>
|
||||
</description>
|
||||
</release>
|
||||
<release version="6.5.1" date="2026-05-02">
|
||||
<url type="details">https://simplex.chat/blog/20260430-simplex-channels-v6-5-consortium-crowdfunding-freedom-of-speech.html</url>
|
||||
<description>
|
||||
<p>New in v6.5.1:</p>
|
||||
<ul>
|
||||
<li>additional preset chat relay.</li>
|
||||
<li>fixed a rare bug when receiving files.</li>
|
||||
</ul>
|
||||
<p>New in v6.5:</p>
|
||||
<p>Public channels - speak freely!</p>
|
||||
<ul>
|
||||
<li>Reliability: many relays per channel.</li>
|
||||
<li>Ownership: you can run your own relays.</li>
|
||||
<li>Security: owners hold channel keys.</li>
|
||||
<li>Privacy: for owners and subscribers.</li>
|
||||
</ul>
|
||||
<p>Easier to invite your friends: we made connecting simpler for new users.</p>
|
||||
<p>Safe web links:</p>
|
||||
<ul>
|
||||
<li>opt-in to send link previews.</li>
|
||||
<li>use SOCKS proxy for previews (if enabled).</li>
|
||||
<li>prevent hyperlink phishing.</li>
|
||||
<li>remove link tracking.</li>
|
||||
</ul>
|
||||
<p>Non-profit governance: to make SimpleX Network last.</p>
|
||||
</description>
|
||||
</release>
|
||||
<release version="6.5.0" date="2026-04-30">
|
||||
<url type="details">https://simplex.chat/blog/20260430-simplex-channels-v6-5-consortium-crowdfunding-freedom-of-speech.html</url>
|
||||
<description>
|
||||
<p>New in v6.5.</p>
|
||||
<p>Public channels - speak freely!</p>
|
||||
<ul>
|
||||
<li>Reliability: many relays per channel.</li>
|
||||
<li>Ownership: you can run your own relays.</li>
|
||||
<li>Security: owners hold channel keys.</li>
|
||||
<li>Privacy: for owners and subscribers.</li>
|
||||
</ul>
|
||||
<p>Easier to invite your friends: we made connecting simpler for new users.</p>
|
||||
<p>Safe web links:</p>
|
||||
<ul>
|
||||
<li>opt-in to send link previews.</li>
|
||||
<li>use SOCKS proxy for previews (if enabled).</li>
|
||||
<li>prevent hyperlink phishing.</li>
|
||||
<li>remove link tracking.</li>
|
||||
</ul>
|
||||
<p>Non-profit governance: to make SimpleX Network last.</p>
|
||||
</description>
|
||||
</release>
|
||||
<release version="6.4.11" date="2026-03-30">
|
||||
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
|
||||
<description>
|
||||
<p>New in v6.4.11:</p>
|
||||
<ul>
|
||||
<li>improve image, video and link messages.</li>
|
||||
</ul>
|
||||
<p>New in v6.4-6.4.10:</p>
|
||||
<ul>
|
||||
<li>new UX to connect.</li>
|
||||
<li>review new group members.</li>
|
||||
<li>chat with group admins.</li>
|
||||
<li>new UI languages: Catalan, Indonesian, Romanian and Vietnamese.</li>
|
||||
<li>Linux app builds for aarch64 CPUs</li>
|
||||
<li>UI support for bot commands.</li>
|
||||
<li>support markdown hyperlinks, such as [click here](https://example.com).</li>
|
||||
<li>option to remove tracking parameters from the links.</li>
|
||||
<li>better information about network errors.</li>
|
||||
</ul>
|
||||
</description>
|
||||
</release>
|
||||
<release version="6.4.10" date="2026-01-29">
|
||||
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
|
||||
<description>
|
||||
<p>New in v6.4.10:</p>
|
||||
<ul>
|
||||
<li>improve error handling</li>
|
||||
</ul>
|
||||
<p>New in v6.4-6.4.8:</p>
|
||||
<ul>
|
||||
<li>new UX to connect.</li>
|
||||
<li>review new group members.</li>
|
||||
<li>chat with group admins.</li>
|
||||
<li>new UI languages: Catalan, Indonesian, Romanian and Vietnamese.</li>
|
||||
<li>Linux app builds for aarch64 CPUs</li>
|
||||
<li>UI support for bot commands.</li>
|
||||
<li>support markdown hyperlinks, such as [click here](https://example.com).</li>
|
||||
<li>option to remove tracking parameters from the links.</li>
|
||||
<li>better information about network errors.</li>
|
||||
</ul>
|
||||
</description>
|
||||
</release>
|
||||
<release version="6.4.8" date="2025-12-11">
|
||||
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
|
||||
<description>
|
||||
<p>New in v6.4.8:</p>
|
||||
<ul>
|
||||
<li>fix stuck message reception and other events after passphrase change (e.g., during desktop app initial start)</li>
|
||||
</ul>
|
||||
<p>New in v6.4-6.4.7:</p>
|
||||
<ul>
|
||||
<li>new UX to connect.</li>
|
||||
<li>review new group members.</li>
|
||||
<li>chat with group admins.</li>
|
||||
<li>new UI languages: Catalan, Indonesian, Romanian and Vietnamese.</li>
|
||||
<li>Linux app builds for aarch64 CPUs</li>
|
||||
<li>UI support for bot commands.</li>
|
||||
<li>support markdown hyperlinks, such as [click here](https://example.com).</li>
|
||||
<li>option to remove tracking parameters from the links.</li>
|
||||
<li>better information about network errors.</li>
|
||||
</ul>
|
||||
</description>
|
||||
</release>
|
||||
<release version="6.4.7" date="2025-11-03">
|
||||
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
|
||||
<description>
|
||||
<p>New in v6.4.7:</p>
|
||||
<ul>
|
||||
<li>fix exporting database larger than 4gb.</li>
|
||||
</ul>
|
||||
<p>New in v6.4-6.4.6:</p>
|
||||
<ul>
|
||||
<li>new UX to connect.</li>
|
||||
<li>review new group members.</li>
|
||||
<li>chat with group admins.</li>
|
||||
<li>new UI languages: Catalan, Indonesian, Romanian and Vietnamese.</li>
|
||||
<li>Linux app builds for aarch64 CPUs</li>
|
||||
<li>UI support for bot commands.</li>
|
||||
<li>support markdown hyperlinks, such as [click here](https://example.com).</li>
|
||||
<li>option to remove tracking parameters from the links.</li>
|
||||
<li>better information about network errors.</li>
|
||||
</ul>
|
||||
</description>
|
||||
</release>
|
||||
<release version="6.4.6" date="2025-10-05">
|
||||
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
|
||||
<description>
|
||||
<p>New in v6.4.6:</p>
|
||||
<ul>
|
||||
<li>fixed opening SimpleX links from outside of the app.</li>
|
||||
</ul>
|
||||
<p>New in v6.4-6.4.5:</p>
|
||||
<ul>
|
||||
<li>new UX to connect.</li>
|
||||
<li>review new group members.</li>
|
||||
<li>chat with group admins.</li>
|
||||
<li>new UI languages: Catalan, Indonesian, Romanian and Vietnamese.</li>
|
||||
<li>Linux app builds for aarch64 CPUs</li>
|
||||
<li>UI support for bot commands.</li>
|
||||
<li>support markdown hyperlinks, such as [click here](https://example.com).</li>
|
||||
<li>option to remove tracking parameters from the links.</li>
|
||||
<li>better information about network errors.</li>
|
||||
</ul>
|
||||
</description>
|
||||
</release>
|
||||
<release version="6.4.5" date="2025-09-08">
|
||||
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
|
||||
<description>
|
||||
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# Copies generated iOS assets into SimpleXAssets.xcassets.
|
||||
# Intended to run as an Xcode Run Script build phase.
|
||||
# Skips silently if SIMPLEX_ASSETS is not in SWIFT_ACTIVE_COMPILATION_CONDITIONS
|
||||
# or if the source directory is not found.
|
||||
#
|
||||
# The source path is resolved in order:
|
||||
# 1. Command-line argument
|
||||
# 2. SIMPLEX_ASSETS_DIR build setting (set in Local.xcconfig)
|
||||
# 3. No default — skips if neither is set
|
||||
#
|
||||
# Manual usage: ./scripts/copy-assets.sh path/to/assets
|
||||
|
||||
# Skip if SIMPLEX_ASSETS flag is not set (unless run manually outside Xcode)
|
||||
if [ -n "${SWIFT_ACTIVE_COMPILATION_CONDITIONS:-}" ]; then
|
||||
case " $SWIFT_ACTIVE_COMPILATION_CONDITIONS " in
|
||||
*" SIMPLEX_ASSETS "*) ;;
|
||||
*) exit 0 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
IOS_DIR="$SCRIPT_DIR/../../apps/ios/Shared/SimpleXAssets.xcassets"
|
||||
|
||||
ASSETS_ROOT="${1:-${SIMPLEX_ASSETS_DIR:-}}"
|
||||
if [ -z "$ASSETS_ROOT" ]; then
|
||||
echo "warning: SIMPLEX_ASSETS_DIR not set and no path argument provided" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
SRC_DIR="$ASSETS_ROOT/ios/Assets.xcassets"
|
||||
|
||||
if [ ! -d "$SRC_DIR" ]; then
|
||||
echo "warning: source assets not found: $SRC_DIR (run resize.sh first)" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Remove old imagesets but keep root Contents.json
|
||||
find "$IOS_DIR" -name "*.imageset" -type d -exec rm -rf {} + 2>/dev/null || true
|
||||
|
||||
# Copy imagesets
|
||||
for imageset in "$SRC_DIR"/*.imageset; do
|
||||
[ -d "$imageset" ] || continue
|
||||
cp -r "$imageset" "$IOS_DIR/"
|
||||
echo "Copied $(basename "$imageset")"
|
||||
done
|
||||
|
||||
echo "Done. Assets copied to $IOS_DIR"
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"https://github.com/simplex-chat/simplexmq.git"."9346b85c3f34f8b12fefef4631ba21087cf5f0e3" = "010xj64rwj4g36f42znjv73c4armsdm6q9rdpxy1bqknwcbsk2sw";
|
||||
"https://github.com/simplex-chat/simplexmq.git"."f03cec7a58ed13a39a52886888c74bcefdb64479" = "0bkd8kqgmwgfh5rwnw7s4p6mx9kwigi4jq9ljlfvzj23pslk1aq7";
|
||||
"https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
|
||||
"https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "1ql13f4kfwkbaq7nygkxgw84213i0zm7c1a8hwvramayxl38dq5d";
|
||||
"https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl";
|
||||
|
||||
+277
@@ -0,0 +1,277 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
SIMPLEX_KEY='3C:52:C4:FD:3C:AD:1C:07:C9:B0:0A:70:80:E3:58:FA:B9:FE:FC:B8:AF:5A:EC:14:77:65:F1:6D:0F:21:AD:85'
|
||||
|
||||
REPO_NAME="simplex-chat"
|
||||
REPO="https://github.com/simplex-chat/${REPO_NAME}"
|
||||
|
||||
IMAGE_NAME='sx-local-android'
|
||||
CONTAINER_NAME='sx-builder-android'
|
||||
DOCKER_PATH_PROJECT='/project'
|
||||
DOCKER_PATH_VERIFY='/verify'
|
||||
|
||||
export DOCKER_BUILDKIT=1
|
||||
|
||||
SIMPLEX_REPO='simplex-chat/simplex-chat'
|
||||
CMDS="curl git docker"
|
||||
|
||||
INIT_DIR="$PWD"
|
||||
TEMPDIR="$(mktemp -d)"
|
||||
PID_MAX_ORIGINAL="$(sysctl -n kernel.pid_max)"
|
||||
|
||||
ARCHES="${ARCHES:-aarch64 armv7a}"
|
||||
|
||||
COLOR_CYAN="\033[36m"
|
||||
COLOR_RESET="\033[0m"
|
||||
|
||||
SUFFIX_BUILT='built'
|
||||
SUFFIX_DOWNLOADED='downloaded'
|
||||
SUFFIX_BUILT_WITH_SIGNATURE='built-with-downloaded-signature'
|
||||
|
||||
cleanup() {
|
||||
rm -rf -- "${TEMPDIR}"
|
||||
docker rm --force "${CONTAINER_NAME}" 2>/dev/null || :
|
||||
docker image rm "${IMAGE_NAME}" 2>/dev/null || :
|
||||
|
||||
if [ "$(sysctl -n kernel.pid_max)" != "$PID_MAX_ORIGINAL" ]; then
|
||||
printf 'Adjusting kernel.pid_max back to original value...\n'
|
||||
if $SUDO sysctl kernel.pid_max="$PID_MAX_ORIGINAL"; then
|
||||
printf 'Successfully adjusted kernel.pid_max\n'
|
||||
else
|
||||
printf 'Failed to adjust kernel.pid_max. Please set the value manually with: %s sysctl kernel.pid_max=%s\n' "$SUDO" "$PID_MAX_ORIGINAL"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
trap 'cleanup' EXIT INT
|
||||
|
||||
check() {
|
||||
commands="$1"
|
||||
|
||||
set +u
|
||||
|
||||
for i in $commands; do
|
||||
if ! command -v "$i" > /dev/null 2>&1; then
|
||||
commands_failed="$i $commands_failed"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$commands_failed" ]; then
|
||||
commands_failed=${commands_failed% *}
|
||||
printf "%s is not found in your \$PATH. Please install them and re-run the script.\n" "$commands_failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$PID_MAX_ORIGINAL" -gt 65535 ]; then
|
||||
SUDO=$(command -v sudo || command -v doas) || { echo "No sudo or doas"; exit 1; }
|
||||
|
||||
printf 'Adjusting kernel.pid_max value to 65535...\n'
|
||||
|
||||
if $SUDO sysctl kernel.pid_max=65535; then
|
||||
printf 'Successfully adjusted kernel.pid_max\n'
|
||||
else
|
||||
printf 'Failed to adjust kernel.pid_max, aborting.\n'
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
set -u
|
||||
}
|
||||
|
||||
download_apk() {
|
||||
tag="$1"
|
||||
filename="$2"
|
||||
file_out="$3"
|
||||
|
||||
curl -L "${REPO}/releases/download/${tag}/${filename}" -o "$file_out"
|
||||
}
|
||||
|
||||
setup_git() {
|
||||
workdir="$1"
|
||||
name="$2"
|
||||
|
||||
git -C "$workdir" clone "${REPO}.git" "$name"
|
||||
}
|
||||
|
||||
checkout_git() {
|
||||
git_dir="$1"
|
||||
tag="$2"
|
||||
|
||||
git -C "$git_dir" reset --hard
|
||||
git -C "$git_dir" clean -dfx
|
||||
git -C "$git_dir" checkout "$tag"
|
||||
}
|
||||
|
||||
check_apk() {
|
||||
apk_name="$1"
|
||||
expected="$2"
|
||||
|
||||
actual=$(docker exec "${CONTAINER_NAME}" apksigner verify --print-certs "${DOCKER_PATH_VERIFY}/${apk_name}" | grep 'SHA-256' | awk '{print $NF}' | fold -w2 | paste -sd: | tr '[:lower:]' '[:upper:]')
|
||||
|
||||
if [ "$expected" = "$actual" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
verify_apk() {
|
||||
apk_name="$1"
|
||||
|
||||
# https://github.com/obfusk/apksigcopier?tab=readme-ov-file#what-about-signatures-made-by-apksigner-from-build-tools--3500-rc1
|
||||
docker exec "${CONTAINER_NAME}" repro-apk zipalign --page-size 16 --pad-like-apksigner --replace "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT}" \
|
||||
"${DOCKER_PATH_VERIFY}/${apk_name}.aligned"
|
||||
docker exec "${CONTAINER_NAME}" mv "${DOCKER_PATH_VERIFY}/${apk_name}.aligned" \
|
||||
"${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT}"
|
||||
|
||||
docker exec "${CONTAINER_NAME}" apksigcopier copy "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_DOWNLOADED}" \
|
||||
"${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT}" \
|
||||
"${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT_WITH_SIGNATURE}"
|
||||
|
||||
downloaded_apk_hash=$(docker exec "${CONTAINER_NAME}" sha256sum "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_DOWNLOADED}" | awk '{print $1}')
|
||||
built_apk_hash=$(docker exec "${CONTAINER_NAME}" sha256sum "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT_WITH_SIGNATURE}" | awk '{print $1}')
|
||||
|
||||
if [ "$downloaded_apk_hash" = "$built_apk_hash" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
print_vercode() {
|
||||
build_dir="$1"
|
||||
awk -F'=' '/android.version_code=/ {print $2}' "${build_dir}/apps/multiplatform/gradle.properties"
|
||||
}
|
||||
|
||||
setup_container() {
|
||||
dir_git="$1"
|
||||
dir_apk="$2"
|
||||
|
||||
docker build \
|
||||
--no-cache \
|
||||
-f "${dir_git}/Dockerfile.build" \
|
||||
-t "${IMAGE_NAME}" \
|
||||
--build-arg=USER_UID="$(id -u)" \
|
||||
--build-arg=USER_GID="$(id -g)" \
|
||||
.
|
||||
|
||||
# Run container in background
|
||||
docker run -t -d \
|
||||
--name "${CONTAINER_NAME}" \
|
||||
--device /dev/fuse \
|
||||
--cap-add SYS_ADMIN \
|
||||
--security-opt apparmor:unconfined \
|
||||
--security-opt seccomp:unconfined \
|
||||
-v "${dir_git}:${DOCKER_PATH_PROJECT}" \
|
||||
-v "${dir_apk}:${DOCKER_PATH_VERIFY}" \
|
||||
"${IMAGE_NAME}"
|
||||
}
|
||||
|
||||
build_apk() {
|
||||
arch="$1"
|
||||
vercode="$2"
|
||||
|
||||
apk_out="simplex-${arch}.apk.${SUFFIX_BUILT}"
|
||||
|
||||
# Gradle setup
|
||||
docker exec -i "${CONTAINER_NAME}" sh << EOF
|
||||
cd $DOCKER_PATH_PROJECT/apps/multiplatform
|
||||
./gradlew
|
||||
EOF
|
||||
|
||||
docker exec -i "${CONTAINER_NAME}" sh << EOF
|
||||
GRADLE_BIN=\$(find \$HOME/.gradle/wrapper/dists -name "gradle" -type f -executable 2>/dev/null | head -1)
|
||||
GRADLE_DIR=\$(dirname "\$GRADLE_BIN")
|
||||
export PATH="\$GRADLE_DIR:\$PATH"
|
||||
|
||||
ARCHES="$arch" ./scripts/android/build-android.sh -gs "$vercode" || ARCHES="$arch" ./scripts/android/build-android.sh -gs "$vercode"
|
||||
|
||||
APK_FILE=\$(find . -maxdepth 1 -type f -name '*.apk')
|
||||
|
||||
mv "\$APK_FILE" $DOCKER_PATH_VERIFY/$apk_out
|
||||
EOF
|
||||
}
|
||||
|
||||
main() {
|
||||
tag="$1"
|
||||
|
||||
build_directory="${TEMPDIR}/${REPO_NAME}"
|
||||
final_directory="$INIT_DIR/${tag}-${REPO_NAME}"
|
||||
apk_directory="${final_directory}/android"
|
||||
|
||||
printf 'This script will:
|
||||
1) build docker container.
|
||||
2) download APK from GitHub and validate signatures.
|
||||
3) build core library with nix (12-24 hours).
|
||||
4) build APK and compare with downloaded one
|
||||
|
||||
The script will ask for sudo password to adjust kernel.pid_max (needed for armv7a build)
|
||||
and set it back to otiginal value when the build is done.
|
||||
|
||||
Continue?'
|
||||
|
||||
read _
|
||||
|
||||
check "$CMDS"
|
||||
|
||||
mkdir -p "${apk_directory}"
|
||||
|
||||
# Setup initial git for Dockerfile.build
|
||||
setup_git "$TEMPDIR" "$REPO_NAME"
|
||||
checkout_git "$build_directory" "$tag"
|
||||
|
||||
printf "${COLOR_CYAN}Building Docker container...${COLOR_RESET}\n"
|
||||
setup_container "$build_directory" "$apk_directory"
|
||||
|
||||
# Check phase
|
||||
for arch in $ARCHES; do
|
||||
filename="simplex-${arch}.apk"
|
||||
|
||||
download_apk "$tag" "$filename" "${apk_directory}/${filename}.${SUFFIX_DOWNLOADED}"
|
||||
|
||||
if check_apk "${filename}.${SUFFIX_DOWNLOADED}" "$SIMPLEX_KEY"; then
|
||||
printf "${COLOR_CYAN}APK for %s is signed by valid key.${COLOR_RESET}\n" "$arch"
|
||||
else
|
||||
printf "${COLOR_CYAN}Signature of APK for %s is invalid., aborting the script.${COLOR_RESET}\n" "$arch"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Build phase
|
||||
for arch in $ARCHES; do
|
||||
case "$arch" in
|
||||
armv7a)
|
||||
build_tag="${tag}-armv7a"
|
||||
;;
|
||||
aarch64)
|
||||
build_tag="${tag}"
|
||||
;;
|
||||
*)
|
||||
printf "${COLOR_CYAN}Unknown architecture: %s! Skipping the build...${COLOR_RESET}\n" "$arch"
|
||||
continue
|
||||
esac
|
||||
|
||||
# Setup the code
|
||||
checkout_git "$build_directory" "$build_tag"
|
||||
vercode=$(print_vercode "$build_directory")
|
||||
|
||||
printf "${COLOR_CYAN}Building APK for for %s...${COLOR_RESET}\n" "$arch"
|
||||
build_apk "$arch" "$vercode"
|
||||
done
|
||||
|
||||
# Verification phase
|
||||
for arch in $ARCHES; do
|
||||
filename="simplex-${arch}.apk"
|
||||
|
||||
if ! verify_apk "$filename"; then
|
||||
printf "${COLOR_CYAN}Failed to verify %s! Aborting.\n${COLOR_RESET}" "$filename"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
printf "${COLOR_CYAN}%s is reproducible.${COLOR_RESET}\n" "$tag"
|
||||
|
||||
cleanup
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -38,7 +38,11 @@ git -C "${tempdir}" clone "${repo}.git" &&\
|
||||
cd "${tempdir}/${repo_name}" &&\
|
||||
git checkout "${TAG}"
|
||||
|
||||
for os in '22.04' '24.04'; do
|
||||
oses="22.04@sha256:5c8b2c0a6c745bc177669abfaa716b4bc57d58e2ea3882fb5da67f4d59e3dda5 24.04@sha256:98ff7968124952e719a8a69bb3cccdd217f5fe758108ac4f21ad22e1df44d237"
|
||||
|
||||
for os_pair in ${oses}; do
|
||||
os="${os_pair%@*}"
|
||||
hash="${os_pair#*@}"
|
||||
os_url="$(printf '%s' "${os}" | tr '.' '_')"
|
||||
|
||||
cli_name="simplex-chat-ubuntu-${os_url}-x86_64"
|
||||
@@ -49,7 +53,10 @@ for os in '22.04' '24.04'; do
|
||||
docker build \
|
||||
--no-cache \
|
||||
--build-arg TAG="${os}" \
|
||||
--build-arg HASH="${hash}" \
|
||||
--build-arg GHC="${ghc}" \
|
||||
--build-arg=USER_UID="$(id -u)" \
|
||||
--build-arg=USER_GID="$(id -g)" \
|
||||
-f "${tempdir}/${repo_name}/Dockerfile.build" \
|
||||
-t "${image_name}" \
|
||||
.
|
||||
@@ -103,7 +110,7 @@ for os in '22.04' '24.04'; do
|
||||
# Desktop: deb
|
||||
docker exec \
|
||||
-t "${container_name}" \
|
||||
sh -c './scripts/desktop/make-deb-linux.sh'
|
||||
sh -c "export ASSETS_DIR='../../assets'; ./scripts/desktop/make-deb-linux.sh"
|
||||
|
||||
# Copy deb
|
||||
docker cp \
|
||||
@@ -121,7 +128,7 @@ for os in '22.04' '24.04'; do
|
||||
# Appimage
|
||||
docker exec \
|
||||
-t "${container_name}" \
|
||||
sh -c './scripts/desktop/make-appimage-linux.sh && mv ./apps/multiplatform/release/main/*imple*.AppImage ./apps/multiplatform/release/main/simplex.appimage'
|
||||
sh -c "export ASSETS_DIR='../../assets'; ./scripts/desktop/make-appimage-linux.sh && mv ./apps/multiplatform/release/main/*imple*.AppImage ./apps/multiplatform/release/main/simplex.appimage"
|
||||
|
||||
# Copy appimage
|
||||
docker cp \
|
||||
|
||||
Reference in New Issue
Block a user