Merge branch 'master' into ab/resize-image

This commit is contained in:
Alexandre Esteves
2025-12-22 03:48:56 +00:00
988 changed files with 169365 additions and 41213 deletions
+12 -4
View File
@@ -102,6 +102,7 @@ build() {
sed -i.bak 's/jniLibs.useLegacyPackaging =.*/jniLibs.useLegacyPackaging = true/' "$folder/apps/multiplatform/android/build.gradle.kts"
sed -i.bak '/android {/a lint {abortOnError = false}' "$folder/apps/multiplatform/android/build.gradle.kts"
sed -i.bak '/tasks/Q' "$folder/apps/multiplatform/android/build.gradle.kts"
sed -i.bak "s/android.version_code=.*/android.version_code=${vercode}/" "$folder/apps/multiplatform/gradle.properties"
for arch in $arches; do
if [ "$arch" = "armv7a" ]; then
@@ -138,10 +139,15 @@ build() {
mkdir -p "$android_tmp_folder"
unzip -oqd "$android_tmp_folder" "$android_apk_output"
# Determenistic build
find "$android_tmp_folder" -type f -exec chmod 644 {} +
find "$android_tmp_folder" -type d -exec chmod 755 {} +
find "$android_tmp_folder" -exec touch -h -d '@1764547200' {} +
(
cd "$android_tmp_folder" && \
zip -rq5 "$tmp/$android_apk_output_final" . && \
zip -rq0 "$tmp/$android_apk_output_final" resources.arsc res
find . -type f -print0 | sort -z | xargs -0 zip -X -rq5 "$tmp/$android_apk_output_final" && \
find res resources.arsc -type f -print0 | sort -z | xargs -0 zip -X -rq0 "$tmp/$android_apk_output_final"
)
zipalign -p -f 4 "$tmp/$android_apk_output_final" "$PWD/$android_apk_output_final"
@@ -164,8 +170,10 @@ pre() {
done
shift $(( $OPTIND - 1 ))
commit="${1:-HEAD}"
vercode="${1}"
commit="${2:-HEAD}"
}
main() {
+30 -7
View File
@@ -29,20 +29,43 @@ for ORIG_NAME in "${ORIG_NAMES[@]}"; do
ORIG_NAME_COPY=$ORIG_NAME-copy
mv "$ORIG_NAME" "$ORIG_NAME_COPY"
(cd apk && zip -r -q -"$level" ../"$ORIG_NAME" .)
# Shouldn't be compressed because of Android requirement
(cd apk && zip -r -q -0 ../"$ORIG_NAME" resources.arsc)
# Determenistic build
find apk -type f -exec chmod 644 {} +
find apk -type d -exec chmod 755 {} +
find apk -exec touch -h -d '2025-12-01T00:00:00' {} +
(
cd apk
find . -not -path './res/*' -not -name 'resources.arsc' -type f -print0 | sort -z | xargs -0 zip -X -r -q -"$level" ../"$ORIG_NAME"
)
if [ $case_insensitive -eq 1 ]; then
# For case-insensitive file systems
list_of_files=$(unzip -l "$ORIG_NAME_COPY" | grep res/ | sed -e "s|.*res/|res/|")
for file in $list_of_files; do unzip -o -q -d apk "$ORIG_NAME_COPY" "$file" && (cd apk && zip -r -q -0 ../"$ORIG_NAME" "$file"); done
list_of_files=$(unzip -l "$ORIG_NAME_COPY" | grep res/ | sed -e "s|.*res/|res/|" | sort -z)
for file in $list_of_files; do
unzip -o -q -d apk "$ORIG_NAME_COPY" "$file"
(
cd apk
chmod 644 "$file"
touch -h -d '2025-12-01T00:00:00' "$file"
zip -X -r -q -0 ../"$ORIG_NAME" "$file"
)
done
else
# This method is not working correctly on case-insensitive file systems since Android AAPT produce the same names of files
# but with different case like xX.png, Xx.png, xx.png, etc
(cd apk && zip -r -q -0 ../"$ORIG_NAME" res)
(
cd apk
find res -type f -print0 | sort -z | xargs -0 zip -X -r -q -0 ../"$ORIG_NAME"
)
fi
# Shouldn't be compressed because of Android requirement
(
cd apk
find resources.arsc -type f -print0 | sort -z | xargs -0 zip -X -r -q -0 ../"$ORIG_NAME"
)
#(cd apk && 7z a -r -mx=$level -tzip -x!resources.arsc ../$ORIG_NAME .)
#(cd apk && 7z a -r -mx=0 -tzip ../$ORIG_NAME resources.arsc)
@@ -61,4 +84,4 @@ for ORIG_NAME in "${ORIG_NAMES[@]}"; do
rm "$ORIG_NAME_COPY" 2> /dev/null || true
rm -rf apk || true
rm "${ORIG_NAME}".idsig 2> /dev/null || true
done
done
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Taken from: https://github.com/apache/arrow/blob/main/ci/scripts/util_free_space.sh
set -eux
df -h
echo "::group::/usr/local/*"
du -hsc /usr/local/*
echo "::endgroup::"
# ~1GB
sudo rm -rf \
/usr/local/aws-sam-cil \
/usr/local/julia* || :
echo "::group::/usr/local/bin/*"
du -hsc /usr/local/bin/*
echo "::endgroup::"
# ~1GB (From 1.2GB to 214MB)
sudo rm -rf \
/usr/local/bin/aliyun \
/usr/local/bin/azcopy \
/usr/local/bin/bicep \
/usr/local/bin/cmake-gui \
/usr/local/bin/cpack \
/usr/local/bin/helm \
/usr/local/bin/hub \
/usr/local/bin/kubectl \
/usr/local/bin/minikube \
/usr/local/bin/node \
/usr/local/bin/packer \
/usr/local/bin/pulumi* \
/usr/local/bin/sam \
/usr/local/bin/stack \
/usr/local/bin/terraform || :
# 142M
sudo rm -rf /usr/local/bin/oc || : \
echo "::group::/usr/local/share/*"
du -hsc /usr/local/share/*
echo "::endgroup::"
# 506MB
sudo rm -rf /usr/local/share/chromium || :
# 1.3GB
sudo rm -rf /usr/local/share/powershell || :
echo "::group::/usr/local/lib/*"
du -hsc /usr/local/lib/*
echo "::endgroup::"
# 15GB
sudo rm -rf /usr/local/lib/android || :
# 341MB
sudo rm -rf /usr/local/lib/heroku || :
# 1.2GB
sudo rm -rf /usr/local/lib/node_modules || :
echo "::group::/opt/*"
du -hsc /opt/*
echo "::endgroup::"
# 679MB
sudo rm -rf /opt/az || :
echo "::group::/opt/microsoft/*"
du -hsc /opt/microsoft/*
echo "::endgroup::"
# 197MB
sudo rm -rf /opt/microsoft/powershell || :
echo "::group::/opt/hostedtoolcache/*"
du -hsc /opt/hostedtoolcache/*
echo "::endgroup::"
# 5.3GB
sudo rm -rf /opt/hostedtoolcache/CodeQL || :
# 1.4GB
sudo rm -rf /opt/hostedtoolcache/go || :
# 489MB
sudo rm -rf /opt/hostedtoolcache/PyPy || :
# 376MB
sudo rm -rf /opt/hostedtoolcache/node || :
# Remove Web browser packages
for pkg in firefox google-chrome-stable microsoft-edge-stable; do
sudo apt purge -y "$pkg" || echo "Failed or not installed: $pkg"
done
df -h
+291
View File
@@ -0,0 +1,291 @@
## Transfer data from SQLite to Postgres database
1. Decrypt SQLite database if it is encrypted.
1. Agent:
1. Open sqlite db:
```sh
sqlcipher simplex_v1_agent.db
```
2. Run in sqlcipher:
```sql
PRAGMA key = '<your_password>'; -- Set your db password
SELECT count(*) FROM sqlite_master; -- Check if db was successfully decrypted
ATTACH DATABASE 'simplex_v1_agent_plaintext.db' AS plaintext KEY ''; -- Attach new empty db
SELECT sqlcipher_export('plaintext'); -- Export opened db to attached db as plaintext
DETACH DATABASE plaintext;
```
2. Chat:
1. Open sqlite db:
```sh
sqlcipher simplex_v1_chat.db
```
2. Run in sqlcipher:
```sql
PRAGMA key = '<your_password>';
SELECT count(*) FROM sqlite_master;
ATTACH DATABASE 'simplex_v1_chat_plaintext.db' AS plaintext KEY '';
SELECT sqlcipher_export('plaintext');
DETACH DATABASE plaintext;
```
2. Prepare Postgres database.
1. Connect to PostgreSQL databse:
```sh
psql -U postgres -h localhost
```
2. Run in psql:
```sql
CREATE USER simplex WITH ENCRYPTED PASSWORD '123123'; -- Create user with password
-- or
-- CREATE USER simplex;
CREATE DATABASE simplex_v1; -- Create database
GRANT ALL PRIVILEGES ON DATABASE simplex_v1 TO simplex; -- Assign permissions
```
3. Prepare database:
You should build the CLI binary from the same `TAG` as the desktop.
1. Build CLI with PostgreSQL support:
```sh
cabal build -fclient_postgres exe:simplex-chat
```
And rename it to:
```sh
mv simplex-chat simplex-chat-pg
```
2. Execute CLI:
```sh
./simplex-chat-pg -d "postgresql://simplex:123123@localhost:5432/simplex_v1" --create-schema
```
Press `Ctrl+C` when CLI ask for a display name.
This should create `simplex_v1_agent_schema` and `simplex_v1_chat_schema` schemas in `simplex_v1` database, with `migrations` tables populated. Some tables would have initialization data - it will be truncated via pgloader command in next step.
3. Load data from decrypted SQLite databases to Postgres database via pgloader.
Install pgloader and add it to PATH. Run in shell (substitute paths):
```sh
export POSTGRES_CONN='postgresql://simplex:123123@localhost:5432/simplex_v1'
```
And then:
```sh
SQLITE_DBPATH='simplex_v1_agent_plaintext.db' \
POSTGRES_SCHEMA='simplex_v1_agent_schema' \
CPU_CORES=$(nproc) WORKERS=$((CPU_CORES - 1)) pgloader --dynamic-space-size 262144 --on-error-stop sqlite.load
SQLITE_DBPATH='simplex_v1_chat_plaintext.db' \
POSTGRES_SCHEMA='simplex_v1_chat_schema' \
CPU_CORES=$(nproc) WORKERS=$((CPU_CORES - 1)) pgloader --dynamic-space-size 262144 --on-error-stop sqlite.load
```
4. Update sequences for Postgres tables.
Connect to db:
```sh
PGPASSWORD=123123 psql -h localhost -U simplex -d simplex_v1
```
Execute the following:
1. For `agent`:
```sql
DO $$
DECLARE
rec RECORD;
BEGIN
EXECUTE 'SET SEARCH_PATH TO simplex_v1_agent_schema';
FOR rec IN
SELECT
table_name,
column_name,
pg_get_serial_sequence(table_name, column_name) AS seq_name
FROM
information_schema.columns
WHERE
table_schema = 'simplex_v1_agent_schema'
AND identity_generation = 'ALWAYS'
LOOP
EXECUTE format(
'SELECT setval(%L, (SELECT MAX(%I) FROM %I))',
rec.seq_name, rec.column_name, rec.table_name
);
END LOOP;
END $$;
```
2. For `chat`:
```sql
DO $$
DECLARE
rec RECORD;
BEGIN
EXECUTE 'SET SEARCH_PATH TO simplex_v1_chat_schema';
FOR rec IN
SELECT
table_name,
column_name,
pg_get_serial_sequence(table_name, column_name) AS seq_name
FROM
information_schema.columns
WHERE
table_schema = 'simplex_v1_chat_schema'
AND identity_generation = 'ALWAYS'
LOOP
EXECUTE format(
'SELECT setval(%L, (SELECT MAX(%I) FROM %I))',
rec.seq_name, rec.column_name, rec.table_name
);
END LOOP;
END $$;
```
5. Compare number of rows between Postgres and SQLite tables.
**PostgreSQL**:
1. For `agent`:
```sql
WITH tbl AS (
SELECT table_schema, table_name
FROM information_schema.Tables
WHERE table_name NOT LIKE 'pg_%'
AND table_schema IN ('simplex_v1_agent_schema')
)
SELECT
table_schema AS schema_name,
table_name,
(xpath('/row/c/text()', query_to_xml(
format('SELECT count(*) AS c FROM %I.%I', table_schema, table_name), false, true, ''
)))[1]::text::int AS records_count
FROM tbl
ORDER BY records_count DESC;
```
2. For `chat`:
```sql
WITH tbl AS (
SELECT table_schema, table_name
FROM information_schema.Tables
WHERE table_name NOT LIKE 'pg_%'
AND table_schema IN ('simplex_v1_chat_schema')
)
SELECT
table_schema AS schema_name,
table_name,
(xpath('/row/c/text()', query_to_xml(
format('SELECT count(*) AS c FROM %I.%I', table_schema, table_name), false, true, ''
)))[1]::text::int AS records_count
FROM tbl
ORDER BY records_count DESC;
```
**SQLite**:
1. For `agent`:
```sh
db="simplex_v1_agent_plaintext.db"
sqlite3 "$db" "SELECT name FROM sqlite_master WHERE type='table';" |
while read table; do
count=$(sqlite3 "$db" "SELECT COUNT(*) FROM \"$table\";")
echo "$table: $count"
done | sort -k2 -nr | less
```
2. For `chat`:
```sh
db="simplex_v1_chat_plaintext.db"
sqlite3 "$db" "SELECT name FROM sqlite_master WHERE type='table';" |
while read table; do
count=$(sqlite3 "$db" "SELECT COUNT(*) FROM \"$table\";")
echo "$table: $count"
done | sort -k2 -nr | less
```
6. Build and run desktop app with Postgres backend.
Run in shell (paths are from project root):
```sh
./scripts/desktop/build-lib-mac.sh arm64 postgres
./gradlew runDistributable -Pdatabase.backend=postgres
# or
./gradlew packageDmg -Pdatabase.backend=postgres
```
## Transfer data from Postgres to SQLite database
1. Prepare sqlite db:
1. Download simplex-chat CLI:
You should download the CLI binary from the same `TAG` as the desktop.
```sh
export TAG='v6.4.3.1'
curl -L "https://github.com/simplex-chat/simplex-chat/releases/download/${TAG}/simplex-chat-ubuntu-22_04-x86_64" -o 'simplex-chat'
```
2. Run the CLI:
```sh
./simplex-chat
```
Press `Ctrl+C` when CLI ask for a display name.
3. Move database:
```sh
mv ~/.simplex/simplex_v1_* ~/.local/share/simplex/
```
2. Transfer data:
```sh
./pg2sqlite.py --verbose 'postgresql://simplex:123123@localhost:5432/simplex_v1' ~/.local/share/simplex/
```
4. Update BLOBs:
```sh
sqlite3 simplex_v1_chat.db
```
```sh
UPDATE group_members SET member_role = CAST(member_role as BLOB);
UPDATE user_contact_links SET group_link_member_role = CAST(group_link_member_role AS BLOB) WHERE group_link_member_role is not null;
```
+899
View File
@@ -0,0 +1,899 @@
#!/usr/bin/env python3
"""
PostgreSQL -> per-schema SQLite migration with colored, clean logging.
Usage example:
python db_migrate.py 'postgresql://user:pass@host:5432/db' /path/to/sqlite/dir --dry-run
Note: color output will be disabled automatically if stdout is not a TTY or if --no-color is passed.
"""
from __future__ import annotations
import argparse
import logging
import os
import sqlite3
import sys
import re
import datetime
from pathlib import Path
from typing import Set, List, Tuple, Dict, Optional, NamedTuple
import psycopg
from psycopg import sql
# ----------------------
# Small utilities
# ----------------------
try:
sqlite3.register_adapter(
datetime.datetime,
lambda v: v.isoformat(sep=" ", timespec="microseconds"),
)
sqlite3.register_adapter(datetime.date, lambda v: v.isoformat())
sqlite3.register_adapter(
datetime.time, lambda v: v.isoformat(timespec="microseconds")
)
except ValueError:
pass # already registered
ANSI = {
"reset": "\x1b[0m",
"bold": "\x1b[1m",
"dim": "\x1b[2m",
"red": "\x1b[31m",
"green": "\x1b[32m",
"yellow": "\x1b[33m",
"blue": "\x1b[34m",
"magenta": "\x1b[35m",
"cyan": "\x1b[36m",
"gray": "\x1b[90m",
}
DEFAULT_BATCH_SIZE = 10000
TYPE_COMPATIBILITY = {
"bytea": ["BLOB", "CHAR", "CLOB", "TEXT", "JSON"],
"int": ["INT", "NUMERIC"],
"serial": ["INT", "NUMERIC"],
"numeric": ["NUMERIC", "DECIMAL", "REAL", "FLOAT", "DOUBLE"],
"decimal": ["NUMERIC", "DECIMAL", "REAL", "FLOAT", "DOUBLE"],
"real": ["NUMERIC", "DECIMAL", "REAL", "FLOAT", "DOUBLE"],
"double": ["NUMERIC", "DECIMAL", "REAL", "FLOAT", "DOUBLE"],
"float": ["NUMERIC", "DECIMAL", "REAL", "FLOAT", "DOUBLE"],
"money": ["NUMERIC", "DECIMAL", "REAL", "FLOAT", "DOUBLE"],
"bool": ["BOOL", "INT", "NUMERIC"],
"varchar": ["CHAR", "CLOB", "TEXT"],
"char": ["CHAR", "CLOB", "TEXT"],
"text": ["CHAR", "CLOB", "TEXT"],
"citext": ["CHAR", "CLOB", "TEXT"],
"timestamp": ["DATE", "TIME", "CHAR", "TEXT", "DATETIME"],
"time": ["DATE", "TIME", "CHAR", "TEXT", "DATETIME"],
"date": ["DATE", "TIME", "CHAR", "TEXT", "DATETIME"],
"uuid": ["CHAR", "TEXT", "UUID", "CLOB"],
"json": ["JSON", "TEXT", "CHAR", "CLOB"],
"jsonb": ["JSON", "TEXT", "CHAR", "CLOB"],
}
def _sanitize_cursor_name(s: str) -> str:
return re.sub(r"[^A-Za-z0-9_]+", "_", s)
def supports_color(force_no: bool) -> bool:
"""Return True when we should emit ANSI colors."""
if force_no:
return False
if os.getenv("NO_COLOR"):
return False
term = os.getenv("TERM", "")
if term == "" or term.lower() == "dumb":
return False
try:
isatty = sys.stdout.isatty()
except Exception:
isatty = False
return isatty
class ColoredFormatter(logging.Formatter):
LEVEL_COLORS = {
logging.DEBUG: ANSI["gray"],
logging.INFO: ANSI["green"],
logging.WARNING: ANSI["yellow"],
logging.ERROR: ANSI["red"],
logging.CRITICAL: ANSI["red"] + ANSI["bold"],
}
TAG_COLORS = {
"SKIP": ANSI["yellow"],
"SCHEMA": ANSI["blue"],
"OK": ANSI["magenta"],
}
def __init__(self, use_color: bool = True):
super().__init__(fmt="%(message)s")
self.use_color = use_color
def format(self, record: logging.LogRecord) -> str:
msg = super().format(record)
parts = msg.split(" ", 1)
tag = parts[0]
rest = parts[1] if len(parts) > 1 else ""
plain_label = f"[{tag}]"
if not self.use_color:
return f"{plain_label}{(' ' + rest) if rest else ''}"
color = self.TAG_COLORS.get(
tag.upper(), self.LEVEL_COLORS.get(record.levelno, "")
)
reset = ANSI["reset"]
return f"{color}{plain_label}{reset}{(' ' + rest) if rest else ''}"
def setup_logger(verbose: bool, no_color: bool) -> logging.Logger:
use_color = supports_color(no_color)
logger = logging.getLogger("db_migrate")
logger.setLevel(logging.DEBUG if verbose else logging.INFO)
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG if verbose else logging.INFO)
handler.setFormatter(ColoredFormatter(use_color=use_color))
logger.handlers.clear()
logger.addHandler(handler)
logging.getLogger("psycopg").setLevel(logging.WARNING)
if verbose:
logger.debug(f"color_support: {use_color}")
try:
isatty = sys.stdout.isatty()
except Exception:
isatty = False
logger.debug(
f"TERM={os.getenv('TERM', '')!r} "
f"NO_COLOR={os.getenv('NO_COLOR')!r} "
f"isatty={isatty}"
)
return logger
def quote_sqlite_identifier(name: str) -> str:
return '"' + name.replace('"', '""') + '"'
def quote_pg_identifier(name: str) -> str:
"""Simple PG identifier quoting for building safe SQL strings."""
return '"' + name.replace('"', '""') + '"'
def sqlite_decl_satisfies(pg_type: str, sqlite_decl: str) -> bool:
# Treat empty/blank SQLite declarations more permissively based on PG type.
decl_raw = sqlite_decl or ""
if not decl_raw.strip():
pg = (pg_type or "").lower()
# arrays -> textual affinity
if pg.endswith("[]"):
return True
# integer-like
if re.search(r"\b(?:int|serial|bigint)\b", pg):
return True
# numeric/float
if re.search(r"\b(?:numeric|decimal|real|double|float|money)\b", pg):
return True
# boolean
if re.search(r"\b(?:bool|boolean)\b", pg):
return True
# binary
if re.search(r"\b(?:bytea)\b", pg):
return True
# textual/json/uuid/timestamps/dates/times
if re.search(
r"\b(?:varchar|char|text|citext|jsonb|json|uuid|timestamp|time|date)\b", pg
):
return True
# conservative fallback: accept empty decl as permissive
return True
decl = decl_raw.upper()
pg = (pg_type or "").lower()
# array type
if pg.endswith("[]"):
return any(tok in decl for tok in ("TEXT", "CHAR", "CLOB", "JSON"))
for key, allowed_types in TYPE_COMPATIBILITY.items():
if re.search(r"\b" + re.escape(key) + r"\b", pg):
return any(tok in decl for tok in allowed_types)
return any(
tok in decl for tok in ("TEXT", "CHAR", "CLOB", "NUMERIC", "BLOB", "INT")
)
# ----------------------
# Postgres helpers
# ----------------------
def list_user_schemas(pg_cursor) -> List[str]:
pg_cursor.execute(
"""
SELECT nspname
FROM pg_namespace
WHERE nspname NOT LIKE 'pg_%'
AND nspname != 'information_schema'
AND nspname != 'public'
ORDER BY nspname;
"""
)
return [r[0] for r in pg_cursor.fetchall()]
def list_tables_in_schema(pg_cursor, schema: str) -> List[str]:
pg_cursor.execute(
sql.SQL(
"""
SELECT table_name
FROM information_schema.tables
WHERE table_schema = %s
ORDER BY table_name;
"""
),
(schema,),
)
return [r[0] for r in pg_cursor.fetchall()]
def get_pg_columns(pg_cursor, schema: str, table: str) -> List[Tuple[str, str]]:
pg_cursor.execute(
sql.SQL(
"""
SELECT column_name, data_type
FROM information_schema.columns
WHERE table_schema = %s AND table_name = %s
ORDER BY ordinal_position;
"""
),
(schema, table),
)
return [(r[0], r[1]) for r in pg_cursor.fetchall()]
# ----------------------
# SQLite helpers
# ----------------------
class SQLiteCol(NamedTuple):
cid: int
name: str
type: str
notnull: int
dflt_value: str
pk: int
def get_sqlite_table_info(sqlite_cursor, table_name: str) -> list[SQLiteCol]:
qname = quote_sqlite_identifier(table_name)
sqlite_cursor.execute(f"PRAGMA table_info({qname});")
return [SQLiteCol(*row) for row in sqlite_cursor.fetchall()]
def sqlite_table_has_autoincrement(sqlite_cursor, table_name: str) -> bool:
sqlite_cursor.execute(
"SELECT sql FROM sqlite_master WHERE type='table' AND name = ?;", (table_name,)
)
row = sqlite_cursor.fetchone()
if not row or not row[0]:
return False
return "AUTOINCREMENT" in row[0].upper()
def sqlite_sequence_table_exists(sqlite_cursor) -> bool:
sqlite_cursor.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='sqlite_sequence';"
)
exists = sqlite_cursor.fetchone() is not None
if not exists:
logging.getLogger("db_migrate").debug("sqlite_sequence table not present")
return exists
# ----------------------
# Core migration logic
# ----------------------
def build_select_for_sqlite_columns(
schema: str,
table: str,
sqlite_cols: list[SQLiteCol],
pg_columns_map: Dict[str, Tuple[str, str]],
) -> str:
"""Return a plain SQL string (no trailing semicolon) selecting PG columns in the order of sqlite_cols.
Uses simple identifier quoting to avoid psycopg.sql objects which may vary by driver version.
"""
select_parts = []
for col in sqlite_cols:
lc = col.name.lower()
if lc in pg_columns_map:
pg_name, pg_type = pg_columns_map[lc]
pg_type_l = (pg_type or "").lower()
if any(tok in pg_type_l for tok in ("timestamp", "time")):
expr = f"{quote_pg_identifier(pg_name)}::text"
else:
expr = f"{quote_pg_identifier(pg_name)}"
select_parts.append(expr)
else:
select_parts.append("NULL")
select_list = ", ".join(select_parts)
# Quote schema and table names simply (this is not comprehensive for every corner case but avoids psycopg.sql use)
return f"SELECT {select_list} FROM {quote_pg_identifier(schema)}.{quote_pg_identifier(table)}"
def validate_column_compatibility(
sqlite_cols: list[SQLiteCol],
pg_columns_map: Dict[str, Tuple[str, str]],
schema: str,
table: str,
) -> None:
for col in sqlite_cols:
lc = col.name.lower()
if lc not in pg_columns_map:
continue
pg_type = pg_columns_map[lc][1]
if not sqlite_decl_satisfies(pg_type, col.type):
raise ValueError(
f"Type mismatch for {schema}.{table}.{col.name}: "
f"PG='{pg_type}' vs SQLite='{col.type}'"
)
def process_row_for_sqlite(row: Tuple, sqlite_cols: list[SQLiteCol]) -> Tuple:
out = []
for i, val in enumerate(row):
col = sqlite_cols[i]
decl_raw = col.type or ""
decl = decl_raw.upper()
decl_is_empty = decl_raw.strip() == ""
if isinstance(val, memoryview):
b = val.tobytes()
if "BLOB" in decl or decl_is_empty:
out.append(sqlite3.Binary(b))
else:
try:
out.append(b.decode("utf-8"))
except UnicodeDecodeError as e:
raise ValueError(
f"UTF-8 decode failed for column '{col.name}': {e}"
)
elif isinstance(val, (bytes, bytearray)):
b = bytes(val)
if "BLOB" in decl or decl_is_empty:
out.append(sqlite3.Binary(b))
else:
try:
out.append(b.decode("utf-8"))
except UnicodeDecodeError as e:
raise ValueError(
f"UTF-8 decode failed for column '{col.name}': {e}"
)
else:
out.append(val)
return tuple(out)
def fetch_and_validate_row_batches(
pg_cursor,
select_sql: str,
sqlite_cols: list[SQLiteCol],
batch_size: int = DEFAULT_BATCH_SIZE,
):
pg_cursor.execute(select_sql)
total_rows_seen = 0
while True:
rows = pg_cursor.fetchmany(batch_size)
if not rows:
break
validated_batch = []
for row in rows:
total_rows_seen += 1
try:
validated_batch.append(process_row_for_sqlite(row, sqlite_cols))
except ValueError as e:
raise ValueError(f"Row {total_rows_seen} validation error: {e}")
yield validated_batch
def _get_postgres_sequence_info(
pg_cursor, schema: str, table: str, pk_col: str
) -> Optional[tuple[str, int]]:
"""Get PostgreSQL sequence name and last_value for a table primary key, if any.
Returns (sequence_name_text, last_value) or None.
"""
pg_cursor.execute(
"""
SELECT data_type
FROM information_schema.columns
WHERE table_schema = %s AND table_name = %s AND column_name = %s;
""",
(schema, table, pk_col),
)
r = pg_cursor.fetchone()
if not r:
return None
pg_type = (r[0] or "").lower()
# Match whole words to avoid matching 'interval' etc.
if not re.search(r"\b(?:int|serial|bigint)\b", pg_type):
return None
# Get sequence name text (NULL if none)
pg_cursor.execute(
"SELECT pg_get_serial_sequence(%s, %s);",
(f"{schema}.{table}", pk_col),
)
row = pg_cursor.fetchone()
seq_name = row[0] if row else None
if not seq_name:
return None
try:
# Read last_value by casting the pg_get_serial_sequence result to regclass.
pg_cursor.execute(
"SELECT last_value FROM pg_get_serial_sequence(%s, %s)::regclass;",
(f"{schema}.{table}", pk_col),
)
rr = pg_cursor.fetchone()
if rr and rr[0] is not None:
return (seq_name, int(rr[0]))
except Exception:
# Swallow errors (best-effort); callers treat missing info as absent
pass
return None
def _get_sqlite_max_pk_value(sqlite_cursor, table: str, pk_col: str) -> Optional[int]:
"""Get the maximum primary key value from SQLite table."""
logger = logging.getLogger("db_migrate")
try:
sqlite_cursor.execute(
f"SELECT MAX({quote_sqlite_identifier(pk_col)}) "
f"FROM {quote_sqlite_identifier(table)};"
)
r3 = sqlite_cursor.fetchone()
if r3 and r3[0] is not None:
return int(r3[0])
except Exception:
logger.debug("Failed to read sqlite max pk", exc_info=True)
return None
def _update_sqlite_sequence(
sqlite_conn: sqlite3.Connection, table: str, sequence_value: int
) -> bool:
"""Update SQLite sqlite_sequence with new value.
IMPORTANT: This function does not commit; caller must commit/rollback.
"""
s_cur = sqlite_conn.cursor()
s_cur.execute(
"UPDATE sqlite_sequence SET seq = ? WHERE name = ?;",
(sequence_value, table),
)
if s_cur.rowcount == 0:
s_cur.execute(
"INSERT INTO sqlite_sequence(name, seq) VALUES (?, ?);",
(table, sequence_value),
)
return True
def try_update_sqlite_sequence(
pg_cursor,
sqlite_conn: sqlite3.Connection,
schema: str,
pg_table: str, # PostgreSQL table name
sqlite_table: str, # SQLite table name
sqlite_cols: list[SQLiteCol],
) -> bool:
"""Update SQLite sequence table based on PostgreSQL sequence values."""
logger = logging.getLogger("db_migrate")
# Check if table has a single primary key
pks = [c for c in sqlite_cols if c.pk]
if len(pks) != 1:
return False
pk_col = pks[0].name
# Check if SQLite has AUTOINCREMENT (use sqlite_table)
s_cur = sqlite_conn.cursor()
if not sqlite_table_has_autoincrement(s_cur, sqlite_table):
return False
if not sqlite_sequence_table_exists(s_cur):
return False
# Get PostgreSQL sequence info (use pg_table)
seq_info = _get_postgres_sequence_info(pg_cursor, schema, pg_table, pk_col)
pg_last_value = seq_info[1] if seq_info else None
# Get SQLite max PK value (use sqlite_table)
sqlite_max = _get_sqlite_max_pk_value(s_cur, sqlite_table, pk_col)
# Determine the value to use
if pg_last_value is not None and sqlite_max is not None:
candidate = max(pg_last_value, sqlite_max)
elif pg_last_value is not None:
candidate = pg_last_value
elif sqlite_max is not None:
candidate = sqlite_max
else:
return False
updated = _update_sqlite_sequence(sqlite_conn, sqlite_table, candidate)
if updated:
try:
sqlite_conn.commit()
except Exception:
logger.debug("Failed to commit sqlite_sequence update", exc_info=True)
# Let caller proceed; treat as best-effort
return updated
# ----------------------
# Flow: per-schema migration
# ----------------------
def migrate_schema(
pg_conn,
sqlite_dir: str,
schema: str,
skipped_tables: Set[str],
logger: logging.Logger,
dry_run: bool = False,
batch_size: int = DEFAULT_BATCH_SIZE,
) -> Tuple[int, int]:
"""Migrate a single schema. Returns (tables_migrated, rows_inserted_total)."""
processed_schema = schema[:-7] if schema.endswith("_schema") else schema
sqlite_db_path = Path(sqlite_dir) / f"{processed_schema}.db"
if not sqlite_db_path.is_file():
logger.error(f"Missing SQLite DB: {sqlite_db_path}")
return 0, 0
tables_migrated = 0
rows_inserted = 0
sqlite_path = sqlite_db_path.resolve()
if dry_run:
uri = sqlite_path.as_uri() + "?mode=ro"
conn_args = {"database": uri, "uri": True}
else:
conn_args = {"database": str(sqlite_path)}
with sqlite3.connect(**conn_args) as sqlite_conn:
sqlite_cur = sqlite_conn.cursor()
sqlite_cur.execute("SELECT name FROM sqlite_master WHERE type='table';")
sqlite_tables = [r[0] for r in sqlite_cur.fetchall()]
# O(1) lookup map for matching by lowercase name
sqlite_table_map = {t.lower(): t for t in sqlite_tables}
with pg_conn.cursor() as pg_cur:
# Ensure we start in a clean state for this connection
try:
pg_conn.rollback()
except Exception:
# Ignore rollback failure; connection is newly opened most likely
logger.debug(
"pg_conn.rollback() at schema start ignored", exc_info=True
)
# Get table list for this schema, but catch/rollback on failure
try:
tables = list_tables_in_schema(pg_cur, schema)
except Exception as e:
logger.error("ERROR %s (list tables): %s", schema, e)
logger.debug("Traceback (list tables):", exc_info=True)
try:
pg_conn.rollback()
except Exception:
logger.debug(
"pg_conn.rollback() failed after list tables error",
exc_info=True,
)
return 0, 0
for table in tables:
# Defensive: ensure connection is in a clean state before any new PG work.
# A prior error can leave the connection in an aborted transaction; calling
# rollback() clears that and allows subsequent SELECTs to run.
try:
pg_conn.rollback()
except Exception:
# ignore: best-effort cleanup
logger.debug(
"pg_conn.rollback() ignored at start of table loop",
exc_info=True,
)
if table.lower() in skipped_tables:
logger.info("SKIP %s.%s (explicit)", schema, table)
continue
if table.lower() not in sqlite_table_map:
logger.info("SKIP %s.%s (no target table)", schema, table)
continue
matched_table = sqlite_table_map[table.lower()]
logger.info("OK %s.%s -> %s", schema, table, matched_table)
# Fetch PG columns, but defend against aborted transaction here
try:
pg_cols = get_pg_columns(pg_cur, schema, table)
except Exception as e:
logger.error("ERROR %s.%s (get columns): %s", schema, table, e)
logger.debug("Traceback (get columns):", exc_info=True)
try:
pg_conn.rollback()
except Exception:
logger.debug(
"pg_conn.rollback() failed after get columns error",
exc_info=True,
)
continue
pg_map: Dict[str, Tuple[str, str]] = {
name.lower(): (name, dtype) for name, dtype in pg_cols
}
sqlite_info = get_sqlite_table_info(sqlite_cur, matched_table)
if not sqlite_info:
logger.warning(f"SKIP {schema}.{table} (no sqlite info)")
continue
# Warn once about schema drift (extra columns)
pg_col_names = {name.lower() for name, _ in pg_cols}
extra_in_sqlite = {
c.name for c in sqlite_info if c.name.lower() not in pg_col_names
}
if extra_in_sqlite:
logger.warning(
f"Schema drift: {schema}.{table} has extra SQLite columns {extra_in_sqlite}"
)
try:
validate_column_compatibility(sqlite_info, pg_map, schema, table)
except ValueError as e:
logger.error(f"ERROR {schema}.{table} (type mismatch): {e}")
continue
select_sql = build_select_for_sqlite_columns(
schema, table, sqlite_info, pg_map
)
csr_name = _sanitize_cursor_name(f"csr_{schema}_{table}")
try:
with pg_conn.cursor(name=csr_name) as data_cur:
batch_gen = fetch_and_validate_row_batches(
data_cur, select_sql, sqlite_info, batch_size=batch_size
)
first_batch = next(batch_gen, None)
if not first_batch:
logger.info(f"SKIP {schema}.{table} (no rows)")
# data_cur will be closed automatically on leaving the 'with'
continue
if dry_run:
count = len(first_batch)
for batch in batch_gen:
count += len(batch)
inserted = 0
logger.info(
f"DRY {schema}.{table} rows_validated={count}"
)
else:
cur = sqlite_conn.cursor()
quoted_table = quote_sqlite_identifier(matched_table)
col_names = [c.name for c in sqlite_info]
quoted_cols = ", ".join(
quote_sqlite_identifier(c) for c in col_names
)
placeholders = ", ".join(["?"] * len(col_names))
cur.execute("BEGIN;")
try:
cur.execute(f"DELETE FROM {quoted_table};")
inserted = 0
if first_batch:
cur.executemany(
f"INSERT INTO {quoted_table} ({quoted_cols}) VALUES ({placeholders});",
first_batch,
)
inserted += len(first_batch)
for batch in batch_gen:
if not batch:
continue
cur.executemany(
f"INSERT INTO {quoted_table} ({quoted_cols}) VALUES ({placeholders});",
batch,
)
inserted += len(batch)
except Exception:
cur.execute("ROLLBACK;")
raise
else:
cur.execute("COMMIT;")
except ValueError as e:
logger.error("ERROR %s.%s (row validation): %s", schema, table, e)
try:
pg_conn.rollback()
except Exception:
logger.debug(
"pg_conn.rollback() failed after row validation error",
exc_info=True,
)
continue
except Exception as e:
logger.error("ERROR %s.%s (select failed): %s", schema, table, e)
logger.debug("Traceback (select failed):", exc_info=True)
try:
pg_conn.rollback()
except Exception:
logger.debug(
"pg_conn.rollback() failed after select failed",
exc_info=True,
)
continue
rows_inserted += inserted
tables_migrated += 1
logger.info(f"DONE {schema}.{table} rows={inserted}")
if not dry_run:
try:
if try_update_sqlite_sequence(
pg_cur,
sqlite_conn,
schema,
table,
matched_table,
sqlite_info,
):
logger.info(
"SEQ %s.%s sqlite_sequence updated", schema, table
)
except Exception as e:
logger.warning(
"SEQ %s.%s update failed (ignored): %s", schema, table, e
)
return tables_migrated, rows_inserted
def migrate_data(
pg_conn_str: str,
sqlite_dir: str,
schema_filter: Optional[str],
dry_run: bool,
skip_tables: str,
logger: logging.Logger,
batch_size: int,
) -> None:
skipped_tables = {t.strip().lower() for t in skip_tables.split(",") if t.strip()}
total_tables = 0
total_rows = 0
total_errors = 0
# List schemas once with a short-lived connection
with psycopg.connect(pg_conn_str) as tmp_conn:
with tmp_conn.cursor() as cur:
schemas = list_user_schemas(cur)
if schema_filter:
schemas = [s for s in schemas if s == schema_filter]
if not schemas:
logger.error("No schemas to process")
return
for schema in schemas:
logger.info("SCHEMA %s", schema)
if dry_run:
logger.info("(dry-run) validating only — no writes will be performed")
# Use a fresh connection per-schema to isolate failures/aborted transactions
try:
with psycopg.connect(pg_conn_str) as pg_conn:
migrated_tables, inserted_rows = migrate_schema(
pg_conn,
sqlite_dir,
schema,
skipped_tables,
logger,
dry_run=dry_run,
batch_size=batch_size,
)
except Exception as e:
logger.error("Schema %s failed: %s", schema, e)
logger.debug("Traceback (schema failure):", exc_info=True)
total_errors += 1
continue
total_tables += migrated_tables
total_rows += inserted_rows
logger.info("---")
logger.info(
f"SUMMARY: schemas={len(schemas)} "
f"tables_migrated={total_tables} "
f"rows_inserted={total_rows} "
f"errors={total_errors}"
)
# ----------------------
# CLI
# ----------------------
def parse_args(argv):
p = argparse.ArgumentParser(
description="Migrate Postgres data into per-schema SQLite DBs (non-invasive)."
)
p.add_argument("pg_conn", help="Postgres connection string")
p.add_argument(
"sqlite_dir", help="Directory containing per-schema sqlite .db files"
)
p.add_argument(
"--schema", help="Only migrate this schema (exact match)", default=None
)
p.add_argument(
"--dry-run",
help="Validate and report only; do not write to sqlite",
action="store_true",
)
p.add_argument("--verbose", help="Verbose logging (debug)", action="store_true")
p.add_argument("--no-color", help="Disable ANSI color output", action="store_true")
p.add_argument(
"--batch-size",
type=int,
default=DEFAULT_BATCH_SIZE,
help="Batch size for data fetching",
)
p.add_argument(
"--skip-tables",
help="Comma-separated list of tables to skip",
default="migrations,servers_stats",
)
return p.parse_args(argv[1:])
def main(argv):
args = parse_args(argv)
logger = setup_logger(args.verbose, args.no_color)
sqlite_dir_path = Path(args.sqlite_dir)
if not sqlite_dir_path.is_dir():
logger.error("SQLite directory does not exist or is not a directory.")
raise SystemExit(1)
try:
migrate_data(
args.pg_conn,
str(sqlite_dir_path),
args.schema,
args.dry_run,
args.skip_tables,
logger,
args.batch_size,
)
except Exception as e:
logger.error(f"Fatal error: {e}")
raise SystemExit(2)
if __name__ == "__main__":
main(sys.argv)
+21
View File
@@ -0,0 +1,21 @@
LOAD DATABASE
FROM {{SQLITE_DBPATH}}
INTO {{POSTGRES_CONN}}
WITH include no drop,
truncate,
disable triggers,
create no tables,
create no indexes,
-- pgloader implementation doesn't find "GENERATED ALWAYS AS IDENTITY" sequences,
-- instead we reset sequences manually via custom query after load
reset no sequences,
data only,
workers = {{WORKERS}}
EXCLUDING TABLE NAMES LIKE 'migrations', 'sqlite_sequence'
SET work_mem to '16MB',
maintenance_work_mem to '512 MB',
search_path to '{{POSTGRES_SCHEMA}}'
;
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env sh
set -eu
export SOURCE_DATE_EPOCH=1764547200
CLI_VERSION="$1"
CLI_PATH_TO_BIN="${2:-/out/simplex-chat}"
BUILD_FOLDER="${3:-/out/deb-build}"
size=$(stat -c '%s' "$CLI_PATH_TO_BIN" | awk '{printf "%.0f\n", ($1+1023)/1024}')
arch=$(case "$(uname -m)" in x86_64) printf "amd64" ;; aarch64) printf "arm64" ;; *) printf "unknown" ;; esac)
package='simplex-chat'
mkdir "$BUILD_FOLDER"
cd "$BUILD_FOLDER"
mkdir -p ./${package}/DEBIAN
mkdir -p ./${package}/usr/bin
cat > ./${package}/DEBIAN/control << EOF
Package: ${package}
Version: ${CLI_VERSION}
Section: Messenger
Priority: optional
Architecture: ${arch}
Maintainer: SimpleX Chat <chat@simplex.chat>
Description: SimpleX - the first messaging platform that has no user identifiers of any kind - 100% private by design! (CLI)
Installed-Size: ${size}
EOF
cp "$CLI_PATH_TO_BIN" ./${package}/usr/bin/simplex-chat
chmod +x ./${package}/usr/bin/simplex-chat
find ./${package} -exec touch -d "@${SOURCE_DATE_EPOCH}" {} +
dpkg-deb --build --root-owner-group --uniform-compression ./${package}
strip-nondeterminism "./${package}.deb"
+2 -2
View File
@@ -7,7 +7,7 @@ function readlink() {
}
OS=linux
ARCH=${1:-`uname -a | rev | cut -d' ' -f2 | rev`}
ARCH="$(uname -m)"
GHC_VERSION=9.6.3
if [ "$ARCH" == "aarch64" ]; then
@@ -25,7 +25,7 @@ for elem in "${exports[@]}"; do count=$(grep -R "$elem$" libsimplex.dll.def | wc
for elem in "${exports[@]}"; do count=$(grep -R "\"$elem\"" flake.nix | wc -l); if [ $count -ne 2 ]; then echo Wrong exports in flake.nix. Add \"$elem\" in two places of the file; exit 1; fi ; done
rm -rf $BUILD_DIR
cabal build lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN -flink-rts -threaded' --constraint 'simplexmq +client_library' --constraint 'jpeg-turbo +static-gcc'
cabal build lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN -flink-rts -threaded' --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library' --constraint 'jpeg-turbo +static-gcc'
cd $BUILD_DIR/build
#patchelf --add-needed libHSrts_thr-ghc${GHC_VERSION}.so libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so
#patchelf --add-rpath '$ORIGIN' libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so
+28 -14
View File
@@ -6,6 +6,7 @@ OS=mac
ARCH="${1:-`uname -a | rev | cut -d' ' -f1 | rev`}"
COMPOSE_ARCH=$ARCH
GHC_VERSION=9.6.3
DATABASE_BACKEND="${2:-sqlite}"
if [ "$ARCH" == "arm64" ]; then
ARCH=aarch64
@@ -24,7 +25,14 @@ for elem in "${exports[@]}"; do count=$(grep -R "$elem$" libsimplex.dll.def | wc
for elem in "${exports[@]}"; do count=$(grep -R "\"$elem\"" flake.nix | wc -l); if [ $count -ne 2 ]; then echo Wrong exports in flake.nix. Add \"$elem\" in two places of the file; exit 1; fi ; done
rm -rf $BUILD_DIR
cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library' --constraint 'jpeg-turbo +static'
if [[ "$DATABASE_BACKEND" == "postgres" ]]; then
echo "Building with postgres backend..."
cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library +client_postgres' --constraint 'simplex-chat +client_library +client_postgres' --constraint 'jpeg-turbo +static'
else
echo "Building with sqlite backend..."
cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library' --constraint 'jpeg-turbo +static'
fi
cd $BUILD_DIR/build
mkdir deps 2> /dev/null || true
@@ -33,12 +41,12 @@ mkdir deps 2> /dev/null || true
#cp $GHC_LIBS_DIR/libffi.dylib ./deps
(
BUILD=$PWD
cp /tmp/libffi-3.4.4/*-apple-darwin*/.libs/libffi.dylib $BUILD/deps || \
cp /tmp/libffi-3.5.2/*-apple-darwin*/.libs/libffi.dylib $BUILD/deps || \
( \
cd /tmp && \
curl --tlsv1.2 "https://gitlab.haskell.org/ghc/libffi-tarballs/-/raw/libffi-3.4.4/libffi-3.4.4.tar.gz?inline=false" -o libffi.tar.gz && \
curl --tlsv1.2 "https://gitlab.haskell.org/ghc/libffi-tarballs/-/raw/3914c27381526ce586ea0ac0359a332fd82987af/libffi-3.5.2.tar.gz?inline=false" -o libffi.tar.gz && \
tar -xzvf libffi.tar.gz && \
cd "libffi-3.4.4" && \
cd "libffi-3.5.2" && \
./configure && \
make && \
cp *-apple-darwin*/.libs/libffi.dylib $BUILD/deps \
@@ -99,8 +107,8 @@ cp $BUILD_DIR/build/libHSsimplex-chat-*-inplace-ghc*.$LIB_EXT apps/multiplatform
cd apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
LIBCRYPTO_PATH=$(otool -l libHSdrct-*.$LIB_EXT | grep libcrypto | cut -d' ' -f11)
install_name_tool -change $LIBCRYPTO_PATH @rpath/libcrypto.3.0.$LIB_EXT libHSdrct-*.$LIB_EXT
LIBCRYPTO_PATH=$(otool -l libHSsmplxmq-*.$LIB_EXT | grep libcrypto | cut -d' ' -f11)
install_name_tool -change $LIBCRYPTO_PATH @rpath/libcrypto.3.0.$LIB_EXT libHSsmplxmq-*.$LIB_EXT
cp $LIBCRYPTO_PATH libcrypto.3.0.$LIB_EXT
chmod 755 libcrypto.3.0.$LIB_EXT
install_name_tool -id "libcrypto.3.0.$LIB_EXT" libcrypto.3.0.$LIB_EXT
@@ -111,14 +119,18 @@ if [ -n "$LIBCRYPTO_PATH" ]; then
install_name_tool -change $LIBCRYPTO_PATH @rpath/libcrypto.3.0.$LIB_EXT $LIB
fi
LIBCRYPTO_PATH=$(otool -l libHSsmplxmq*.$LIB_EXT | grep libcrypto | cut -d' ' -f11)
if [ -n "$LIBCRYPTO_PATH" ]; then
install_name_tool -change $LIBCRYPTO_PATH @rpath/libcrypto.3.0.$LIB_EXT libHSsmplxmq*.$LIB_EXT
fi
# We could change libpq and libHSpstgrsql for postgres (?), remove sqlite condition for exit below.
# Unnecessary for now as app with postgres backend is not for distribution.
if [[ "$DATABASE_BACKEND" == "sqlite" ]]; then
LIBCRYPTO_PATH=$(otool -l libHSdrct-*.$LIB_EXT | grep libcrypto | cut -d' ' -f11)
if [ -n "$LIBCRYPTO_PATH" ]; then
install_name_tool -change $LIBCRYPTO_PATH @rpath/libcrypto.3.0.$LIB_EXT libHSdrct-*.$LIB_EXT
fi
LIBCRYPTO_PATH=$(otool -l libHSsqlcphr-*.$LIB_EXT | grep libcrypto | cut -d' ' -f11)
if [ -n "$LIBCRYPTO_PATH" ]; then
install_name_tool -change $LIBCRYPTO_PATH @rpath/libcrypto.3.0.$LIB_EXT libHSsqlcphr-*.$LIB_EXT
LIBCRYPTO_PATH=$(otool -l libHSsqlcphr-*.$LIB_EXT | grep libcrypto | cut -d' ' -f11)
if [ -n "$LIBCRYPTO_PATH" ]; then
install_name_tool -change $LIBCRYPTO_PATH @rpath/libcrypto.3.0.$LIB_EXT libHSsqlcphr-*.$LIB_EXT
fi
fi
for lib in $(find . -type f -name "*.$LIB_EXT"); do
@@ -132,7 +144,9 @@ LOCAL_DIRS=`for lib in $(find . -type f -name "*.$LIB_EXT"); do otool -l $lib |
if [ -n "$LOCAL_DIRS" ]; then
echo These libs still point to local directories:
echo $LOCAL_DIRS
exit 1
if [[ "$DATABASE_BACKEND" == "sqlite" ]]; then
exit 1
fi
fi
cd -
+1 -1
View File
@@ -53,7 +53,7 @@ echo " ghc-options: -shared -threaded -optl-L$openssl_windows_style_path -opt
# Very important! Without it the build fails on linking step since the linker can't find exported symbols.
# It looks like GHC bug because with such random path the build ends successfully
sed -i "s/ld.lld.exe/abracadabra.exe/" `ghc --print-libdir`/settings
cabal build lib:simplex-chat --constraint 'simplexmq +client_library'
cabal build lib:simplex-chat --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library'
rm -rf apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
rm -rf apps/multiplatform/desktop/build/cmake
+24 -7
View File
@@ -2,6 +2,9 @@
set -e
ARCH="$(uname -m)"
function readlink() {
echo "$(cd "$(dirname "$1")"; pwd -P)"
}
@@ -36,14 +39,28 @@ sed -i 's|Icon=.*|Icon=simplex|g' *imple*.desktop
cp *imple*.desktop usr/share/applications/
cp $multiplatform_dir/desktop/src/jvmMain/resources/distribute/*.appdata.xml usr/share/metainfo
if [ ! -f ../appimagetool-x86_64.AppImage ]; then
wget --secure-protocol=TLSv1_3 https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-x86_64.AppImage -O ../appimagetool-x86_64.AppImage
chmod +x ../appimagetool-x86_64.AppImage
if [ ! -f ../appimagetool-${ARCH}.AppImage ]; then
wget --secure-protocol=TLSv1_3 https://github.com/simplex-chat/appimagetool/releases/download/continuous/appimagetool-${ARCH}.AppImage -O ../appimagetool-${ARCH}.AppImage
chmod +x ../appimagetool-${ARCH}.AppImage
fi
if [ ! -f ../runtime-x86_64 ]; then
wget --secure-protocol=TLSv1_3 https://github.com/AppImage/type2-runtime/releases/download/continuous/runtime-x86_64 -O ../runtime-x86_64
chmod +x ../runtime-x86_64
if [ ! -f ../runtime-${ARCH} ]; then
wget --secure-protocol=TLSv1_3 https://github.com/simplex-chat/type2-runtime/releases/download/continuous/runtime-${ARCH} -O ../runtime-${ARCH}
chmod +x ../runtime-${ARCH}
fi
../appimagetool-x86_64.AppImage --runtime-file ../runtime-x86_64 .
# Determenistic build
export SOURCE_DATE_EPOCH=1704067200
# Delete redundant jar file and modify cfg
rm -f ./usr/lib/app/*skiko-awt-runtime-linux*
sed -i -e '/skiko-awt-runtime-linux/d' ./usr/lib/app/simplex.cfg
# Set all files to fixed time
find . -exec touch -d "@$SOURCE_DATE_EPOCH" {} +
../appimagetool-${ARCH}.AppImage --verbose --no-appstream --runtime-file ../runtime-${ARCH} .
mv *imple*.AppImage ../../
# Just a safeguard
strip-nondeterminism ../../*imple*.AppImage
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env sh
ARCH="$(uname -m)"
scripts/desktop/build-lib-linux.sh
cd apps/multiplatform
./gradlew packageDeb
# Workaround for skiko library
#
# Compose Multiplatform depends on skiko library, that
# handles all of the window managment and graphics drawing.
#
# This skiko library comes with two jar's:
# - platform-agnostic "skiko-awt"
# - and platform-specific "skiko-awt-runtime"
#
# In case of Linux, second jar is called "skiko-awt-runtime-linux-x64".
# Essentially, this jar has the Linux .so library called "libskiko-linux-x64.so"
# that is being unpacked to runtime libs.
#
# Since the jar is nothing more than a zip archive, extracting library
# from "skiko-awt-runtime-linux-x64" modifies it's timestamps
# with current time, which in changes it's hash, which in turn
# makes the whole build unreproducible.
#
# It seems to be there is no way to handle this extraction in our code and
# https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives
# unfortunately doesn't solve the issue.
#
# Instead, just modify the deb, removing the redundant skiko library.
#
# Also, it seems this is related to:
# https://youtrack.jetbrains.com/issue/CMP-1971/createDistributable-produces-duplicated-skiko-awt.jar-and-skiko-awt-runtime-windows-x64.jar
export SOURCE_DATE_EPOCH=1704067200
dpkg-deb -R ./release/main/deb/simplex*.deb ./extracted
# Source the distribution variables (VERSION_CODENAME)
. /etc/os-release
rm -f ./extracted/opt/*imple*/lib/app/*skiko-awt-runtime-linux*
sed -i -e '/skiko-awt-runtime-linux/d' ./extracted/opt/*imple*/lib/app/simplex.cfg
sed -i "/Version/ s/\$/~$VERSION_CODENAME/" ./extracted/DEBIAN/control
find ./extracted/ -exec touch -d "@$SOURCE_DATE_EPOCH" {} +
dpkg-deb --build --root-owner-group --uniform-compression ./extracted ./release/main/deb/simplex_${ARCH}.deb
strip-nondeterminism ./release/main/deb/simplex_${ARCH}.deb
+6 -2
View File
@@ -2,19 +2,23 @@
set -e
ARCH="$(uname -m)"
function readlink() {
echo "$(cd "$(dirname "$1")"; pwd -P)"
}
root_dir="$(dirname "$(dirname "$(readlink "$0")")")"
vlc_dir=$root_dir/apps/multiplatform/common/src/commonMain/cpp/desktop/libs/linux-x86_64/vlc
vlc_dir=$root_dir/apps/multiplatform/common/src/commonMain/cpp/desktop/libs/linux-${ARCH}/vlc
mkdir $vlc_dir || exit 0
vlc_tag='v3.0.21-1'
vlc_url="https://github.com/simplex-chat/vlc/releases/download/${vlc_tag}/vlc-linux-${ARCH}.appimage"
cd /tmp
mkdir tmp 2>/dev/null || true
cd tmp
curl --tlsv1.2 https://github.com/cmatomic/VLCplayer-AppImage/releases/download/3.0.11.1/VLC_media_player-3.0.11.1-x86_64.AppImage -L -o appimage
curl --tlsv1.2 "${vlc_url}" -L -o appimage
chmod +x appimage
./appimage --appimage-extract
cp -r squashfs-root/usr/lib/* $vlc_dir
+4 -2
View File
@@ -9,7 +9,9 @@ if [ "$ARCH" == "arm64" ]; then
else
vlc_arch=intel64
fi
vlc_version=3.0.19
vlc_tag='v3.0.21-1'
vlc_url="https://github.com/simplex-chat/vlc/releases/download/${vlc_tag}/vlc-macos-${ARCH}.zip"
function readlink() {
echo "$(cd "$(dirname "$1")"; pwd -P)"
@@ -23,7 +25,7 @@ mkdir -p $vlc_dir/vlc || exit 0
cd /tmp
mkdir tmp 2>/dev/null || true
cd tmp
curl --tlsv1.2 https://github.com/simplex-chat/vlc/releases/download/v$vlc_version/vlc-macos-$ARCH.zip -L -o vlc
curl --tlsv1.2 "${vlc_url}" -L -o vlc
unzip -oqq vlc
install_name_tool -add_rpath "@loader_path/VLC.app/Contents/MacOS/lib" vlc-cache-gen
cd VLC.app/Contents/MacOS/lib
+4 -1
View File
@@ -10,10 +10,13 @@ vlc_dir=$root_dir/apps/multiplatform/common/src/commonMain/cpp/desktop/libs/wind
rm -rf $vlc_dir
mkdir -p $vlc_dir/vlc || exit 0
vlc_tag='v3.0.21-1'
vlc_url="https://github.com/simplex-chat/vlc/releases/download/${vlc_tag}/vlc-win-x86_64.zip"
cd /tmp
mkdir tmp 2>/dev/null || true
cd tmp
curl --tlsv1.2 https://irltoolkit.mm.fcix.net/videolan-ftp/vlc/3.0.18/win64/vlc-3.0.18-win64.zip -L -o vlc
curl --tlsv1.2 "${vlc_url}" -L -o vlc
$WINDIR\\System32\\tar.exe -xf vlc
cd vlc-*
# Setting the same date as the date that will be on the file after extraction from JAR to make VLC cache checker happy
@@ -38,6 +38,379 @@
</description>
<releases>
<release version="6.4.8" date="2025-12-11">
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
<description>
<p>New in v6.4.8:</p>
<ul>
<li>fix stuck message reception and other events after passphrase change (e.g., during desktop app initial start)</li>
</ul>
<p>New in v6.4-6.4.7:</p>
<ul>
<li>new UX to connect.</li>
<li>review new group members.</li>
<li>chat with group admins.</li>
<li>new UI languages: Catalan, Indonesian, Romanian and Vietnamese.</li>
<li>Linux app builds for aarch64 CPUs</li>
<li>UI support for bot commands.</li>
<li>support markdown hyperlinks, such as [click here](https://example.com).</li>
<li>option to remove tracking parameters from the links.</li>
<li>better information about network errors.</li>
</ul>
</description>
</release>
<release version="6.4.7" date="2025-11-03">
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
<description>
<p>New in v6.4.7:</p>
<ul>
<li>fix exporting database larger than 4gb.</li>
</ul>
<p>New in v6.4-6.4.6:</p>
<ul>
<li>new UX to connect.</li>
<li>review new group members.</li>
<li>chat with group admins.</li>
<li>new UI languages: Catalan, Indonesian, Romanian and Vietnamese.</li>
<li>Linux app builds for aarch64 CPUs</li>
<li>UI support for bot commands.</li>
<li>support markdown hyperlinks, such as [click here](https://example.com).</li>
<li>option to remove tracking parameters from the links.</li>
<li>better information about network errors.</li>
</ul>
</description>
</release>
<release version="6.4.6" date="2025-10-05">
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
<description>
<p>New in v6.4.6:</p>
<ul>
<li>fixed opening SimpleX links from outside of the app.</li>
</ul>
<p>New in v6.4-6.4.5:</p>
<ul>
<li>new UX to connect.</li>
<li>review new group members.</li>
<li>chat with group admins.</li>
<li>new UI languages: Catalan, Indonesian, Romanian and Vietnamese.</li>
<li>Linux app builds for aarch64 CPUs</li>
<li>UI support for bot commands.</li>
<li>support markdown hyperlinks, such as [click here](https://example.com).</li>
<li>option to remove tracking parameters from the links.</li>
<li>better information about network errors.</li>
</ul>
</description>
</release>
<release version="6.4.5" date="2025-09-08">
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
<description>
<p>New in v6.4.5:</p>
<ul>
<li>fixes for chats with admins.</li>
<li>better information about network errors.</li>
</ul>
<p>New in v6.4-6.4.4:</p>
<ul>
<li>new UX to connect.</li>
<li>review new group members.</li>
<li>chat with group admins.</li>
<li>new UI languages: Catalan, Indonesian, Romanian and Vietnamese.</li>
<li>Linux app builds for aarch64 CPUs</li>
<li>UI support for bot commands.</li>
<li>support markdown hyperlinks, such as [click here](https://example.com).</li>
<li>option to remove tracking parameters from the links.</li>
</ul>
</description>
</release>
<release version="6.4.4" date="2025-08-28">
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
<description>
<p>New in v6.4.4:</p>
<ul>
<li>reduced battery usage.</li>
<li>fixes.</li>
</ul>
<p>New in v6.4-6.4.3.1:</p>
<ul>
<li>new UX to connect.</li>
<li>review new group members.</li>
<li>chat with group admins.</li>
<li>new UI languages: Catalan, Indonesian, Romanian and Vietnamese.</li>
<li>Linux app builds for aarch64 CPUs</li>
<li>UI support for bot commands.</li>
<li>support markdown hyperlinks, such as [click here](https://example.com).</li>
<li>option to remove tracking parameters from the links.</li>
</ul>
</description>
</release>
<release version="6.4.3.1" date="2025-08-10">
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
<description>
<p>New in v6.4.3.1:</p>
<ul>
<li>UI support for bot commands.</li>
<li>support markdown hyperlinks, such as [click here](https://example.com).</li>
<li>option to remove tracking parameters from the links.</li>
</ul>
<p>New in v6.4-6.4.2:</p>
<ul>
<li>new UX to connect.</li>
<li>review new group members.</li>
<li>chat with group admins.</li>
<li>new UI languages: Catalan, Indonesian, Romanian and Vietnamese.</li>
<li>Linux app builds for aarch64 CPUs</li>
</ul>
</description>
</release>
<release version="6.4.2" date="2025-08-02">
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
<description>
<p>New in v6.4.2:</p>
<ul>
<li>manually accept member contact requests, with option to auto-accept.</li>
<li>ignore contact requests from blocked group members.</li>
<li>markdown (links etc.) in profile bio/group purpose.</li>
<li>Linux app builds for aarch64 CPUs</li>
</ul>
</description>
</release>
<release version="6.4.1" date="2025-07-29">
<url type="details">https://simplex.chat/blog/20250729-simplex-chat-v6-4-1-welcome-contacts-protect-groups-app-security.html</url>
<description>
<p>New in v6.4.1:</p>
<ul>
<li>welcome your contacts: set profile bio and welcome message.</li>
<li>enable disappearing messages by default for new contacts.</li>
<li>short SimpleX addresses and group links now include profile images and welcome messages.</li>
</ul>
</description>
</release>
<release version="6.4.0" date="2025-07-16">
<url type="details">https://simplex.chat/blog/20250703-simplex-network-protocol-extension-for-securely-connecting-people.html</url>
<description>
<p>New in v6.4.0:</p>
<ul>
<li>Connect faster: message instantly once you tap Connect.</li>
<li>Review group members: chat with new members before they join.</li>
<li>Chat with admins: send your private feedback to group owners.</li>
<li>New group role: Moderator - can remove messages and block members.</li>
<li>Improved message delivery - less traffic on mobile networks.</li>
</ul>
</description>
</release>
<release version="6.3.7" date="2025-07-01">
<url type="details">https://simplex.chat/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.html</url>
<description>
<p>New in v6.3.1-7:</p>
<ul>
<li>fix changing the user profile of the invitation link.</li>
<li>forward compatibility with short link data.</li>
<li>fixes</li>
<li>fixes mentions with trailing punctuation (e.g., hello @name!).</li>
<li>recognizes domain names as links (e.g., simplex.chat).</li>
<li>forward compatibility with "knocking" (a feature for group admins to review and to chat with the new members prior to admitting them to groups, it will be released in 6.4)</li>
<li>support for connecting via short connection links.</li>
<li>fix related to backward/forward compatibility of the app in some rare cases.</li>
<li>scrolling/navigation improvements.</li>
<li>faster onboarding (conditions and operators are combined to one screen).</li>
</ul>
<p>New in v6.3.0:</p>
<ul>
<li>Mention members and get notified when mentioned.</li>
<li>Send private reports to moderators.</li>
<li>Delete, block and change role for multiple members at once</li>
<li>Faster sending messages and faster deletion.</li>
<li>Organize chats into lists to keep track of what's important.</li>
<li>Jump to found and forwarded messages.</li>
<li>Private media file names.</li>
<li>Message expiration in chats.</li>
</ul>
</description>
</release>
<release version="6.3.6" date="2025-06-14">
<url type="details">https://simplex.chat/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.html</url>
<description>
<p>New in v6.3.1-6:</p>
<ul>
<li>forward compatibility with short link data.</li>
<li>fixes</li>
<li>fixes mentions with trailing punctuation (e.g., hello @name!).</li>
<li>recognizes domain names as links (e.g., simplex.chat).</li>
<li>forward compatibility with "knocking" (a feature for group admins to review and to chat with the new members prior to admitting them to groups, it will be released in 6.4)</li>
<li>support for connecting via short connection links.</li>
<li>fix related to backward/forward compatibility of the app in some rare cases.</li>
<li>scrolling/navigation improvements.</li>
<li>faster onboarding (conditions and operators are combined to one screen).</li>
</ul>
<p>New in v6.3.0:</p>
<ul>
<li>Mention members and get notified when mentioned.</li>
<li>Send private reports to moderators.</li>
<li>Delete, block and change role for multiple members at once</li>
<li>Faster sending messages and faster deletion.</li>
<li>Organize chats into lists to keep track of what's important.</li>
<li>Jump to found and forwarded messages.</li>
<li>Private media file names.</li>
<li>Message expiration in chats.</li>
</ul>
</description>
</release>
<release version="6.3.5" date="2025-06-09">
<url type="details">https://simplex.chat/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.html</url>
<description>
<p>New in v6.3.1-5:</p>
<ul>
<li>fixes</li>
<li>fixes mentions with trailing punctuation (e.g., hello @name!).</li>
<li>recognizes domain names as links (e.g., simplex.chat).</li>
<li>forward compatibility with "knocking" (a feature for group admins to review and to chat with the new members prior to admitting them to groups, it will be released in 6.4)</li>
<li>support for connecting via short connection links.</li>
<li>fix related to backward/forward compatibility of the app in some rare cases.</li>
<li>scrolling/navigation improvements.</li>
<li>faster onboarding (conditions and operators are combined to one screen).</li>
</ul>
<p>New in v6.3.0:</p>
<ul>
<li>Mention members and get notified when mentioned.</li>
<li>Send private reports to moderators.</li>
<li>Delete, block and change role for multiple members at once</li>
<li>Faster sending messages and faster deletion.</li>
<li>Organize chats into lists to keep track of what's important.</li>
<li>Jump to found and forwarded messages.</li>
<li>Private media file names.</li>
<li>Message expiration in chats.</li>
</ul>
</description>
</release>
<release version="6.3.4" date="2025-05-12">
<url type="details">https://simplex.chat/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.html</url>
<description>
<p>New in v6.3.1-4:</p>
<ul>
<li>fixes mentions with trailing punctuation (e.g., hello @name!).</li>
<li>recognizes domain names as links (e.g., simplex.chat).</li>
<li>forward compatibility with "knocking" (a feature for group admins to review and to chat with the new members prior to admitting them to groups, it will be released in 6.4)</li>
<li>support for connecting via short connection links.</li>
<li>fix related to backward/forward compatibility of the app in some rare cases.</li>
<li>scrolling/navigation improvements.</li>
<li>faster onboarding (conditions and operators are combined to one screen).</li>
</ul>
<p>New in v6.3.0:</p>
<ul>
<li>Mention members and get notified when mentioned.</li>
<li>Send private reports to moderators.</li>
<li>Delete, block and change role for multiple members at once</li>
<li>Faster sending messages and faster deletion.</li>
<li>Organize chats into lists to keep track of what's important.</li>
<li>Jump to found and forwarded messages.</li>
<li>Private media file names.</li>
<li>Message expiration in chats.</li>
</ul>
</description>
</release>
<release version="6.3.3" date="2025-04-24">
<url type="details">https://simplex.chat/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.html</url>
<description>
<p>New in v6.3.1-3:</p>
<ul>
<li>support for connecting via short connection links.</li>
<li>fix related to backward/forward compatibility of the app in some rare cases.</li>
<li>scrolling/navigation improvements.</li>
<li>faster onboarding (conditions and operators are combined to one screen).</li>
</ul>
<p>New in v6.3.0:</p>
<ul>
<li>Mention members and get notified when mentioned.</li>
<li>Send private reports to moderators.</li>
<li>Delete, block and change role for multiple members at once</li>
<li>Faster sending messages and faster deletion.</li>
<li>Organize chats into lists to keep track of what's important.</li>
<li>Jump to found and forwarded messages.</li>
<li>Private media file names.</li>
<li>Message expiration in chats.</li>
</ul>
</description>
</release>
<release version="6.3.2" date="2025-04-13">
<url type="details">https://simplex.chat/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.html</url>
<description>
<p>New in v6.3.1-2:</p>
<ul>
<li>fix related to backward/forward compatibility of the app in some rare cases.</li>
<li>scrolling/navigation improvements.</li>
<li>faster onboarding (conditions and operators are combined to one screen).</li>
</ul>
<p>New in v6.3.0:</p>
<ul>
<li>Mention members and get notified when mentioned.</li>
<li>Send private reports to moderators.</li>
<li>Delete, block and change role for multiple members at once</li>
<li>Faster sending messages and faster deletion.</li>
<li>Organize chats into lists to keep track of what's important.</li>
<li>Jump to found and forwarded messages.</li>
<li>Private media file names.</li>
<li>Message expiration in chats.</li>
</ul>
</description>
</release>
<release version="6.3.1" date="2025-03-31">
<url type="details">https://simplex.chat/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.html</url>
<description>
<p>New in v6.3.1:</p>
<ul>
<li>scrolling/navigation improvements.</li>
<li>faster onboarding (conditions and operators are combined to one screen).</li>
</ul>
<p>New in v6.3.0:</p>
<ul>
<li>Mention members and get notified when mentioned.</li>
<li>Send private reports to moderators.</li>
<li>Delete, block and change role for multiple members at once</li>
<li>Faster sending messages and faster deletion.</li>
<li>Organize chats into lists to keep track of what's important.</li>
<li>Jump to found and forwarded messages.</li>
<li>Private media file names.</li>
<li>Message expiration in chats.</li>
</ul>
</description>
</release>
<release version="6.3.0" date="2025-03-08">
<url type="details">https://simplex.chat/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.html</url>
<description>
<p>New in v6.3.0:</p>
<ul>
<li>Mention members and get notified when mentioned.</li>
<li>Send private reports to moderators.</li>
<li>Delete, block and change role for multiple members at once</li>
<li>Faster sending messages and faster deletion.</li>
<li>Organize chats into lists to keep track of what's important.</li>
<li>Jump to found and forwarded messages.</li>
<li>Private media file names.</li>
<li>Message expiration in chats.</li>
</ul>
</description>
</release>
<release version="6.2.5" date="2025-02-16">
<url type="details">https://simplex.chat/blog/20241210-simplex-network-v6-2-servers-by-flux-business-chats.html</url>
<description>
<p>New in v6.2.1-5:</p>
<ul>
<li>change media filenames when forwarding.</li>
<li>fully delete wallpapers when deleting user or chat.</li>
<li>important fixes</li>
<li>offer to "fix" encryption when calling or making direct connection with member.</li>
<li>broken layout.</li>
<li>option to enable debug logs (disabled by default).</li>
<li>show who reacted in direct chats.</li>
</ul>
<p>New in v6.2:</p>
<ul>
<li>SimpleX Chat and Flux made an agreement to include servers operated by Flux into the app to improve metadata privacy.</li>
<li>Business chats your customers privacy.</li>
<li>Improved user experience in chats: open on the first unread, jump to quoted messages, see who reacted.</li>
</ul>
</description>
</release>
<release version="6.2.4" date="2025-01-14">
<url type="details">https://simplex.chat/blog/20241210-simplex-network-v6-2-servers-by-flux-business-chats.html</url>
<description>
+2 -2
View File
@@ -1,12 +1,12 @@
{
"https://github.com/simplex-chat/simplexmq.git"."23189753751dc52046865ce2d992335495020e91" = "0f1c0bfjqwycsb2nkphhbdiv77zx6q47jdigk7bjal1c4rfla8gy";
"https://github.com/simplex-chat/simplexmq.git"."2ca440dd2dfd494ff2bb40cc0409d08069d02e04" = "1jc1a9vh59l0l5hxlin1spv03afrgmmiml5xnakhbi4rk67n0wwr";
"https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
"https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "1ql13f4kfwkbaq7nygkxgw84213i0zm7c1a8hwvramayxl38dq5d";
"https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl";
"https://github.com/simplex-chat/aeson.git"."aab7b5a14d6c5ea64c64dcaee418de1bb00dcc2b" = "0jz7kda8gai893vyvj96fy962ncv8dcsx71fbddyy8zrvc88jfrr";
"https://github.com/simplex-chat/haskell-terminal.git"."f708b00009b54890172068f168bf98508ffcd495" = "0zmq7lmfsk8m340g47g5963yba7i88n4afa6z93sg9px5jv1mijj";
"https://github.com/simplex-chat/android-support.git"."9aa09f148089d6752ce563b14c2df1895718d806" = "0pbf2pf13v2kjzi397nr13f1h3jv0imvsq8rpiyy2qyx5vd50pqn";
"https://github.com/simplex-chat/zip.git"."bd421c6b19cc4c465cd7af1f6f26169fb8ee1ebc" = "1csqfjhvc8wb5h4kxxndmb6iw7b4ib9ff2n81hrizsmnf45a6gg0";
"https://github.com/simplex-chat/zip.git"."2eff156c3aac389e35d38bf10a52733d7061640a" = "052vahd5d4lxnazjrb6l60i261aycn2js7jhzafyb72n15ns4r6p";
"https://github.com/yesodweb/wai.git"."ec5e017d896a78e787a5acea62b37a4e677dec2e" = "1ckcpmpjfy9jiqrb52q20lj7ln4hmq9v2jk6kpkf3m68c1m9c2bx";
"https://github.com/simplex-chat/wai.git"."2f6e5aa5f05ba9140ac99e195ee647b4f7d926b0" = "199g4rjdf1zp1fcw8nqdsyr1h36hmg424qqx03071jk7j00z7ay4";
"https://gitlab.com/dpwiz/hs-jpeg-turbo"."f1bc073427e60341a3359e2345c1683d17ea2a35" = "09hj5y3077k2cr1djw24i2gn9bdyp24q6qvi255l92ibk3pjx1nm";
+253
View File
@@ -0,0 +1,253 @@
#!/usr/bin/env sh
set -eu
SIMPLEX_KEY='3C:52:C4:FD:3C:AD:1C:07:C9:B0:0A:70:80:E3:58:FA:B9:FE:FC:B8:AF:5A:EC:14:77:65:F1:6D:0F:21:AD:85'
REPO_NAME="simplex-chat"
REPO="https://github.com/simplex-chat/${REPO_NAME}"
IMAGE_NAME='sx-local-android'
CONTAINER_NAME='sx-builder-android'
DOCKER_PATH_PROJECT='/project'
DOCKER_PATH_VERIFY='/verify'
export DOCKER_BUILDKIT=1
SIMPLEX_REPO='simplex-chat/simplex-chat'
CMDS="curl git docker"
INIT_DIR="$PWD"
TEMPDIR="$(mktemp -d)"
ARCHES="${ARCHES:-aarch64 armv7a}"
COLOR_CYAN="\033[36m"
COLOR_RESET="\033[0m"
SUFFIX_BUILT='built'
SUFFIX_DOWNLOADED='downloaded'
SUFFIX_BUILT_WITH_SIGNATURE='built-with-downloaded-signature'
cleanup() {
rm -rf -- "${TEMPDIR}"
docker rm --force "${CONTAINER_NAME}" 2>/dev/null || :
docker image rm "${IMAGE_NAME}" 2>/dev/null || :
}
trap 'cleanup' EXIT INT
check() {
set +u
for i in $commands; do
case $i in
*)
if ! command -v "$i" > /dev/null 2>&1; then
commands_failed="$i $commands_failed"
fi
;;
esac
done
if [ -n "$commands_failed" ]; then
commands_failed=${commands_failed% *}
printf "%s is not found in your \$PATH. Please install them and re-run the script.\n" "$commands_failed"
exit 1
fi
set -u
}
download_apk() {
tag="$1"
filename="$2"
file_out="$3"
curl -L "${REPO}/releases/download/${tag}/${filename}" -o "$file_out"
}
setup_git() {
workdir="$1"
name="$2"
git -C "$workdir" clone "${REPO}.git" "$name"
}
checkout_git() {
git_dir="$1"
tag="$2"
git -C "$git_dir" reset --hard
git -C "$git_dir" clean -dfx
git -C "$git_dir" checkout "$tag"
}
check_apk() {
apk_name="$1"
expected="$2"
actual=$(docker exec "${CONTAINER_NAME}" apksigner verify --print-certs "${DOCKER_PATH_VERIFY}/${apk_name}" | grep 'SHA-256' | awk '{print $NF}' | fold -w2 | paste -sd: | tr '[:lower:]' '[:upper:]')
if [ "$expected" = "$actual" ]; then
return 0
else
return 1
fi
}
verify_apk() {
apk_name="$1"
# https://github.com/obfusk/apksigcopier?tab=readme-ov-file#what-about-signatures-made-by-apksigner-from-build-tools--3500-rc1
docker exec "${CONTAINER_NAME}" repro-apk zipalign --page-size 16 --pad-like-apksigner --replace "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT}" \
"${DOCKER_PATH_VERIFY}/${apk_name}.aligned"
docker exec "${CONTAINER_NAME}" mv "${DOCKER_PATH_VERIFY}/${apk_name}.aligned" \
"${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT}"
docker exec "${CONTAINER_NAME}" apksigcopier copy "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_DOWNLOADED}" \
"${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT}" \
"${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT_WITH_SIGNATURE}"
downloaded_apk_hash=$(docker exec "${CONTAINER_NAME}" sha256sum "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_DOWNLOADED}" | awk '{print $1}')
built_apk_hash=$(docker exec "${CONTAINER_NAME}" sha256sum "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT_WITH_SIGNATURE}" | awk '{print $1}')
if [ "$downloaded_apk_hash" = "$built_apk_hash" ]; then
return 0
else
return 1
fi
}
print_vercode() {
build_dir="$1"
awk -F'=' '/android.version_code=/ {print $2}' "${build_dir}/apps/multiplatform/gradle.properties"
}
setup_container() {
dir_git="$1"
dir_apk="$2"
docker build \
--no-cache \
-f "${dir_git}/Dockerfile.build" \
-t "${IMAGE_NAME}" \
--build-arg=USER_UID="$(id -u)" \
--build-arg=USER_GID="$(id -g)" \
.
# Run container in background
docker run -t -d \
--name "${CONTAINER_NAME}" \
--device /dev/fuse \
--cap-add SYS_ADMIN \
--security-opt apparmor:unconfined \
--security-opt seccomp:unconfined \
-v "${dir_git}:${DOCKER_PATH_PROJECT}" \
-v "${dir_apk}:${DOCKER_PATH_VERIFY}" \
"${IMAGE_NAME}"
}
build_apk() {
arch="$1"
vercode="$2"
apk_out="simplex-${arch}.apk.${SUFFIX_BUILT}"
# Gradle setup
docker exec -i "${CONTAINER_NAME}" sh << EOF
cd $DOCKER_PATH_PROJECT/apps/multiplatform
./gradlew
EOF
docker exec -i "${CONTAINER_NAME}" sh << EOF
GRADLE_BIN=\$(find \$HOME/.gradle/wrapper/dists -name "gradle" -type f -executable 2>/dev/null | head -1)
GRADLE_DIR=\$(dirname "\$GRADLE_BIN")
export PATH="\$GRADLE_DIR:\$PATH"
ARCHES="$arch" ./scripts/android/build-android.sh -gs "$vercode" || ARCHES="$arch" ./scripts/android/build-android.sh -gs "$vercode"
APK_FILE=\$(find . -maxdepth 1 -type f -name '*.apk')
mv "\$APK_FILE" $DOCKER_PATH_VERIFY/$apk_out
EOF
}
main() {
tag="$1"
build_directory="${TEMPDIR}/${REPO_NAME}"
final_directory="$INIT_DIR/${tag}-${REPO_NAME}"
apk_directory="${final_directory}/android"
printf 'This script will:
1) build docker container.
2) download APK from GitHub and validate signatures.
3) build core library with nix (12-24 hours).
4) build APK and compare with downloaded one
Continue?'
read _
check
mkdir -p "${apk_directory}"
# Setup initial git for Dockerfile.build
setup_git "$TEMPDIR" "$REPO_NAME"
checkout_git "$build_directory" "$tag"
printf "${COLOR_CYAN}Building Docker container...${COLOR_RESET}\n"
setup_container "$build_directory" "$apk_directory"
# Check phase
for arch in $ARCHES; do
filename="simplex-${arch}.apk"
download_apk "$tag" "$filename" "${apk_directory}/${filename}.${SUFFIX_DOWNLOADED}"
if check_apk "${filename}.${SUFFIX_DOWNLOADED}" "$SIMPLEX_KEY"; then
printf "${COLOR_CYAN}APK for %s is signed by valid key.${COLOR_RESET}\n" "$arch"
else
printf "${COLOR_CYAN}Signature of APK for %s is invalid., aborting the script.${COLOR_RESET}\n" "$arch"
exit 1
fi
done
# Build phase
for arch in $ARCHES; do
case "$arch" in
armv7a)
build_tag="${tag}-armv7a"
;;
aarch64)
build_tag="${tag}"
;;
*)
printf "${COLOR_CYAN}Unknown architecture: %s! Skipping the build...${COLOR_RESET}\n" "$arch"
continue
esac
# Setup the code
checkout_git "$build_directory" "$build_tag"
vercode=$(print_vercode "$build_directory")
printf "${COLOR_CYAN}Building APK for for %s...${COLOR_RESET}\n" "$arch"
build_apk "$arch" "$vercode"
done
# Verification phase
for arch in $ARCHES; do
filename="simplex-${arch}.apk"
if ! verify_apk "$filename"; then
printf "${COLOR_CYAN}Failed to verify %s! Aborting.\n${COLOR_RESET}" "$filename"
exit 1
fi
done
printf "${COLOR_CYAN}%s is reproducible.${COLOR_RESET}\n" "$tag"
cleanup
}
main "$@"
+191
View File
@@ -0,0 +1,191 @@
#!/usr/bin/env sh
set -eu
TAG="$1"
tempdir="$(mktemp -d)"
init_dir="$PWD"
ghc='9.6.3'
repo_name="simplex-chat"
repo="https://github.com/simplex-chat/${repo_name}"
image_name='sx-local'
container_name='sx-builder'
cabal_local='ignore-project: False
package direct-sqlcipher
flags: +openssl'
export DOCKER_BUILDKIT=1
version=${TAG#v}
version=${version%-*}
cleanup() {
docker exec -t "${container_name}" sh -c 'rm -rf ./dist-newstyle ./apps' 2>/dev/null || :
rm -rf -- "${tempdir}"
docker rm --force "${container_name}" 2>/dev/null || :
docker image rm "${image_name}" 2>/dev/null || :
cd "${init_dir}"
}
trap 'cleanup' EXIT INT
mkdir -p "${init_dir}/${TAG}-${repo_name}/from-source" "${init_dir}/${TAG}-${repo_name}/prebuilt"
git -C "${tempdir}" clone "${repo}.git" &&\
cd "${tempdir}/${repo_name}" &&\
git checkout "${TAG}"
for os in '22.04' '24.04'; do
os_url="$(printf '%s' "${os}" | tr '.' '_')"
cli_name="simplex-chat-ubuntu-${os_url}-x86_64"
deb_name="simplex-desktop-ubuntu-${os_url}-x86_64.deb"
appimage_name="simplex-desktop-x86_64.AppImage"
# Build image
docker build \
--no-cache \
--build-arg TAG="${os}" \
--build-arg GHC="${ghc}" \
--build-arg=USER_UID="$(id -u)" \
--build-arg=USER_GID="$(id -g)" \
-f "${tempdir}/${repo_name}/Dockerfile.build" \
-t "${image_name}" \
.
printf '%s' "${cabal_local}" > "${tempdir}/${repo_name}/cabal.project.local"
# Run container in background
docker run -t -d \
--name "${container_name}" \
--device /dev/fuse \
--cap-add SYS_ADMIN \
--security-opt apparmor:unconfined \
-v "${tempdir}/${repo_name}:/project" \
"${image_name}"
# Consistent permissions
docker exec \
-t "${container_name}" \
sh -c 'find /project -type d -exec chmod 755 {} \; ; find /project -type f -perm /111 -exec chmod 755 {} \; ; find /project -type f ! -perm /111 -exec chmod 644 {} \;'
# CLI
docker exec \
-t "${container_name}" \
sh -c 'cabal clean && cabal update && cabal build -j && mkdir -p /out && for i in simplex-chat; do bin=$(find /project/dist-newstyle -name "$i" -type f -executable) && chmod +x "$bin" && mv "$bin" /out/; done && strip /out/simplex-chat'
# Copy CLI
docker cp \
"${container_name}":/out/simplex-chat \
"${init_dir}/${TAG}-${repo_name}/from-source/${cli_name}"
# Download prebuilt CLI binary
curl -L \
--output-dir "${init_dir}/${TAG}-${repo_name}/prebuilt/" \
-O "${repo}/releases/download/${TAG}/${cli_name}"
# CLI: deb
docker exec \
-t "${container_name}" \
sh -c "./scripts/desktop/build-cli-deb.sh ${version}"
# Copy CLI: deb
docker cp \
"${container_name}":/out/deb-build/simplex-chat.deb \
"${init_dir}/${TAG}-${repo_name}/from-source/${cli_name}.deb"
# Download prebuilt CLI: deb binary
curl -L \
--output-dir "${init_dir}/${TAG}-${repo_name}/prebuilt/" \
-O "${repo}/releases/download/${TAG}/${cli_name}.deb"
# Desktop: deb
docker exec \
-t "${container_name}" \
sh -c './scripts/desktop/make-deb-linux.sh'
# Copy deb
docker cp \
"${container_name}":/project/apps/multiplatform/release/main/deb/simplex_x86_64.deb \
"${init_dir}/${TAG}-${repo_name}/from-source/${deb_name}"
# Download prebuilt deb package
curl -L \
--output-dir "${init_dir}/${TAG}-${repo_name}/prebuilt/" \
-O "${repo}/releases/download/${TAG}/${deb_name}"
# Desktop: appimage. Build only on 22.04
case "$os" in
22.04)
# Appimage
docker exec \
-t "${container_name}" \
sh -c './scripts/desktop/make-appimage-linux.sh && mv ./apps/multiplatform/release/main/*imple*.AppImage ./apps/multiplatform/release/main/simplex.appimage'
# Copy appimage
docker cp \
"${container_name}":/project/apps/multiplatform/release/main/simplex.appimage \
"${init_dir}/${TAG}-${repo_name}/from-source/${appimage_name}"
# Download prebuilt appimage binary
curl -L \
--output-dir "${init_dir}/${TAG}-${repo_name}/prebuilt/" \
-O "${repo}/releases/download/${TAG}/${appimage_name}"
;;
esac
# Important! Remove dist-newstyle for the next interation
docker exec \
-t "${container_name}" \
sh -c 'rm -rf ./dist-newstyle ./apps/multiplatform'
# Also restore git to previous state
git reset --hard && git clean -dfx
# Stop containers, delete images
docker stop "${container_name}"
docker rm --force "${container_name}"
docker image rm "${image_name}"
done
# Cleanup
rm -rf -- "${tempdir}"
cd "${init_dir}"
# Final stage: compare hashes
# Path to binaries
path_bin="${init_dir}/${TAG}-${repo_name}"
# Assume everything is okay for now
bad=0
# Check hashes for all binaries
for file in "${path_bin}"/from-source/*; do
# Extract binary name
app="$(basename ${file})"
# Compute hash for compiled binary
compiled=$(sha256sum "${path_bin}/from-source/${app}" | awk '{print $1}')
# Compute hash for prebuilt binary
prebuilt=$(sha256sum "${path_bin}/prebuilt/${app}" | awk '{print $1}')
# Compare
if [ "${compiled}" != "${prebuilt}" ]; then
# If hashes doesn't match, set bad...
bad=1
# ... and print affected binary
printf "%s - sha256sum hash doesn't match\n" "${app}"
fi
done
# If everything is still okay, compute checksums file
if [ "${bad}" = 0 ]; then
sha256sum "${path_bin}"/from-source/* | sed -e "s|$PWD/||g" -e 's|from-source/||g' -e "s|-$repo_name||g" > "${path_bin}/_sha256sums"
printf 'Checksums computed - %s\n' "${path_bin}/_sha256sums"
fi