dandri/simplex-chat
1
0
Fork 0
mirror of https://github.com/simplex-chat/simplex-chat.git synced 2026-09-27 09:19:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

core, libs: configurable queue size, library fixes (#7542)

Browse Source 
* core: add chat_migrate_init_queue FFI export

* bots: fix BadgeServiceErrorCode API type

* nodejs: pass required command fields

* nodejs: fix migration error types

* nodejs: install libsimplex from SIMPLEX_LIBS_DIR

* nodejs: add queue size option

* nodejs: regenerate docs

* python: add queue size option

* bots: pass incognito in APIConnect

* nodejs: accept documented success responses

* python: accept documented success responses

* nodejs: dispatch each bot message once

* nodejs: fix startChat events loop lifecycle

* nodejs, python: parse multi-line bot commands

* nodejs: fix file buffer handling in addon

* nodejs: keep events loop when chat stop fails

* python: fix send_and_wait race, load lib off loop

* python: make queue size export optional

* nodejs: receive events on a dedicated thread

* nodejs: release haskell thread after receive

* python: receive on a dedicated thread per chat

* python: test receive shutdown order

* nodejs: one receive thread per chat controller

* nodejs: harden receiver shutdown and tests

* nodejs: stop chat before closing store

* python: stop chat before closing store

* nodejs, python: harden close regression and retry

* python: free results with ucrt on windows

* nodejs: enable c++ exceptions on mac and windows
This commit is contained in:
sh
2026-09-19 12:18:46 +01:00
committed by GitHub
parent ecb9d87157
commit c8f20bcc91
78 changed files with 1670 additions and 571 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/simplex-chat-python/src/simplex_chat/_native.py
+19 -2
View File
@@ -16,7 +16,7 @@ import urllib.request
import zipfile
from ctypes import POINTER, c_char_p, c_int, c_uint8, c_void_p
from pathlib import Path
from typing import Literal
from typing import Any, Literal
from ._version import LIBS_VERSION
@@ -166,7 +166,8 @@ _backend: Backend | None = None
def _load_libc() -> ctypes.CDLL:
if sys.platform == "win32":
return ctypes.CDLL("msvcrt")
# libsimplex.dll allocates results with UCRT malloc; msvcrt free would corrupt the heap.
return ctypes.CDLL("ucrtbase")
return ctypes.CDLL(None) # libc on POSIX is the process's own symbol table
@@ -255,3 +256,19 @@ def lib() -> ctypes.CDLL:
if _lib is None:
raise RuntimeError("lib_for() must be called before lib()")
return _lib
QUEUE_SIZE_UNSUPPORTED = (
"loaded libsimplex does not export chat_migrate_init_queue; queue size needs a newer libsimplex"
)
def migrate_init_queue() -> Any:
"""`chat_migrate_init_queue`, which older libsimplex releases do not export."""
try:
fn = lib().chat_migrate_init_queue
except AttributeError as e:
raise RuntimeError(QUEUE_SIZE_UNSUPPORTED) from e
fn.argtypes = [c_char_p, c_char_p, c_char_p, c_int, POINTER(c_void_p)]
fn.restype = c_void_p
return fn
packages/simplex-chat-python/src/simplex_chat/api.py
+35 -8
View File
@@ -2,7 +2,9 @@
from __future__ import annotations
import asyncio
import json
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from typing import Any, Literal
@@ -65,17 +67,20 @@ class ChatApi:
def __init__(self, ctrl: int):
self._ctrl: int | None = ctrl
self._started = False
self._recv_executor: ThreadPoolExecutor | None = None
@classmethod
async def init(
cls,
db: Db,
confirm: MigrationConfirmation = MigrationConfirmation.YES_UP,
queue_size: int | None = None,
) -> ChatApi:
path_or_prefix, key_or_conn, backend = _db_to_migrate_args(db)
# Trigger lazy lib load with the right backend BEFORE chat_migrate_init.
_native.lib_for(backend)
ctrl = await core.chat_migrate_init(path_or_prefix, key_or_conn, confirm)
# It may download ~100 MB, so it must not block the event loop.
await asyncio.to_thread(_native.lib_for, backend)
ctrl = await core.chat_migrate_init(path_or_prefix, key_or_conn, confirm, queue_size)
return cls(ctrl)
@property
@@ -114,6 +119,14 @@ class ChatApi:
self._started = False
async def close(self) -> None:
"""Stop the chat and close its store; the store stays open if stopping fails."""
# a running controller keeps using the database connections that closing frees
await self.stop_chat()
if self._recv_executor is not None:
# Waits for a receive already in flight (up to wait_us) so the store
# never closes underneath one; run off-loop since shutdown blocks.
await asyncio.to_thread(self._recv_executor.shutdown, wait=True)
self._recv_executor = None
await core.chat_close_store(self.ctrl)
self._ctrl = None
self._started = False
@@ -122,7 +135,14 @@ class ChatApi:
return await core.chat_send_cmd(self.ctrl, cmd)
async def recv_chat_event(self, wait_us: int = 500_000) -> CEvt.ChatEvent | None:
return await core.chat_recv_msg_wait(self.ctrl, wait_us)
ctrl = self.ctrl # raises before touching the executor if close() was called
if self._recv_executor is None:
# A receive blocks for up to wait_us almost back to back, so it would
# otherwise pin one of the default executor's few worker threads.
self._recv_executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="simplex-recv"
)
return await core.chat_recv_msg_wait(ctrl, wait_us, self._recv_executor)
# ------------------------------------------------------------------ #
# Address commands
@@ -158,6 +178,8 @@ class ChatApi:
)
if r["type"] == "userProfileUpdated":
return r["updateSummary"]
if r["type"] == "userProfileNoChange":
return {"updateSuccesses": 0, "updateFailures": 0, "changedContacts": []}
raise ChatCommandError("error setting profile address", r)
async def api_set_address_settings(self, user_id: int, settings: T.AddressSettings) -> None:
@@ -236,6 +258,8 @@ class ChatApi:
)
if r["type"] == "chatItemUpdated":
return r["chatItem"]["chatItem"]
if r["type"] == "chatItemNotChanged":
return r["chatItem"]["chatItem"]
raise ChatCommandError("error updating chat item", r)
async def api_delete_chat_items(
@@ -302,6 +326,8 @@ class ChatApi:
)
if r["type"] == "rcvFileAccepted":
return r["chatItem"]
if r["type"] == "rcvFileAcceptedSndCancelled":
raise ChatCommandError("file cancelled by sender", r)
raise ChatCommandError("error receiving file", r)
async def api_cancel_file(self, file_id: int) -> None:
@@ -477,12 +503,13 @@ class ChatApi:
self,
user_id: int,
incognito: bool,
prepared_link: T.CreatedConnLink | None = None,
prepared_link: T.CreatedConnLink,
) -> ConnReqType:
args: CC.APIConnect = {"userId": user_id, "incognito": incognito}
if prepared_link is not None:
args["preparedLink_"] = prepared_link
r = await self.send_chat_cmd(CC.APIConnect_cmd_string(args))
r = await self.send_chat_cmd(
CC.APIConnect_cmd_string(
{"userId": user_id, "incognito": incognito, "preparedLink_": prepared_link}
)
)
return self._handle_connect_result(r)
async def api_connect_active_user(self, conn_link: str) -> ConnReqType:
packages/simplex-chat-python/src/simplex_chat/bot.py
+2
View File
@@ -90,6 +90,7 @@ class Bot(Client):
welcome: str | T.MsgContent | None = None,
commands: list[BotCommand] | None = None,
confirm_migrations: MigrationConfirmation = MigrationConfirmation.YES_UP,
queue_size: int | None = None,
create_address: bool = True,
update_address: bool = True,
update_profile: bool = True,
@@ -103,6 +104,7 @@ class Bot(Client):
profile=profile,
db=db,
confirm_migrations=confirm_migrations,
queue_size=queue_size,
update_profile=update_profile,
log_contacts=log_contacts,
log_network=log_network,
packages/simplex-chat-python/src/simplex_chat/client.py
+10 -10
View File
@@ -149,6 +149,7 @@ class Client:
profile: Profile,
db: Db,
confirm_migrations: MigrationConfirmation = MigrationConfirmation.YES_UP,
queue_size: int | None = None,
update_profile: bool = True,
log_contacts: bool = False,
log_network: bool = False,
@@ -156,6 +157,7 @@ class Client:
self._profile = profile
self._db = db
self._confirm_migrations = confirm_migrations
self._queue_size = queue_size
self._update_profile = update_profile
self._log_contacts = log_contacts
self._log_network = log_network
@@ -343,7 +345,7 @@ class Client:
# do post-start setup (profile sync; Bot adds address sync).
# `_stop_event` is never cleared: a stop requested during startup has
# to survive into the receive loop. A stopped client is spent.
self._api = await ChatApi.init(self._db, self._confirm_migrations)
self._api = await ChatApi.init(self._db, self._confirm_migrations, self._queue_size)
try:
user = await self._ensure_active_user()
await self._api.start_chat()
@@ -362,11 +364,6 @@ class Client:
api = self._api
if api is None:
return
if api.started:
try:
await api.stop_chat()
except Exception:
log.exception("stop_chat failed during init rollback")
try:
await api.close()
except Exception:
@@ -379,13 +376,16 @@ class Client:
if api is None:
return
# Null out the reference up-front so the Client appears closed even
# if stop_chat / close raise — otherwise `client.api` would still
# if close raises — otherwise `client.api` would still
# hand back a half-shutdown controller after `async with` exits.
self._api = None
try:
await api.stop_chat()
finally:
await api.close()
except BaseException:
# A failed stop leaves the store open; keep it so the caller can retry.
if api.initialized:
self._api = api
raise
async def _post_start(self, user: T.User) -> None:
"""Hook for subclasses to add work between `start_chat` and serving.
@@ -617,7 +617,7 @@ class Client:
# message resolve a future no one is waiting on.
if waiter in waiters:
waiters.remove(waiter)
if not waiters:
if not waiters and self._reply_waiters.get(contact_id) is waiters:
self._reply_waiters.pop(contact_id, None)
async def _receive_loop(self) -> None:
packages/simplex-chat-python/src/simplex_chat/core.py
+28 -10
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
import asyncio
import ctypes
import json
from concurrent.futures import Executor
from enum import StrEnum
from typing import Any, TypedDict
@@ -102,7 +103,9 @@ async def chat_send_cmd(ctrl: int, cmd: str) -> CR.ChatResponse:
raise ChatAPIError(f"invalid chat command result: {raw[:200]}")
async def chat_recv_msg_wait(ctrl: int, wait_us: int = 500_000) -> CEvt.ChatEvent | None:
async def chat_recv_msg_wait(
ctrl: int, wait_us: int = 500_000, executor: Executor | None = None
) -> CEvt.ChatEvent | None:
def _call() -> str:
# On timeout, the C side returns a non-NULL pointer to a single NUL byte
# (see Mobile.hs `fromMaybe ""`), so `_read_and_free` returns "" — no
@@ -110,7 +113,10 @@ async def chat_recv_msg_wait(ctrl: int, wait_us: int = 500_000) -> CEvt.ChatEven
ptr = _native.lib().chat_recv_msg_wait(ctrl, wait_us)
return _read_and_free(ptr)
raw = await asyncio.to_thread(_call)
if executor is None:
raw = await asyncio.to_thread(_call)
else:
raw = await asyncio.get_running_loop().run_in_executor(executor, _call)
if not raw:
return None
parsed = json.loads(raw)
@@ -122,17 +128,29 @@ async def chat_recv_msg_wait(ctrl: int, wait_us: int = 500_000) -> CEvt.ChatEven
raise ChatAPIError(f"invalid chat event: {raw[:200]}")
async def chat_migrate_init(db_path: str, db_key: str, confirm: MigrationConfirmation) -> int:
"""Initialize chat controller. Returns opaque ctrl pointer as Python int."""
async def chat_migrate_init(
db_path: str,
db_key: str,
confirm: MigrationConfirmation,
queue_size: int | None = None,
) -> int:
"""Initialize chat controller. Returns opaque ctrl pointer as Python int.
`queue_size` is the size of internal queues; the core default is used when None.
"""
# ctypes silently wraps ints that do not fit C int.
if queue_size is not None and ctypes.c_int(queue_size).value != queue_size:
raise ValueError(f"queue_size {queue_size} does not fit C int")
init_queue = _native.migrate_init_queue() if queue_size is not None else None
def _call() -> tuple[int, str]:
ctrl = ctypes.c_void_p()
ptr = _native.lib().chat_migrate_init(
db_path.encode("utf-8"),
db_key.encode("utf-8"),
confirm.encode("utf-8"),
ctypes.byref(ctrl),
)
args = (db_path.encode("utf-8"), db_key.encode("utf-8"), confirm.encode("utf-8"))
if init_queue is None:
ptr = _native.lib().chat_migrate_init(*args, ctypes.byref(ctrl))
else:
ptr = init_queue(*args, queue_size, ctypes.byref(ctrl))
return (ctrl.value or 0, _read_and_free(ptr))
ctrl_val, raw = await asyncio.to_thread(_call)
packages/simplex-chat-python/src/simplex_chat/types/_commands.py
+1 -1
View File
@@ -498,7 +498,7 @@ class APIConnect(TypedDict):
def APIConnect_cmd_string(self: APIConnect) -> str:
return '/_connect ' + str(self['userId']) + ((' ' + T.CreatedConnLink_cmd_string(self.get('preparedLink_'))) if self.get('preparedLink_') is not None else '')
return '/_connect ' + str(self['userId']) + (' incognito=on' if self['incognito'] else '') + ((' ' + T.CreatedConnLink_cmd_string(self.get('preparedLink_'))) if self.get('preparedLink_') is not None else '')
APIConnect_Response = CR.SentConfirmation | CR.ContactAlreadyExists | CR.SentInvitation | CR.ChatCmdError
packages/simplex-chat-python/src/simplex_chat/types/_types.py
+1 -77
View File
@@ -220,83 +220,7 @@ BadgeRedeemError = (
BadgeRedeemError_Tag = Literal["invalidCode", "serviceNotConfigured", "badgeActive", "serviceError", "invalidResponse", "unknownKeyIndex", "credentialNotVerified"]
class BadgeServiceErrorCode_badRequest(TypedDict):
type: Literal["badRequest"]
class BadgeServiceErrorCode_unsupportedVersion(TypedDict):
type: Literal["unsupportedVersion"]
class BadgeServiceErrorCode_unknownPurchaseKey(TypedDict):
type: Literal["unknownPurchaseKey"]
class BadgeServiceErrorCode_unknownOfferId(TypedDict):
type: Literal["unknownOfferId"]
class BadgeServiceErrorCode_offerDisabled(TypedDict):
type: Literal["offerDisabled"]
class BadgeServiceErrorCode_offerMismatch(TypedDict):
type: Literal["offerMismatch"]
class BadgeServiceErrorCode_productUnavailable(TypedDict):
type: Literal["productUnavailable"]
class BadgeServiceErrorCode_paymentNotEntitled(TypedDict):
type: Literal["paymentNotEntitled"]
class BadgeServiceErrorCode_paymentPending(TypedDict):
type: Literal["paymentPending"]
class BadgeServiceErrorCode_providerUnavailable(TypedDict):
type: Literal["providerUnavailable"]
class BadgeServiceErrorCode_rateLimited(TypedDict):
type: Literal["rateLimited"]
class BadgeServiceErrorCode_codeInvalid(TypedDict):
type: Literal["codeInvalid"]
class BadgeServiceErrorCode_codeUsed(TypedDict):
type: Literal["codeUsed"]
class BadgeServiceErrorCode_codeExpired(TypedDict):
type: Literal["codeExpired"]
class BadgeServiceErrorCode_receiptInvalid(TypedDict):
type: Literal["receiptInvalid"]
class BadgeServiceErrorCode_receiptUsed(TypedDict):
type: Literal["receiptUsed"]
class BadgeServiceErrorCode_internal(TypedDict):
type: Literal["internal"]
class BadgeServiceErrorCode_unknown(TypedDict):
type: Literal["unknown"]
: str
BadgeServiceErrorCode = (
BadgeServiceErrorCode_badRequest
| BadgeServiceErrorCode_unsupportedVersion
| BadgeServiceErrorCode_unknownPurchaseKey
| BadgeServiceErrorCode_unknownOfferId
| BadgeServiceErrorCode_offerDisabled
| BadgeServiceErrorCode_offerMismatch
| BadgeServiceErrorCode_productUnavailable
| BadgeServiceErrorCode_paymentNotEntitled
| BadgeServiceErrorCode_paymentPending
| BadgeServiceErrorCode_providerUnavailable
| BadgeServiceErrorCode_rateLimited
| BadgeServiceErrorCode_codeInvalid
| BadgeServiceErrorCode_codeUsed
| BadgeServiceErrorCode_codeExpired
| BadgeServiceErrorCode_receiptInvalid
| BadgeServiceErrorCode_receiptUsed
| BadgeServiceErrorCode_internal
| BadgeServiceErrorCode_unknown
)
BadgeServiceErrorCode_Tag = Literal["badRequest", "unsupportedVersion", "unknownPurchaseKey", "unknownOfferId", "offerDisabled", "offerMismatch", "productUnavailable", "paymentNotEntitled", "paymentPending", "providerUnavailable", "rateLimited", "codeInvalid", "codeUsed", "codeExpired", "receiptInvalid", "receiptUsed", "internal", "unknown"]
BadgeServiceErrorCode = Literal["bad_request", "unsupported_version", "unknown_purchase_key", "unknown_offer_id", "offer_disabled", "offer_mismatch", "product_unavailable", "payment_not_entitled", "payment_pending", "provider_unavailable", "rate_limited", "code_invalid", "code_used", "code_expired", "receipt_invalid", "receipt_used", "internal"]
BadgeStatus = Literal["active", "expired", "expiredOld", "failed", "unknownKey"]
packages/simplex-chat-python/src/simplex_chat/util.py
+1 -1
View File
@@ -101,7 +101,7 @@ def ci_content_text(chat_item: T.ChatItem) -> str | None:
return None
_BOT_COMMAND_RE = re.compile(r"^/([^\s]+)(.*)$")
_BOT_COMMAND_RE = re.compile(r"^/([^\s]+)(.*)$", re.DOTALL)
def ci_bot_command(chat_item: T.ChatItem) -> tuple[str, str] | None: