Merge branch 'master' into badge-in-app-checkout

# Conflicts:
#	simplex-chat.cabal
#	src/Simplex/Chat/Store/Postgres/Migrations.hs
#	src/Simplex/Chat/Store/SQLite/Migrations.hs
#	src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt
#	src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql

scripts/calculator-bot/Dockerfile

@@ -66,8 +66,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     rm -rf /var/lib/apt/lists/*
 
 COPY --from=libsimplex /libs /opt/simplex/libs
-# Without it, npm re-running the library's preinstall downloads the released libs.
-ENV SIMPLEX_LIBS_DIR=/opt/simplex/libs
+# The library loads libsimplex and the add-on built below instead of downloading the released ones.
+ENV SIMPLEX_LIBS_DIR=/opt/simplex/libs \
+    SIMPLEX_ADDON_PATH=/src/packages/simplex-chat-nodejs/build/Release/simplex.node
 
 WORKDIR /src
 COPY packages/simplex-chat-client/types/typescript packages/simplex-chat-client/types/typescript
@@ -79,6 +80,7 @@ COPY packages/simplex-chat-nodejs packages/simplex-chat-nodejs
 RUN cd packages/simplex-chat-nodejs && \
     npm install --no-audit --no-fund && \
     npm install --no-save --no-audit --no-fund ../simplex-chat-client/types/typescript && \
+    npx node-gyp rebuild && \
     npx tsc && \
     cp src/simplex.* dist/
 
@@ -105,7 +107,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         ca-certificates dumb-init libffi8 libgmp10 libssl3 zlib1g && \
     rm -rf /var/lib/apt/lists/*
 
+COPY --from=build /opt/simplex/libs /opt/simplex/libs
 COPY --from=build --chown=node:node /src /src
+ENV SIMPLEX_LIBS_DIR=/opt/simplex/libs \
+    SIMPLEX_ADDON_PATH=/src/packages/simplex-chat-nodejs/build/Release/simplex.node
 
 # A named volume takes this ownership; a bind mount must be owned by uid 1000.
 RUN mkdir -p /src/apps/simplex-calculator-bot/data && \

scripts/ci/packages_release.py

@@ -0,0 +1,670 @@
+#!/usr/bin/env python3
+"""Release automation for simplex-chat and @simplex-chat/types on npm and simplex-chat on PyPI.
+
+addon-tag      print the add-on release tag for the current sources
+changes        print whether the changes since a commit touch the chat core and the add-on
+build-plan     print the platforms to build the add-on on and whether to run the package tests
+prepare        update pins and versions for a release PR
+publish-plan   print the package versions that are not published yet
+record         record releases and published versions in releases.toml of simplex-chat-libs
+test-names     rename the packages to NPM_SCOPE and PYPI_PROJECT (test repositories)
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import copy
+import hashlib
+import json
+import os
+import re
+import subprocess
+import sys
+import tomllib
+import urllib.error
+import urllib.request
+from collections.abc import Callable
+from dataclasses import dataclass, field, replace
+from typing import Any
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+NODEJS_DIR = "packages/simplex-chat-nodejs"
+TYPES_DIR = "packages/simplex-chat-client/types/typescript"
+PYTHON_DIR = "packages/simplex-chat-python"
+RELEASE_FILE = f"{NODEJS_DIR}/src/release.ts"
+PYTHON_VERSION_FILE = f"{PYTHON_DIR}/src/simplex_chat/_version.py"
+ADDON_SOURCES = [f"{NODEJS_DIR}/binding.gyp", f"{NODEJS_DIR}/cpp"]
+CORE_SOURCES = ["src", "simplex-chat.cabal", "cabal.project"]
+ADDON_TAG_HASH_LENGTH = 12
+LIBS_ASSETS = [
+    "simplex-chat-libs-linux-aarch64.zip",
+    "simplex-chat-libs-linux-x86_64-postgres.zip",
+    "simplex-chat-libs-linux-x86_64.zip",
+    "simplex-chat-libs-macos-aarch64.zip",
+    "simplex-chat-libs-macos-x86_64.zip",
+    "simplex-chat-libs-windows-x86_64.zip",
+]
+ADDON_RUNNERS = {
+    "linux-aarch64": "ubuntu-22.04-arm",
+    "linux-x86_64": "ubuntu-22.04",
+    "macos-aarch64": "macos-latest",
+    "macos-x86_64": "macos-15-intel",
+    "windows-x86_64": "windows-latest",
+}
+ADDON_PLATFORMS = list(ADDON_RUNNERS)
+# build.yml builds libsimplex of pull requests on this platform only
+CORE_TEST_PLATFORM = "linux-x86_64"
+ADDON_ASSETS = [f"simplex-chat-nodejs-{platform}.node" for platform in ADDON_PLATFORMS]
+SHA256_HEX = re.compile(r"^[0-9a-f]{64}$")
+RELEASES_FILE = "releases.toml"
+RECORD_ATTEMPTS = 5
+TOML_BARE_KEY = re.compile(r"^[A-Za-z0-9_-]+$")
+
+
+# Pure functions
+
+
+def addon_tag(files: dict[str, bytes], node_addon_api: str) -> str:
+    digest = hashlib.sha256()
+    for name in sorted(files):
+        digest.update(name.encode() + b"\0" + files[name] + b"\0")
+    digest.update(f"node-addon-api@{node_addon_api}".encode())
+    return f"nodejs-addon-{digest.hexdigest()[:ADDON_TAG_HASH_LENGTH]}"
+
+
+def bump_npm_version(version: str) -> str:
+    """A package release of app version 7.1.0-beta.6 is 7.1.0-beta.6.1, then 7.1.0-beta.6.2."""
+    _, _, prerelease = version.partition("-")
+    if len(prerelease.split(".")) == 2:
+        return f"{version}.1"
+    head, _, last = version.rpartition(".")
+    return f"{head}.{int(last) + 1}" if last.isdigit() else f"{version}.1"
+
+
+def bump_python_version(version: str) -> str:
+    """A package release of app version 7.1.0b6 is 7.1.0b6.post1, then 7.1.0b6.post2."""
+    post = re.search(r"\.post(\d+)$", version)
+    return f"{version[: post.start()]}.post{int(post[1]) + 1}" if post else f"{version}.post1"
+
+
+def dist_tag(version: str) -> str:
+    return "beta" if "-" in version else "latest"
+
+
+def registry_name(name: str, scope: str | None) -> str:
+    """Package name on npm: test repositories publish under their own scope."""
+    return f"{scope}/{name.split('/')[-1]}" if scope else name
+
+
+@dataclass(frozen=True)
+class Pins:
+    libs_release: str
+    sha256: dict[str, str]
+    addon_release: str | None = None
+
+
+def parse_release(source: str) -> Pins:
+    libs = re.search(r'export const LIBS_RELEASE = "([^"]+)"', source)
+    addon = re.search(r'export const ADDON_RELEASE: string \| undefined = (?:"([^"]+)"|undefined)', source)
+    if not libs or not addon:
+        raise ValueError(f"cannot parse {RELEASE_FILE}")
+    hashes = dict(re.findall(r'^ {2}"([^"]+)": "([0-9a-f]{64})",$', source, re.M))
+    return Pins(libs[1], hashes, addon[1])
+
+
+def render_release(pins: Pins) -> str:
+    hashes = "".join(f'  "{asset}": "{pins.sha256[asset]}",\n' for asset in sorted(pins.sha256))
+    addon = f'"{pins.addon_release}"' if pins.addon_release else "undefined"
+    return (
+        "// Written by scripts/ci/packages_release.py from the simplex-chat-libs releases.\n"
+        "\n"
+        f'export const LIBS_RELEASE = "{pins.libs_release}"\n'
+        "\n"
+        f"export const ADDON_RELEASE: string | undefined = {addon}\n"
+        "\n"
+        "export const SHA256: {readonly [asset: string]: string} = {\n"
+        f"{hashes}}}\n"
+    )
+
+
+def parse_python_version(source: str) -> tuple[str, Pins]:
+    version = re.search(r'^__version__ = "([^"]+)"', source, re.M)
+    libs = re.search(r'^LIBS_RELEASE = "([^"]+)"', source, re.M)
+    if not version or not libs:
+        raise ValueError(f"cannot parse {PYTHON_VERSION_FILE}")
+    hashes = dict(re.findall(r'^ {4}"([^"]+)": "([0-9a-f]{64})",$', source, re.M))
+    return version[1], Pins(libs[1], hashes)
+
+
+def render_python_version(version: str, pins: Pins) -> str:
+    hashes = "".join(f'    "{asset}": "{pins.sha256[asset]}",\n' for asset in sorted(pins.sha256))
+    return (
+        '"""Package version and the simplex-chat-libs release it loads.\n'
+        "\n"
+        "Written by scripts/ci/packages_release.py: the release PR bumps the version\n"
+        "and pins new library releases with the SHA-256 of their assets.\n"
+        '"""\n'
+        "\n"
+        f'__version__ = "{version}"  # PEP 440 — read by hatchling for wheel metadata\n'
+        f'LIBS_RELEASE = "{pins.libs_release}"  # simplex-chat-libs release tag\n'
+        "LIBS_SHA256 = {\n"
+        f"{hashes}}}\n"
+    )
+
+
+def release_digests(release: dict | None, assets: list[str]) -> dict[str, str] | None:
+    """Asset digests of a GitHub release, or None when the release is missing or incomplete."""
+    if release is None:
+        return None
+    digests = {a["name"]: (a.get("digest") or "").removeprefix("sha256:") for a in release["assets"]}
+    if not all(SHA256_HEX.match(digests.get(asset, "")) for asset in assets):
+        return None
+    return {asset: digests[asset] for asset in assets}
+
+
+@dataclass(frozen=True)
+class Package:
+    version: str
+    changed: bool
+    published: bool
+
+
+@dataclass(frozen=True)
+class State:
+    libs_tag: str
+    libs_digests: dict[str, str] | None
+    addon_tag: str
+    addon_digests: dict[str, str] | None
+    nodejs_pins: Pins
+    python_pins: Pins
+    types: Package
+    nodejs: Package
+    python: Package
+    types_version: str | None = None
+    nodejs_version: str | None = None
+    python_version: str | None = None
+
+
+@dataclass(frozen=True)
+class Plan:
+    nodejs_pins: Pins
+    python_pins: Pins
+    types_version: str
+    types_bumped: bool
+    nodejs_version: str
+    nodejs_changed: bool
+    python_version: str
+    python_changed: bool
+    notes: list[str] = field(default_factory=list)
+
+    @property
+    def types_dependency(self) -> str | None:
+        return f"^{self.types_version}" if self.types_bumped else None
+
+    @property
+    def title(self) -> str:
+        parts = []
+        if self.types_bumped:
+            parts.append(f"types {self.types_version}")
+        if self.nodejs_changed:
+            parts.append(f"nodejs {self.nodejs_version}")
+        if self.python_changed:
+            parts.append(f"python {self.python_version}")
+        return f"packages: release {', '.join(parts)}"
+
+
+def plan(state: State) -> Plan:
+    """What a release PR changes, from the state of the branch and the releases."""
+    notes: list[str] = []
+
+    def pin(pins: Pins, field_name: str, tag: str, digests: dict[str, str] | None, what: str) -> tuple[Pins, bool]:
+        current = getattr(pins, field_name)
+        if current == tag:
+            return pins, False
+        if digests is None:
+            notes.append(f"{what} release {tag} is missing or incomplete; {current or 'none'} stays pinned")
+            return pins, False
+        notes.append(f"{what}: {tag}")
+        return replace(pins, **{field_name: tag, "sha256": {**pins.sha256, **digests}}), True
+
+    nodejs_pins, nodejs_libs = pin(
+        state.nodejs_pins, "libs_release", state.libs_tag, state.libs_digests, "Node.js libsimplex"
+    )
+    nodejs_pins, addon = pin(nodejs_pins, "addon_release", state.addon_tag, state.addon_digests, "Node.js add-on")
+    python_pins, python_libs = pin(
+        state.python_pins, "libs_release", state.libs_tag, state.libs_digests, "Python libsimplex"
+    )
+
+    def next_version(pkg: Package, changed: bool, override: str | None, bump) -> str:
+        if override:
+            return override
+        return bump(pkg.version) if changed and pkg.published else pkg.version
+
+    types_version = next_version(state.types, state.types.changed, state.types_version, bump_npm_version)
+    types_bumped = types_version != state.types.version
+    nodejs_changed = nodejs_libs or addon or types_bumped or state.nodejs.changed
+    nodejs_version = next_version(state.nodejs, nodejs_changed, state.nodejs_version, bump_npm_version)
+    python_changed = python_libs or state.python.changed
+    python_version = next_version(state.python, python_changed, state.python_version, bump_python_version)
+    for name, old, new in [
+        ("@simplex-chat/types", state.types.version, types_version),
+        ("simplex-chat (npm)", state.nodejs.version, nodejs_version),
+        ("simplex-chat (PyPI)", state.python.version, python_version),
+    ]:
+        if old != new:
+            notes.append(f"{name}: {old} → {new}")
+    return Plan(
+        nodejs_pins=nodejs_pins,
+        python_pins=python_pins,
+        types_version=types_version,
+        types_bumped=types_bumped,
+        nodejs_version=nodejs_version,
+        nodejs_changed=nodejs_changed or nodejs_version != state.nodejs.version,
+        python_version=python_version,
+        python_changed=python_changed or python_version != state.python.version,
+        notes=notes,
+    )
+
+
+@dataclass(frozen=True)
+class BuildPlan:
+    platforms: list[str]
+    test: bool
+
+
+def build_plan(
+    event: str, release_missing: bool, libs_artifact: bool, libs_release: str, core_changed: bool, addon_changed: bool
+) -> BuildPlan:
+    """Each package test runs once, against the libsimplex of the change: a pull request that changes the core
+    is tested by build.yml with its own libsimplex (libs_artifact), a new library release with that release."""
+    if libs_artifact:
+        return BuildPlan([CORE_TEST_PLATFORM], test=True)
+    if libs_release:
+        return BuildPlan(ADDON_PLATFORMS, test=True)
+    if event == "pull_request":
+        if core_changed:
+            return BuildPlan(ADDON_PLATFORMS if addon_changed else [], test=False)
+        return BuildPlan(ADDON_PLATFORMS, test=True)
+    return BuildPlan(ADDON_PLATFORMS if release_missing else [], test=True)
+
+
+def build_matrix(platforms: list[str]) -> str:
+    return json.dumps([{"platform": platform, "os": ADDON_RUNNERS[platform]} for platform in platforms])
+
+
+def recorded_commit(releases: dict, key: str, version: str) -> str | None:
+    """The simplex-chat commit recorded for a version; key is "core", "npm/<name>" or "pypi/<name>"."""
+    table = releases
+    for part in key.split("/", 1):
+        table = table.get(part, {})
+    commit = table.get(version)
+    return commit if isinstance(commit, str) else None
+
+
+def add_releases(releases: dict, entries: list[tuple[str, str]], commit: str) -> dict:
+    """A recorded version keeps its first commit: re-running a release does not rewrite history."""
+    result = copy.deepcopy(releases)
+    for key, version in entries:
+        table = result
+        for part in key.split("/", 1):
+            table = table.setdefault(part, {})
+        table.setdefault(version, commit)
+    return result
+
+
+def render_releases(releases: dict) -> str:
+    def toml_key(key: str) -> str:
+        return key if TOML_BARE_KEY.match(key) else json.dumps(key)
+
+    sections: list[str] = []
+
+    # in the order of recording: entries are only added, so that order is chronological
+    def walk(path: list[str], table: dict) -> None:
+        values = [(k, v) for k, v in table.items() if isinstance(v, str)]
+        if values:
+            lines = [f"[{'.'.join(toml_key(part) for part in path)}]"]
+            lines += [f"{toml_key(version)} = {json.dumps(commit)}" for version, commit in values]
+            sections.append("\n".join(lines) + "\n")
+        for name, child in table.items():
+            if isinstance(child, dict):
+                walk([*path, name], child)
+
+    walk([], releases)
+    return "\n".join(sections)
+
+
+# Repository and registry access
+
+
+def git(*args: str) -> str:
+    return subprocess.run(["git", *args], cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip()
+
+
+def read_text(file: str) -> str:
+    return (ROOT / file).read_text()
+
+
+def read_field(file: str, pattern: str) -> str:
+    found = re.search(pattern, read_text(file), re.M)
+    if not found:
+        raise ValueError(f"no match for {pattern} in {file}")
+    return found[1]
+
+
+def read_json(file: str) -> dict:
+    return json.loads(read_text(file))
+
+
+def write_json(file: str, value: dict) -> None:
+    (ROOT / file).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n")
+
+
+def current_addon_tag() -> str:
+    files = {
+        path.relative_to(ROOT).as_posix(): path.read_bytes()
+        for source in ADDON_SOURCES
+        for path in ([ROOT / source] if (ROOT / source).is_file() else sorted((ROOT / source).rglob("*")))
+        if path.is_file()
+    }
+    return addon_tag(files, read_json(f"{NODEJS_DIR}/package.json")["devDependencies"]["node-addon-api"])
+
+
+def changed_since(base: str, sources: list[str]) -> bool:
+    diff = subprocess.run(["git", "diff", "--quiet", base, "HEAD", "--", *sources], cwd=ROOT)
+    return diff.returncode != 0
+
+
+def changes(base: str) -> None:
+    print(f"core={'true' if changed_since(base, CORE_SOURCES) else ''}")
+    print(f"addon={'true' if changed_since(base, ADDON_SOURCES) else ''}")
+
+
+def print_build_plan(args: argparse.Namespace) -> None:
+    pull_request = args.event == "pull_request" and not args.libs_artifact
+    result = build_plan(
+        args.event,
+        release_missing=args.release_missing,
+        libs_artifact=args.libs_artifact,
+        libs_release=args.libs_release,
+        core_changed=pull_request and changed_since(args.base, CORE_SOURCES),
+        addon_changed=pull_request and changed_since(args.base, ADDON_SOURCES),
+    )
+    print(f"platforms={build_matrix(result.platforms)}")
+    print(f"test={'true' if result.test else ''}")
+
+
+def package_state(
+    key: str,
+    version: str,
+    directory: str,
+    version_file: str,
+    version_pattern: str,
+    releases: dict,
+    registry_published: Callable[[], bool],
+) -> Package:
+    """A recorded version is published from its recorded commit; the registry is asked about other versions,
+    which are compared with the commit that set them."""
+    recorded = recorded_commit(releases, key, version)
+    base = (
+        recorded
+        if recorded and commit_exists(recorded)
+        else git("log", "-1", "--format=%H", "-G", version_pattern, "--", version_file)
+    )
+    changed = changed_since(base, [directory, f":!{directory}/tests"])
+    return Package(version, changed, recorded is not None or registry_published())
+
+
+def http_json(url: str, headers: dict[str, str] | None = None, method: str = "GET", body: dict | None = None) -> Any:
+    """The JSON response, or None for 404."""
+    request = urllib.request.Request(
+        url,
+        headers={"User-Agent": "simplex-chat-release", **(headers or {})},
+        method=method,
+        data=json.dumps(body).encode() if body is not None else None,
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=60) as response:
+            return json.load(response)
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return None
+        raise
+
+
+# Workflows pass unset repository variables as empty strings.
+def npm_published(name: str, version: str) -> bool:
+    registry = os.environ.get("NPM_REGISTRY") or "https://registry.npmjs.org"
+    package = registry_name(name, os.environ.get("NPM_SCOPE")).replace("/", "%2F")
+    return http_json(f"{registry}/{package}/{version}") is not None
+
+
+def python_project() -> str:
+    return read_field(f"{PYTHON_DIR}/pyproject.toml", r'^name = "([^"]+)"')
+
+
+def pypi_published(version: str) -> bool:
+    index = os.environ.get("PYPI_URL") or "https://pypi.org"
+    project = os.environ.get("PYPI_PROJECT") or python_project()
+    return http_json(f"{index}/pypi/{project}/{version}/json") is not None
+
+
+def libs_api(path: str, method: str = "GET", body: dict | None = None) -> Any:
+    """GitHub API of the simplex-chat-libs repository."""
+    repo = os.environ.get("LIBS_REPOSITORY") or "simplex-chat/simplex-chat-libs"
+    headers = {"Accept": "application/vnd.github+json"}
+    if token := os.environ.get("GH_TOKEN"):
+        headers["Authorization"] = f"Bearer {token}"
+    return http_json(f"https://api.github.com/repos/{repo}/{path}", headers, method, body)
+
+
+def github_release(tag: str) -> dict | None:
+    return libs_api(f"releases/tags/{tag}")
+
+
+def read_releases() -> tuple[dict, str | None]:
+    """The releases and the blob of the file, which a change must name."""
+    file = libs_api(f"contents/{RELEASES_FILE}")
+    if file is None:
+        return {}, None
+    return tomllib.loads(base64.b64decode(file["content"]).decode()), file["sha"]
+
+
+def record_releases(entries: list[tuple[str, str]], commit: str) -> str:
+    """Adds the entries in one commit of simplex-chat-libs and returns that commit."""
+    message = ", ".join(f"{key} {version}" for key, version in entries)
+    attempt = 1
+    while True:
+        releases, blob = read_releases()
+        updated = add_releases(releases, entries, commit)
+        if blob and updated == releases:
+            return libs_api(f"commits?path={RELEASES_FILE}&per_page=1")[0]["sha"]
+        body = {
+            "message": message,
+            "content": base64.b64encode(render_releases(updated).encode()).decode(),
+            "committer": {"name": "SimpleX Chat", "email": "build@simplex.chat"},
+            **({"sha": blob} if blob else {}),
+        }
+        try:
+            return libs_api(f"contents/{RELEASES_FILE}", "PUT", body)["commit"]["sha"]
+        except urllib.error.HTTPError as e:
+            # 409 and 422: another writer changed or created the file since it was read
+            if e.code not in (409, 422) or attempt == RECORD_ATTEMPTS:
+                raise
+            attempt += 1
+
+
+def commit_exists(commit: str) -> bool:
+    found = subprocess.run(["git", "cat-file", "-e", f"{commit}^{{commit}}"], cwd=ROOT, capture_output=True)
+    return found.returncode == 0
+
+
+def npm_key(directory: str) -> str:
+    return f"npm/{read_json(f'{directory}/package.json')['name']}"
+
+
+def pypi_key() -> str:
+    return f"pypi/{python_project()}"
+
+
+def npm_package(directory: str, releases: dict) -> Package:
+    package = read_json(f"{directory}/package.json")
+    name, version = package["name"], package["version"]
+    return package_state(
+        npm_key(directory),
+        version,
+        directory,
+        f"{directory}/package.json",
+        '"version":',
+        releases,
+        lambda: npm_published(name, version),
+    )
+
+
+def prepare(args: argparse.Namespace) -> None:
+    cabal_version = read_field("simplex-chat.cabal", r"^version:\s*(\S+)")
+    libs_tag = f"core-{cabal_version}"
+    addon = current_addon_tag()
+    python_version, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE))
+    releases, _ = read_releases()
+    result = plan(
+        State(
+            libs_tag=libs_tag,
+            libs_digests=release_digests(github_release(libs_tag), LIBS_ASSETS),
+            addon_tag=addon,
+            addon_digests=release_digests(github_release(addon), ADDON_ASSETS),
+            nodejs_pins=parse_release(read_text(RELEASE_FILE)),
+            python_pins=python_pins,
+            types=npm_package(TYPES_DIR, releases),
+            nodejs=npm_package(NODEJS_DIR, releases),
+            python=package_state(
+                pypi_key(),
+                python_version,
+                PYTHON_DIR,
+                PYTHON_VERSION_FILE,
+                "^__version__ =",
+                releases,
+                lambda: pypi_published(python_version),
+            ),
+            types_version=args.types_version,
+            nodejs_version=args.nodejs_version,
+            python_version=args.python_version,
+        )
+    )
+    (ROOT / RELEASE_FILE).write_text(render_release(result.nodejs_pins))
+    (ROOT / PYTHON_VERSION_FILE).write_text(render_python_version(result.python_version, result.python_pins))
+    types = read_json(f"{TYPES_DIR}/package.json")
+    write_json(f"{TYPES_DIR}/package.json", {**types, "version": result.types_version})
+    nodejs = read_json(f"{NODEJS_DIR}/package.json")
+    dependencies = nodejs["dependencies"]
+    if result.types_dependency:
+        dependencies = {**dependencies, "@simplex-chat/types": result.types_dependency}
+    write_json(f"{NODEJS_DIR}/package.json", {**nodejs, "version": result.nodejs_version, "dependencies": dependencies})
+    summary = result.title + "\n\n" + "".join(f"- {note}\n" for note in result.notes)
+    if args.summary:
+        Path(args.summary).write_text(summary)
+    print(summary, end="")
+
+
+def publish_plan() -> None:
+    """A recorded version is published: npm and PyPI list a new version minutes after it is published."""
+    releases, _ = read_releases()
+    for key, directory in [("types", TYPES_DIR), ("nodejs", NODEJS_DIR)]:
+        package = read_json(f"{directory}/package.json")
+        published = recorded_commit(releases, npm_key(directory), package["version"]) or npm_published(
+            package["name"], package["version"]
+        )
+        print(f"{key}_version={'' if published else package['version']}")
+        print(f"{key}_dist_tag={dist_tag(package['version'])}")
+    version, _ = parse_python_version(read_text(PYTHON_VERSION_FILE))
+    published = recorded_commit(releases, pypi_key(), version) or pypi_published(version)
+    print(f"python_version={'' if published else version}")
+
+
+def record(args: argparse.Namespace) -> None:
+    versions = [
+        ("core", args.core),
+        (npm_key(TYPES_DIR), args.types),
+        (npm_key(NODEJS_DIR), args.nodejs),
+        (pypi_key(), args.python),
+    ]
+    entries = [(key, version) for key, version in versions if version]
+    if not entries:
+        sys.exit("nothing to record")
+    print(record_releases(entries, args.commit))
+
+
+def test_names() -> None:
+    if scope := os.environ.get("NPM_SCOPE"):
+        # npm provenance requires the repository of the publishing workflow.
+        url = f"git+https://github.com/{os.environ['GITHUB_REPOSITORY']}.git"
+        types = read_json(f"{TYPES_DIR}/package.json")
+        write_json(
+            f"{TYPES_DIR}/package.json",
+            {**types, "name": registry_name(types["name"], scope), "repository": {**types["repository"], "url": url}},
+        )
+        nodejs = read_json(f"{NODEJS_DIR}/package.json")
+        alias = f"npm:{registry_name(types['name'], scope)}@{nodejs['dependencies'][types['name']]}"
+        write_json(
+            f"{NODEJS_DIR}/package.json",
+            {
+                **nodejs,
+                "name": registry_name(nodejs["name"], scope),
+                "repository": {**nodejs["repository"], "url": url},
+                "dependencies": {**nodejs["dependencies"], types["name"]: alias},
+            },
+        )
+    if project := os.environ.get("PYPI_PROJECT"):
+        pyproject = ROOT / PYTHON_DIR / "pyproject.toml"
+        pyproject.write_text(
+            re.sub(r'^name = "[^"]+"', f'name = "{project}"', pyproject.read_text(), count=1, flags=re.M)
+        )
+
+
+def github_bool(value: str) -> bool:
+    """GitHub expressions pass booleans to the shell as "true", "false" or ""."""
+    return value == "true"
+
+
+def main(argv: list[str]) -> None:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    commands = parser.add_subparsers(dest="command", required=True)
+    commands.add_parser("addon-tag")
+    changes_parser = commands.add_parser("changes")
+    changes_parser.add_argument("base", help="the commit to compare with")
+    build_parser = commands.add_parser("build-plan")
+    build_parser.add_argument("--event", required=True, help="github.event_name")
+    build_parser.add_argument("--base", default="", help="the base commit of a pull request")
+    build_parser.add_argument("--release-missing", type=github_bool, default=False)
+    build_parser.add_argument("--libs-artifact", type=github_bool, default=False)
+    build_parser.add_argument("--libs-release", default="")
+    prepare_parser = commands.add_parser("prepare")
+    for package in ["nodejs", "types", "python"]:
+        prepare_parser.add_argument(f"--{package}-version", help="this version instead of a bump when needed")
+    prepare_parser.add_argument("--summary", help="file for the PR title and body")
+    commands.add_parser("publish-plan")
+    record_parser = commands.add_parser("record")
+    record_parser.add_argument("--commit", required=True, help="the simplex-chat commit released or published from")
+    for name in ["core", "types", "nodejs", "python"]:
+        record_parser.add_argument(f"--{name}", default="", help=f"{name} version, recorded when not empty")
+    commands.add_parser("test-names")
+    args = parser.parse_args(argv)
+    if args.command == "addon-tag":
+        print(current_addon_tag())
+    elif args.command == "changes":
+        changes(args.base)
+    elif args.command == "build-plan":
+        print_build_plan(args)
+    elif args.command == "prepare":
+        prepare(args)
+    elif args.command == "publish-plan":
+        publish_plan()
+    elif args.command == "record":
+        record(args)
+    else:
+        test_names()
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

scripts/ci/test_packages_release.py

@@ -0,0 +1,476 @@
+import base64
+import contextlib
+import email.message
+import io
+import subprocess
+import tomllib
+import urllib.error
+import tempfile
+import unittest
+from dataclasses import replace
+from pathlib import Path
+from unittest import mock
+
+import packages_release
+from packages_release import (
+    ADDON_ASSETS,
+    NODEJS_DIR,
+    PYTHON_VERSION_FILE,
+    TYPES_DIR,
+    ADDON_PLATFORMS,
+    BuildPlan,
+    LIBS_ASSETS,
+    Package,
+    Pins,
+    State,
+    addon_tag,
+    build_matrix,
+    build_plan,
+    bump_npm_version,
+    bump_python_version,
+    changed_since,
+    add_releases,
+    package_state,
+    publish_plan,
+    record_releases,
+    recorded_commit,
+    render_releases,
+    dist_tag,
+    npm_published,
+    parse_python_version,
+    parse_release,
+    plan,
+    pypi_published,
+    registry_name,
+    release_digests,
+    render_python_version,
+    render_release,
+)
+
+
+def sha(n: int) -> str:
+    return f"{n:x}" * 64
+
+
+def digests(assets: list[str], n: int) -> dict[str, str]:
+    return {asset: sha(n) for asset in assets}
+
+
+def github_release(assets: list[str], n: int) -> dict:
+    return {"assets": [{"name": name, "digest": f"sha256:{sha(n)}"} for name in assets]}
+
+
+def state(**overrides) -> State:
+    current = State(
+        libs_tag="core-1.0.0.1",
+        libs_digests=digests(LIBS_ASSETS, 1),
+        addon_tag="nodejs-addon-aaa",
+        addon_digests=digests(ADDON_ASSETS, 2),
+        nodejs_pins=Pins("core-1.0.0.1", {**digests(LIBS_ASSETS, 1), **digests(ADDON_ASSETS, 2)}, "nodejs-addon-aaa"),
+        python_pins=Pins("core-1.0.0.1", digests(LIBS_ASSETS, 1)),
+        types=Package("0.12.1", changed=False, published=True),
+        nodejs=Package("7.1.0-beta.6.1", changed=False, published=True),
+        python=Package("7.1.0b6", changed=False, published=True),
+    )
+    return replace(current, **overrides)
+
+
+NEW_LIBS = {"libs_tag": "core-1.0.0.2", "libs_digests": digests(LIBS_ASSETS, 3)}
+NEW_ADDON = {"addon_tag": "nodejs-addon-bbb", "addon_digests": digests(ADDON_ASSETS, 4)}
+
+
+class VersionTest(unittest.TestCase):
+    def test_npm_bump_adds_a_package_release_to_an_app_version(self):
+        self.assertEqual(bump_npm_version("7.1.0-beta.6"), "7.1.0-beta.6.1")
+        self.assertEqual(bump_npm_version("7.1.0-beta.6.1"), "7.1.0-beta.6.2")
+        self.assertEqual(bump_npm_version("0.12.9"), "0.12.10")
+        self.assertEqual(bump_npm_version("7.0.2"), "7.0.3")
+
+    def test_python_bump_uses_post_releases(self):
+        self.assertEqual(bump_python_version("7.1.0b6"), "7.1.0b6.post1")
+        self.assertEqual(bump_python_version("7.1.0b6.post9"), "7.1.0b6.post10")
+        self.assertEqual(bump_python_version("7.0.2"), "7.0.2.post1")
+
+    def test_prereleases_publish_as_beta(self):
+        self.assertEqual(dist_tag("7.1.0-beta.6.2"), "beta")
+        self.assertEqual(dist_tag("7.1.0"), "latest")
+
+    def test_test_scope_renames_npm_packages(self):
+        self.assertEqual(registry_name("@simplex-chat/types", "@test"), "@test/types")
+        self.assertEqual(registry_name("simplex-chat", "@test"), "@test/simplex-chat")
+        self.assertEqual(registry_name("simplex-chat", None), "simplex-chat")
+
+
+class AddonTagTest(unittest.TestCase):
+    def test_depends_on_sources_and_node_addon_api(self):
+        files = {"a.cc": b"1", "b.h": b"2"}
+        tag = addon_tag(files, "^8.5.0")
+        self.assertRegex(tag, r"^nodejs-addon-[0-9a-f]{12}$")
+        self.assertEqual(addon_tag(dict(reversed(files.items())), "^8.5.0"), tag)
+        self.assertNotEqual(addon_tag({**files, "b.h": b"3"}, "^8.5.0"), tag)
+        self.assertNotEqual(addon_tag({"a.cc": b"1", "c.h": b"2"}, "^8.5.0"), tag)
+        self.assertNotEqual(addon_tag(files, "^8.6.0"), tag)
+
+
+class PinFilesTest(unittest.TestCase):
+    def test_release_ts_round_trip(self):
+        pins = Pins("core-1.0.0.1", {"b.zip": sha(2), "a.zip": sha(1)}, "nodejs-addon-aaa")
+        self.assertEqual(parse_release(render_release(pins)), pins)
+        self.assertIsNone(parse_release(render_release(replace(pins, addon_release=None))).addon_release)
+
+    def test_python_version_round_trip(self):
+        pins = Pins("core-1.0.0.1", {"b.zip": sha(2), "a.zip": sha(1)})
+        self.assertEqual(parse_python_version(render_python_version("7.1.0b6.post1", pins)), ("7.1.0b6.post1", pins))
+
+
+class ReleaseDigestsTest(unittest.TestCase):
+    def test_requires_every_asset_with_a_digest(self):
+        self.assertEqual(release_digests(github_release(LIBS_ASSETS, 3), LIBS_ASSETS), digests(LIBS_ASSETS, 3))
+        self.assertIsNone(release_digests(github_release(LIBS_ASSETS[1:], 3), LIBS_ASSETS))
+        self.assertIsNone(release_digests(None, LIBS_ASSETS))
+        no_digests = {"assets": [{"name": name, "digest": None} for name in LIBS_ASSETS]}
+        self.assertIsNone(release_digests(no_digests, LIBS_ASSETS))
+
+
+class PlanTest(unittest.TestCase):
+    def test_nothing_changes_when_current(self):
+        result = plan(state())
+        self.assertEqual((result.nodejs_changed, result.python_changed, result.types_bumped), (False, False, False))
+        self.assertEqual(result.notes, [])
+
+    def test_new_libs_release_updates_both_libraries(self):
+        result = plan(state(**NEW_LIBS))
+        self.assertEqual(result.nodejs_pins.libs_release, "core-1.0.0.2")
+        self.assertEqual(result.python_pins.libs_release, "core-1.0.0.2")
+        self.assertEqual(result.nodejs_pins.sha256[LIBS_ASSETS[0]], sha(3))
+        self.assertEqual(result.nodejs_pins.sha256[ADDON_ASSETS[0]], sha(2))
+        self.assertEqual(result.python_pins.sha256[LIBS_ASSETS[0]], sha(3))
+        self.assertEqual(result.title, "packages: release nodejs 7.1.0-beta.6.2, python 7.1.0b6.post1")
+
+    def test_new_addon_release_updates_nodejs_only(self):
+        result = plan(state(**NEW_ADDON))
+        self.assertEqual(result.nodejs_pins.addon_release, "nodejs-addon-bbb")
+        self.assertEqual(result.nodejs_pins.sha256[ADDON_ASSETS[0]], sha(4))
+        self.assertEqual(result.title, "packages: release nodejs 7.1.0-beta.6.2")
+
+    def test_incomplete_release_keeps_the_pins(self):
+        result = plan(state(addon_tag="nodejs-addon-bbb", addon_digests=None))
+        self.assertEqual(result.nodejs_pins.addon_release, "nodejs-addon-aaa")
+        self.assertFalse(result.nodejs_changed)
+        self.assertEqual(
+            result.notes,
+            ["Node.js add-on release nodejs-addon-bbb is missing or incomplete; nodejs-addon-aaa stays pinned"],
+        )
+
+    def test_source_changes_bump_their_package(self):
+        self.assertEqual(
+            plan(state(nodejs=Package("7.1.0-beta.6.1", True, True))).title, "packages: release nodejs 7.1.0-beta.6.2"
+        )
+        self.assertEqual(
+            plan(state(python=Package("7.1.0b6", True, True))).title, "packages: release python 7.1.0b6.post1"
+        )
+
+    def test_types_bump_updates_the_nodejs_dependency(self):
+        result = plan(state(types=Package("0.12.1", True, True)))
+        self.assertEqual(result.types_dependency, "^0.12.2")
+        self.assertEqual(result.title, "packages: release types 0.12.2, nodejs 7.1.0-beta.6.2")
+
+    def test_all_changes_combine_into_one_release(self):
+        result = plan(
+            state(
+                **NEW_LIBS,
+                **NEW_ADDON,
+                types=Package("0.12.1", True, True),
+                nodejs=Package("7.1.0-beta.6.1", True, True),
+                python=Package("7.1.0b6", True, True),
+            )
+        )
+        self.assertEqual(
+            (result.nodejs_pins.libs_release, result.nodejs_pins.addon_release), ("core-1.0.0.2", "nodejs-addon-bbb")
+        )
+        self.assertEqual(result.title, "packages: release types 0.12.2, nodejs 7.1.0-beta.6.2, python 7.1.0b6.post1")
+
+    def test_unpublished_versions_are_not_bumped_again(self):
+        result = plan(
+            state(
+                **NEW_LIBS, nodejs=Package("7.1.0-beta.6.2", True, False), python=Package("7.1.0b6.post1", False, False)
+            )
+        )
+        self.assertEqual(result.title, "packages: release nodejs 7.1.0-beta.6.2, python 7.1.0b6.post1")
+
+    def test_requested_versions_win(self):
+        result = plan(state(nodejs_version="7.2.0", types_version="0.13.0", python_version="7.2.0"))
+        self.assertEqual(result.types_dependency, "^0.13.0")
+        self.assertEqual(result.title, "packages: release types 0.13.0, nodejs 7.2.0, python 7.2.0")
+
+
+class BuildPlanTest(unittest.TestCase):
+    def plan(self, event="push", release_missing=False, libs_artifact=False, libs_release="", core=False, addon=False):
+        return build_plan(event, release_missing, libs_artifact, libs_release, core, addon)
+
+    def test_push_builds_a_missing_addon_release_and_tests_it(self):
+        self.assertEqual(self.plan(release_missing=True), BuildPlan(ADDON_PLATFORMS, test=True))
+        self.assertEqual(self.plan(), BuildPlan([], test=True))
+
+    def test_pull_request_tests_with_the_pinned_libs(self):
+        self.assertEqual(self.plan("pull_request"), BuildPlan(ADDON_PLATFORMS, test=True))
+        self.assertEqual(self.plan("pull_request", addon=True), BuildPlan(ADDON_PLATFORMS, test=True))
+
+    def test_core_change_leaves_tests_to_build_yml(self):
+        self.assertEqual(self.plan("pull_request", core=True), BuildPlan([], test=False))
+        self.assertEqual(self.plan("pull_request", core=True, addon=True), BuildPlan(ADDON_PLATFORMS, test=False))
+
+    def test_libsimplex_of_the_pull_request_is_tested_on_its_platform(self):
+        self.assertEqual(
+            self.plan("pull_request", libs_artifact=True, core=True), BuildPlan(["linux-x86_64"], test=True)
+        )
+
+    def test_new_library_release_is_tested_on_every_platform(self):
+        self.assertEqual(self.plan(libs_release="core-1.0.0.2"), BuildPlan(ADDON_PLATFORMS, test=True))
+
+    def test_matrix_names_the_runner_of_each_platform(self):
+        self.assertEqual(
+            build_matrix(["linux-x86_64", "windows-x86_64"]),
+            '[{"platform": "linux-x86_64", "os": "ubuntu-22.04"}, {"platform": "windows-x86_64", "os": "windows-latest"}]',
+        )
+
+
+class RegistryUrlTest(unittest.TestCase):
+    def urls(self, env: dict[str, str]) -> list[str]:
+        with (
+            mock.patch.dict("os.environ", env),
+            mock.patch.object(packages_release, "http_json", return_value={}) as get,
+        ):
+            npm_published("simplex-chat", "1.0.0")
+            pypi_published("1.0.0")
+            packages_release.github_release("core-1.0.0.1")
+        return [call.args[0] for call in get.call_args_list]
+
+    def test_empty_variables_use_the_public_registries(self):
+        self.assertEqual(
+            self.urls({"NPM_REGISTRY": "", "NPM_SCOPE": "", "PYPI_URL": "", "PYPI_PROJECT": "", "LIBS_REPOSITORY": ""}),
+            [
+                "https://registry.npmjs.org/simplex-chat/1.0.0",
+                "https://pypi.org/pypi/simplex-chat/1.0.0/json",
+                "https://api.github.com/repos/simplex-chat/simplex-chat-libs/releases/tags/core-1.0.0.1",
+            ],
+        )
+
+    def test_variables_select_test_registries(self):
+        self.assertEqual(
+            self.urls(
+                {
+                    "NPM_SCOPE": "@test",
+                    "PYPI_URL": "https://test.pypi.org",
+                    "PYPI_PROJECT": "sx-test",
+                    "LIBS_REPOSITORY": "a/b-libs",
+                }
+            ),
+            [
+                "https://registry.npmjs.org/@test%2Fsimplex-chat/1.0.0",
+                "https://test.pypi.org/pypi/sx-test/1.0.0/json",
+                "https://api.github.com/repos/a/b-libs/releases/tags/core-1.0.0.1",
+            ],
+        )
+
+
+class ReleasesFileTest(unittest.TestCase):
+    RELEASES = {
+        "core": {"7.1.0.11": "c1"},
+        "npm": {"simplex-chat": {"7.1.0-beta.6.3": "n1"}, "@simplex-chat/types": {"0.12.2": "t1"}},
+        "pypi": {"simplex-chat": {"7.1.0b6": "p1"}},
+    }
+
+    def test_renders_toml_that_reads_back(self):
+        text = render_releases(self.RELEASES)
+        self.assertEqual(tomllib.loads(text), self.RELEASES)
+        self.assertIn('[core]\n"7.1.0.11" = "c1"\n', text)
+        self.assertIn('[npm."@simplex-chat/types"]\n', text)
+        self.assertIn("[npm.simplex-chat]\n", text)
+
+    def test_keeps_the_order_of_recording(self):
+        releases = {"npm": {"simplex-chat": {"7.1.0-beta.6.9": "a"}}}
+        updated = add_releases(releases, [("npm/simplex-chat", "7.1.0-beta.6.10"), ("core", "7.1.0.15")], "b")
+        self.assertEqual(
+            render_releases(updated),
+            '[npm.simplex-chat]\n"7.1.0-beta.6.9" = "a"\n"7.1.0-beta.6.10" = "b"\n\n[core]\n"7.1.0.15" = "b"\n',
+        )
+
+    def test_finds_the_recorded_commit(self):
+        self.assertEqual(recorded_commit(self.RELEASES, "npm/@simplex-chat/types", "0.12.2"), "t1")
+        self.assertEqual(recorded_commit(self.RELEASES, "core", "7.1.0.11"), "c1")
+        self.assertIsNone(recorded_commit(self.RELEASES, "npm/simplex-chat", "7.1.0-beta.6.4"))
+        self.assertIsNone(recorded_commit({}, "pypi/simplex-chat", "7.1.0b6"))
+
+    def test_adds_entries_and_keeps_recorded_commits(self):
+        updated = add_releases(self.RELEASES, [("npm/simplex-chat", "7.1.0-beta.6.4"), ("core", "7.1.0.11")], "x")
+        self.assertEqual(updated["npm"]["simplex-chat"], {"7.1.0-beta.6.3": "n1", "7.1.0-beta.6.4": "x"})
+        self.assertEqual(updated["core"], {"7.1.0.11": "c1"})
+        self.assertEqual(self.RELEASES["npm"]["simplex-chat"], {"7.1.0-beta.6.3": "n1"})
+
+
+class RecordReleasesTest(unittest.TestCase):
+    def file(self, releases, blob):
+        return {"content": base64.b64encode(render_releases(releases).encode()).decode(), "sha": blob}
+
+    def conflict(self):
+        error = urllib.error.HTTPError("url", 409, "Conflict", email.message.Message(), io.BytesIO())
+        self.addCleanup(error.close)
+        return error
+
+    def test_creates_the_file_and_returns_the_commit(self):
+        calls = []
+
+        def api(path, method="GET", body=None):
+            calls.append((path, method, body))
+            return None if method == "GET" else {"commit": {"sha": "libs1"}}
+
+        with mock.patch.object(packages_release, "libs_api", api):
+            self.assertEqual(record_releases([("core", "7.1.0.12")], "sx1"), "libs1")
+        _, method, body = calls[1]
+        self.assertEqual((method, body["message"], "sha" in body), ("PUT", "core 7.1.0.12", False))
+        self.assertEqual(tomllib.loads(base64.b64decode(body["content"]).decode()), {"core": {"7.1.0.12": "sx1"}})
+
+    def test_retries_when_another_writer_changed_the_file(self):
+        reads = iter([self.file({"core": {"1": "a"}}, "blob1"), self.file({"core": {"1": "a", "2": "b"}}, "blob2")])
+        puts = []
+
+        def api(path, method="GET", body=None):
+            if method == "GET":
+                return next(reads)
+            puts.append(body)
+            if len(puts) == 1:
+                raise self.conflict()
+            return {"commit": {"sha": "libs2"}}
+
+        with mock.patch.object(packages_release, "libs_api", api):
+            self.assertEqual(record_releases([("npm/simplex-chat", "3")], "sx3"), "libs2")
+        self.assertEqual([body["sha"] for body in puts], ["blob1", "blob2"])
+        self.assertEqual(
+            tomllib.loads(base64.b64decode(puts[1]["content"]).decode()),
+            {"core": {"1": "a", "2": "b"}, "npm": {"simplex-chat": {"3": "sx3"}}},
+        )
+
+    def test_recorded_entries_return_the_last_commit_of_the_file(self):
+        def api(path, method="GET", body=None):
+            self.assertEqual(method, "GET")
+            return [{"sha": "libs0"}] if path.startswith("commits") else self.file({"core": {"1": "a"}}, "blob1")
+
+        with mock.patch.object(packages_release, "libs_api", api):
+            self.assertEqual(record_releases([("core", "1")], "other"), "libs0")
+
+
+class PackageStateTest(unittest.TestCase):
+    def setUp(self):
+        self.repo = Path(tempfile.mkdtemp())
+        self.git("init", "-q")
+        self.write("pkg/version.txt", 'version = "1"\n')
+        self.write("pkg/src.txt", "a")
+        self.commit("set version")
+
+    def git(self, *args):
+        subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=self.repo, check=True)
+
+    def write(self, file, text):
+        (self.repo / file).parent.mkdir(parents=True, exist_ok=True)
+        (self.repo / file).write_text(text)
+
+    def commit(self, message):
+        self.git("add", "-A")
+        self.git("commit", "-q", "-m", message)
+
+    def state(self, releases=None, registry_published=False):
+        with mock.patch.object(packages_release, "ROOT", self.repo):
+            return package_state(
+                "npm/pkg", "1", "pkg", "pkg/version.txt", "^version =", releases or {}, lambda: registry_published
+            )
+
+    def changed(self):
+        return self.state().changed
+
+    def head(self):
+        return subprocess.run(
+            ["git", "rev-parse", "HEAD"], cwd=self.repo, capture_output=True, text=True
+        ).stdout.strip()
+
+    def test_recorded_version_is_published_and_compared_with_its_commit(self):
+        self.write("pkg/src.txt", "b")
+        self.commit("change published with version 1")
+        releases = {"npm": {"pkg": {"1": self.head()}}}
+        self.assertEqual(self.state(releases), Package("1", changed=False, published=True))
+        self.write("pkg/src.txt", "c")
+        self.commit("change after the publish")
+        self.assertEqual(self.state(releases), Package("1", changed=True, published=True))
+
+    def test_unrecorded_version_asks_the_registry(self):
+        self.assertFalse(self.state(registry_published=False).published)
+        self.assertTrue(self.state(registry_published=True).published)
+
+    def test_publish_plan_skips_recorded_versions_before_the_registries_list_them(self):
+        self.write(f"{TYPES_DIR}/package.json", '{"name": "@simplex-chat/types", "version": "0.1.0"}')
+        self.write(f"{NODEJS_DIR}/package.json", '{"name": "simplex-chat", "version": "1.0.0-beta.1.1"}')
+        self.write(PYTHON_VERSION_FILE, render_python_version("1.0.0b1.post1", Pins("core-1", {})))
+        self.write("pyproject.toml", "")
+        self.commit("packages")
+        releases = {"npm": {"simplex-chat": {"1.0.0-beta.1.1": "c"}}, "pypi": {"simplex-chat": {"1.0.0b1.post1": "c"}}}
+        output = io.StringIO()
+        with (
+            mock.patch.object(packages_release, "ROOT", self.repo),
+            mock.patch.object(packages_release, "read_releases", return_value=(releases, "blob")),
+            mock.patch.object(packages_release, "python_project", return_value="simplex-chat"),
+            mock.patch.object(packages_release, "npm_published", return_value=False),
+            mock.patch.object(packages_release, "pypi_published", return_value=False),
+            contextlib.redirect_stdout(output),
+        ):
+            publish_plan()
+        self.assertEqual(
+            output.getvalue().splitlines(),
+            [
+                "types_version=0.1.0",
+                "types_dist_tag=latest",
+                "nodejs_version=",
+                "nodejs_dist_tag=beta",
+                "python_version=",
+            ],
+        )
+
+    def test_unknown_recorded_commit_falls_back_to_the_version_commit(self):
+        self.write("pkg/src.txt", "b")
+        self.commit("change")
+        state = self.state({"npm": {"pkg": {"1": "0" * 40}}})
+        self.assertEqual(state, Package("1", changed=True, published=True))
+
+    def test_compares_with_the_commit_that_set_the_version(self):
+        self.assertFalse(self.changed())
+        self.write("pkg/src.txt", "b")
+        self.commit("change")
+        self.assertTrue(self.changed())
+
+    def test_new_version_resets_the_comparison(self):
+        self.write("pkg/src.txt", "b")
+        self.commit("change")
+        self.write("pkg/version.txt", 'version = "2"\n')
+        self.commit("set version 2")
+        self.assertFalse(self.changed())
+
+    def test_changed_since_looks_only_at_the_sources(self):
+        base = subprocess.run(
+            ["git", "rev-parse", "HEAD"], cwd=self.repo, capture_output=True, text=True
+        ).stdout.strip()
+        self.write("other/file.txt", "o")
+        self.commit("other change")
+        with mock.patch.object(packages_release, "ROOT", self.repo):
+            self.assertFalse(changed_since(base, ["pkg"]))
+            self.write("pkg/src.txt", "b")
+            self.commit("source change")
+            self.assertTrue(changed_since(base, ["pkg"]))
+
+    def test_ignores_tests_and_other_directories(self):
+        self.write("pkg/tests/t.txt", "t")
+        self.write("other/file.txt", "o")
+        self.commit("tests and other files")
+        self.assertFalse(self.changed())
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/desktop/build-lib-linux.sh

@@ -53,7 +53,7 @@ rm -rf apps/multiplatform/desktop/build/cmake
 mkdir -p apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
 cp -r $BUILD_DIR/build/deps/* apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
 cp $BUILD_DIR/build/libsimplex.so apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
-scripts/desktop/prepare-vlc-linux.sh
+[ -n "$SKIP_VLC" ] || scripts/desktop/prepare-vlc-linux.sh
 
 links_dir=apps/multiplatform/build/links
 mkdir -p $links_dir

scripts/desktop/build-lib-mac.sh

@@ -151,7 +151,7 @@ if [ -n "$LOCAL_DIRS" ]; then
 fi
 
 cd -
-scripts/desktop/prepare-vlc-mac.sh
+[ -n "$SKIP_VLC" ] || scripts/desktop/prepare-vlc-mac.sh
 
 links_dir=apps/multiplatform/build/links
 mkdir -p $links_dir

scripts/desktop/build-lib-windows.sh

@@ -61,7 +61,7 @@ mkdir -p apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
 cp dist-newstyle/openssl-3.0.15/libcrypto-3-x64.dll apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
 cp libsimplex.dll apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
 
-scripts/desktop/prepare-vlc-windows.sh
+[ -n "$SKIP_VLC" ] || scripts/desktop/prepare-vlc-windows.sh
 
 links_dir=apps/multiplatform/build/links
 mkdir -p $links_dir

scripts/nix/sha256map.nix

@@ -1,5 +1,5 @@
 {
-  "https://github.com/simplex-chat/simplexmq.git"."e24d537eaaa32230eec1fefcd957d474d7b8d4d5" = "1ximwqi1jaiq1ml05y0mcsrgvabhp5is1ry2h91w5jjarjb1a5fj";
+  "https://github.com/simplex-chat/simplexmq.git"."cba13823567d7cd564d8a1f354046174fba375d8" = "0nhd6zsmqs55b7shh945b0v9pn0qqkpx9djq9dldair82hsmw1if";
   "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
   "https://github.com/simplex-chat/direct-sqlcipher.git"."2330df3dc8c4b660674d3c02d76bb86f66c2abee" = "11pfj1vqsjdl26nplkr2l6z1xj7cn153qijj5gknmlzq3krn7x1i";
   "https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl";