ci: package releases with drafts in simplex-chat-libs (#7754)

* ci: package releases with drafts in simplex-chat-libs

* test release

* core: 7.1.0.12

* packages: release nodejs and python libraries

* fixes

---------

Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>

scripts/ci/package-tests.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -euo pipefail
+
+cd "$(dirname "$0")/../.."
+nodejs=packages/simplex-chat-nodejs
+python_package=packages/simplex-chat-python
+python=$(command -v python3 || command -v python)
+temp=${RUNNER_TEMP:-$(mktemp -d)}
+
+usage() {
+  echo "usage: $0 build <platform> | test | addon <platform> | publish <platform> [nodejs] [python]" >&2
+  exit 1
+}
+
+build_package() {
+  (cd packages/simplex-chat-client/types/typescript && npm install --no-audit --no-fund && npx tsc)
+  (cd "$nodejs" && npm install --ignore-scripts --no-save --no-audit --no-fund ../simplex-chat-client/types/typescript)
+  if [ -n "${1:-}" ]; then
+    (cd "$nodejs" && npx --yes node-gyp rebuild)
+  fi
+  (cd "$nodejs" && npx tsc && cp src/simplex.* dist)
+  if [ -n "${1:-}" ]; then
+    cp "$nodejs/build/Release/simplex.node" "$nodejs/build/simplex-chat-nodejs-$1.node"
+  fi
+}
+
+test_nodejs() {
+  (cd "$nodejs" && npx jest --forceExit --testNamePattern '^(?!.*use preset servers)')
+}
+
+test_python() {
+  (unset SIMPLEX_LIBS_DIR && cd "$python_package" && "$python" -m pip install -e ".[test]" && "$python" -m pytest -q)
+}
+
+case "${1:-}" in
+  build)
+    [ $# -eq 2 ] || usage
+    build_package "$2"
+    ;;
+  test)
+    [ $# -eq 1 ] || usage
+    test_nodejs
+    test_python
+    ;;
+  addon)
+    [ $# -eq 2 ] || usage
+    build_package "$2"
+    zip=simplex-chat-libs-$2.zip
+    "$python" scripts/ci/packages_release.py download "$LIBS_RELEASE" "$zip" "$temp/libs"
+    "$python" -m zipfile -e "$temp/libs/$zip" "$temp/libs"
+    SIMPLEX_LIBS_DIR=$temp/libs/libs test_nodejs
+    gh release upload "$ADDON_TAG" "$nodejs/build/simplex-chat-nodejs-$2.node" --clobber --repo "$LIBS_REPOSITORY"
+    ;;
+  publish)
+    [ $# -ge 2 ] || usage
+    platform=$2
+    shift 2
+    for suite in "$@"; do
+      case "$suite" in
+        nodejs)
+          build_package
+          addon=simplex-chat-nodejs-$platform.node
+          "$python" scripts/ci/packages_release.py download-pinned "$addon" "$temp/addon"
+          mkdir -p "$nodejs/build/Release"
+          cp "$temp/addon/$addon" "$nodejs/build/Release/simplex.node"
+          test_nodejs
+          ;;
+        python)
+          test_python
+          ;;
+        *)
+          usage
+          ;;
+      esac
+    done
+    ;;
+  *)
+    usage
+    ;;
+esac

scripts/ci/packages_release.py

@@ -1,13 +1,21 @@
 #!/usr/bin/env python3
 """Release automation for simplex-chat and @simplex-chat/types on npm and simplex-chat on PyPI.
 
-addon-tag      print the add-on release tag for the current sources
-changes        print whether the changes since a commit touch the chat core and the add-on
-build-plan     print the platforms to build the add-on on and whether to run the package tests
-prepare        update pins and versions for a release PR
-publish-plan   print the package versions that are not published yet
-record         record releases and published versions in releases.toml of simplex-chat-libs
-test-names     rename the packages to NPM_SCOPE and PYPI_PROJECT (test repositories)
+core-release      set the core version and the types version for a library release
+claim-core        claim the library release of the core version for a commit
+release-status    print whether a release of simplex-chat-libs is published, a draft or absent
+addon-release     print the add-on release of the current add-on sources
+latest-libs       print the newest published library release
+publish-release   publish a draft release that has every asset
+check-pins        check the pins of the packages against the published releases
+download          download a release asset and check its digest
+download-pinned   download a pinned asset and check its pin
+changes           print whether the changes since a commit touch the chat core and the add-on
+build-plan        print the platforms to build the add-on on and whether to run the package tests
+prepare           update pins and versions of the packages
+publish-plan      print the package versions that are not published yet
+record            record releases and published versions in releases.toml of simplex-chat-libs
+test-names        rename the packages to NPM_SCOPE and PYPI_PROJECT (test repositories)
 """
 
 from __future__ import annotations
@@ -19,12 +27,14 @@ import hashlib
 import json
 import os
 import re
+import shlex
 import subprocess
 import sys
+import tempfile
 import tomllib
 import urllib.error
 import urllib.request
-from collections.abc import Callable
+from collections.abc import Callable, Collection
 from dataclasses import dataclass, field, replace
 from typing import Any
 from pathlib import Path
@@ -37,7 +47,15 @@ RELEASE_FILE = f"{NODEJS_DIR}/src/release.ts"
 PYTHON_VERSION_FILE = f"{PYTHON_DIR}/src/simplex_chat/_version.py"
 ADDON_SOURCES = [f"{NODEJS_DIR}/binding.gyp", f"{NODEJS_DIR}/cpp"]
 CORE_SOURCES = ["src", "simplex-chat.cabal", "cabal.project"]
-ADDON_TAG_HASH_LENGTH = 12
+CABAL_FILE = "simplex-chat.cabal"
+CABAL_VERSION = re.compile(r"^(version:\s*)(\S+)", re.M)
+TYPES_SOURCES = [f"{TYPES_DIR}/src"]
+TYPES_README = f"{TYPES_DIR}/README.md"
+TYPES_README_CORE = re.compile(r"^SimpleX Chat core: .*$", re.M)
+PYTHON_TYPES_SOURCES = [f"{PYTHON_DIR}/src/simplex_chat/types"]
+ADDON_HASH_LENGTH = 12
+CORE_TAG = re.compile(r"^core-(\d+(?:\.\d+)*)$")
+ADDON_TAG_PREFIX = "nodejs-addon-"
 LIBS_ASSETS = [
     "simplex-chat-libs-linux-aarch64.zip",
     "simplex-chat-libs-linux-x86_64-postgres.zip",
@@ -58,18 +76,27 @@ ADDON_ASSETS = [f"simplex-chat-nodejs-{platform}.node" for platform in ADDON_PLA
 SHA256_HEX = re.compile(r"^[0-9a-f]{64}$")
 RELEASES_FILE = "releases.toml"
 RECORD_ATTEMPTS = 5
+DOWNLOAD_CHUNK = 1 << 20
 TOML_BARE_KEY = re.compile(r"^[A-Za-z0-9_-]+$")
 
 
 # Pure functions
 
 
-def addon_tag(files: dict[str, bytes], node_addon_api: str) -> str:
+def addon_hash(files: dict[str, bytes], node_addon_api: str) -> str:
     digest = hashlib.sha256()
     for name in sorted(files):
         digest.update(name.encode() + b"\0" + files[name] + b"\0")
     digest.update(f"node-addon-api@{node_addon_api}".encode())
-    return f"nodejs-addon-{digest.hexdigest()[:ADDON_TAG_HASH_LENGTH]}"
+    return digest.hexdigest()[:ADDON_HASH_LENGTH]
+
+
+def addon_notes(sources_hash: str) -> str:
+    return f"sources: {sources_hash}"
+
+
+def version_key(version: str) -> tuple[int, ...]:
+    return tuple(int(part) for part in version.split("."))
 
 
 def bump_npm_version(version: str) -> str:
@@ -141,7 +168,7 @@ def render_python_version(version: str, pins: Pins) -> str:
     return (
         '"""Package version and the simplex-chat-libs release it loads.\n'
         "\n"
-        "Written by scripts/ci/packages_release.py: the release PR bumps the version\n"
+        "Written by scripts/ci/packages_release.py prepare: it bumps the version\n"
         "and pins new library releases with the SHA-256 of their assets.\n"
         '"""\n'
         "\n"
@@ -152,16 +179,84 @@ def render_python_version(version: str, pins: Pins) -> str:
     )
 
 
+def asset_digests(release: dict) -> dict[str, str]:
+    return {a["name"]: (a.get("digest") or "").removeprefix("sha256:") for a in release["assets"]}
+
+
 def release_digests(release: dict | None, assets: list[str]) -> dict[str, str] | None:
     """Asset digests of a GitHub release, or None when the release is missing or incomplete."""
     if release is None:
         return None
-    digests = {a["name"]: (a.get("digest") or "").removeprefix("sha256:") for a in release["assets"]}
+    digests = asset_digests(release)
     if not all(SHA256_HEX.match(digests.get(asset, "")) for asset in assets):
         return None
     return {asset: digests[asset] for asset in assets}
 
 
+def release_assets(tag: str) -> list[str]:
+    if CORE_TAG.match(tag):
+        return LIBS_ASSETS
+    if tag.startswith(ADDON_TAG_PREFIX):
+        return ADDON_ASSETS
+    raise ValueError(f"no asset set for release {tag}")
+
+
+def preferred_release(releases: list[dict]) -> dict | None:
+    """The published release among them, otherwise the first draft."""
+    return next((r for r in releases if not r["draft"]), next(iter(releases), None))
+
+
+def release_status(release: dict | None) -> str:
+    if release is None:
+        return "absent"
+    return "draft" if release["draft"] else "published"
+
+
+def tag_release(releases: list[dict], tag: str) -> dict | None:
+    return preferred_release([r for r in releases if r["tag_name"] == tag])
+
+
+def addon_release(releases: list[dict], sources_hash: str) -> dict | None:
+    notes = addon_notes(sources_hash)
+    return preferred_release(
+        [r for r in releases if r["tag_name"].startswith(ADDON_TAG_PREFIX) and r.get("body") == notes]
+    )
+
+
+def core_version(tag: str) -> str | None:
+    match = CORE_TAG.match(tag)
+    return match[1] if match else None
+
+
+def latest_libs(releases: list[dict], versions: Collection[str]) -> str | None:
+    """The newest published release of the core versions that has every library asset."""
+    tags = [
+        r["tag_name"]
+        for r in releases
+        if not r["draft"] and core_version(r["tag_name"]) in versions and release_digests(r, LIBS_ASSETS)
+    ]
+    return max(tags, key=lambda tag: version_key(core_version(tag)), default=None)
+
+
+def missing_assets(release: dict) -> list[str]:
+    present = {a["name"] for a in release["assets"]}
+    return [asset for asset in release_assets(release["tag_name"]) if asset not in present]
+
+
+def pin_problems(pins: Pins, published: dict[str, dict | None]) -> list[str]:
+    """Differences between the pins and the published releases of the pinned tags."""
+    problems = []
+    for tag, release in published.items():
+        if release is None:
+            problems.append(f"{tag} is not published")
+            continue
+        digests = asset_digests(release)
+        for asset in release_assets(tag):
+            if pins.sha256.get(asset) != digests.get(asset):
+                problems.append(f"{asset} of {tag}: pinned {pins.sha256.get(asset)}, published {digests.get(asset)}")
+    return problems
+
+
 @dataclass(frozen=True)
 class Package:
     version: str
@@ -177,10 +272,11 @@ class State:
     addon_digests: dict[str, str] | None
     nodejs_pins: Pins
     python_pins: Pins
-    types: Package
+    types_dependency: str
+    nodejs_types_dependency: str
     nodejs: Package
     python: Package
-    types_version: str | None = None
+    packages: frozenset[str] = frozenset({"nodejs", "python"})
     nodejs_version: str | None = None
     python_version: str | None = None
 
@@ -189,23 +285,16 @@ class State:
 class Plan:
     nodejs_pins: Pins
     python_pins: Pins
-    types_version: str
-    types_bumped: bool
+    types_dependency: str | None
     nodejs_version: str
     nodejs_changed: bool
     python_version: str
     python_changed: bool
     notes: list[str] = field(default_factory=list)
 
-    @property
-    def types_dependency(self) -> str | None:
-        return f"^{self.types_version}" if self.types_bumped else None
-
     @property
     def title(self) -> str:
         parts = []
-        if self.types_bumped:
-            parts.append(f"types {self.types_version}")
         if self.nodejs_changed:
             parts.append(f"nodejs {self.nodejs_version}")
         if self.python_changed:
@@ -214,7 +303,7 @@ class Plan:
 
 
 def plan(state: State) -> Plan:
-    """What a release PR changes, from the state of the branch and the releases."""
+    """What a package release changes, from the state of the branch and the releases."""
     notes: list[str] = []
 
     def pin(pins: Pins, field_name: str, tag: str, digests: dict[str, str] | None, what: str) -> tuple[Pins, bool]:
@@ -227,27 +316,30 @@ def plan(state: State) -> Plan:
         notes.append(f"{what}: {tag}")
         return replace(pins, **{field_name: tag, "sha256": {**pins.sha256, **digests}}), True
 
-    nodejs_pins, nodejs_libs = pin(
-        state.nodejs_pins, "libs_release", state.libs_tag, state.libs_digests, "Node.js libsimplex"
-    )
-    nodejs_pins, addon = pin(nodejs_pins, "addon_release", state.addon_tag, state.addon_digests, "Node.js add-on")
-    python_pins, python_libs = pin(
-        state.python_pins, "libs_release", state.libs_tag, state.libs_digests, "Python libsimplex"
-    )
-
     def next_version(pkg: Package, changed: bool, override: str | None, bump) -> str:
         if override:
             return override
         return bump(pkg.version) if changed and pkg.published else pkg.version
 
-    types_version = next_version(state.types, state.types.changed, state.types_version, bump_npm_version)
-    types_bumped = types_version != state.types.version
-    nodejs_changed = nodejs_libs or addon or types_bumped or state.nodejs.changed
-    nodejs_version = next_version(state.nodejs, nodejs_changed, state.nodejs_version, bump_npm_version)
-    python_changed = python_libs or state.python.changed
-    python_version = next_version(state.python, python_changed, state.python_version, bump_python_version)
+    nodejs_pins, nodejs_version, nodejs_changed, types_dependency = state.nodejs_pins, state.nodejs.version, False, None
+    if "nodejs" in state.packages:
+        nodejs_pins, nodejs_libs = pin(
+            state.nodejs_pins, "libs_release", state.libs_tag, state.libs_digests, "Node.js libsimplex"
+        )
+        nodejs_pins, addon = pin(nodejs_pins, "addon_release", state.addon_tag, state.addon_digests, "Node.js add-on")
+        if state.types_dependency != state.nodejs_types_dependency:
+            types_dependency = state.types_dependency
+            notes.append(f"@simplex-chat/types: {types_dependency}")
+        nodejs_changed = nodejs_libs or addon or types_dependency is not None or state.nodejs.changed
+        nodejs_version = next_version(state.nodejs, nodejs_changed, state.nodejs_version, bump_npm_version)
+    python_pins, python_version, python_changed = state.python_pins, state.python.version, False
+    if "python" in state.packages:
+        python_pins, python_libs = pin(
+            state.python_pins, "libs_release", state.libs_tag, state.libs_digests, "Python libsimplex"
+        )
+        python_changed = python_libs or state.python.changed
+        python_version = next_version(state.python, python_changed, state.python_version, bump_python_version)
     for name, old, new in [
-        ("@simplex-chat/types", state.types.version, types_version),
         ("simplex-chat (npm)", state.nodejs.version, nodejs_version),
         ("simplex-chat (PyPI)", state.python.version, python_version),
     ]:
@@ -256,8 +348,7 @@ def plan(state: State) -> Plan:
     return Plan(
         nodejs_pins=nodejs_pins,
         python_pins=python_pins,
-        types_version=types_version,
-        types_bumped=types_bumped,
+        types_dependency=types_dependency,
         nodejs_version=nodejs_version,
         nodejs_changed=nodejs_changed or nodejs_version != state.nodejs.version,
         python_version=python_version,
@@ -272,31 +363,49 @@ class BuildPlan:
     test: bool
 
 
-def build_plan(
-    event: str, release_missing: bool, libs_release: str, core_changed: bool, addon_changed: bool
-) -> BuildPlan:
-    """Each package test runs once, against the libsimplex of the change: a pull request that changes the core
-    is tested by build.yml with its own libsimplex, a new library release with that release."""
-    if libs_release:
-        return BuildPlan(ADDON_PLATFORMS, test=True)
-    if event == "pull_request":
-        if core_changed:
-            return BuildPlan(ADDON_PLATFORMS if addon_changed else [], test=False)
-        return BuildPlan(ADDON_PLATFORMS, test=True)
-    return BuildPlan(ADDON_PLATFORMS if release_missing else [], test=True)
+def build_plan(core_changed: bool, addon_changed: bool) -> BuildPlan:
+    """The package tests of a pull request that changes the core run in build.yml, with its own libsimplex."""
+    if core_changed:
+        return BuildPlan(ADDON_PLATFORMS if addon_changed else [], test=False)
+    return BuildPlan(ADDON_PLATFORMS, test=True)
 
 
 def build_matrix(platforms: list[str]) -> str:
     return json.dumps([{"platform": platform, "os": ADDON_RUNNERS[platform]} for platform in platforms])
 
 
-def recorded_commit(releases: dict, key: str, version: str) -> str | None:
-    """The simplex-chat commit recorded for a version; key is "core", "npm/<name>" or "pypi/<name>"."""
+def core_key(branch: str) -> str:
+    return f"core/{branch}"
+
+
+def recorded_versions(releases: dict, key: str) -> dict[str, str]:
+    """The recorded versions and their simplex-chat commits; key is "core/<branch>", "npm/<name>" or "pypi/<name>"."""
     table = releases
     for part in key.split("/", 1):
         table = table.get(part, {})
-    commit = table.get(version)
-    return commit if isinstance(commit, str) else None
+    return {version: commit for version, commit in table.items() if isinstance(commit, str)}
+
+
+def recorded_commit(releases: dict, key: str, version: str) -> str | None:
+    return recorded_versions(releases, key).get(version)
+
+
+def highest_recorded(releases: dict, key: str) -> str | None:
+    return max(recorded_versions(releases, key), key=version_key, default=None)
+
+
+def recorded_core_commit(releases: dict, version: str) -> str | None:
+    """The commit recorded for the core version on any branch: a release tag is shared by the branches."""
+    for branch in releases.get("core", {}):
+        if commit := recorded_commit(releases, core_key(branch), version):
+            return commit
+    return None
+
+
+def previous_core_commit(releases: dict, branch: str) -> str | None:
+    """The commit of the highest core version recorded for the branch."""
+    previous = highest_recorded(releases, core_key(branch))
+    return recorded_commit(releases, core_key(branch), previous) if previous else None
 
 
 def add_releases(releases: dict, entries: list[tuple[str, str]], commit: str) -> dict:
@@ -357,14 +466,18 @@ def write_json(file: str, value: dict) -> None:
     (ROOT / file).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n")
 
 
-def current_addon_tag() -> str:
+def current_addon_hash() -> str:
     files = {
         path.relative_to(ROOT).as_posix(): path.read_bytes()
         for source in ADDON_SOURCES
         for path in ([ROOT / source] if (ROOT / source).is_file() else sorted((ROOT / source).rglob("*")))
         if path.is_file()
     }
-    return addon_tag(files, read_json(f"{NODEJS_DIR}/package.json")["devDependencies"]["node-addon-api"])
+    return addon_hash(files, read_json(f"{NODEJS_DIR}/package.json")["devDependencies"]["node-addon-api"])
+
+
+def cabal_version() -> str:
+    return CABAL_VERSION.search(read_text(CABAL_FILE))[2]
 
 
 def changed_since(base: str, sources: list[str]) -> bool:
@@ -378,13 +491,9 @@ def changes(base: str) -> None:
 
 
 def print_build_plan(args: argparse.Namespace) -> None:
-    pull_request = args.event == "pull_request"
     result = build_plan(
-        args.event,
-        release_missing=args.release_missing,
-        libs_release=args.libs_release,
-        core_changed=pull_request and changed_since(args.base, CORE_SOURCES),
-        addon_changed=pull_request and changed_since(args.base, ADDON_SOURCES),
+        core_changed=changed_since(args.base, CORE_SOURCES),
+        addon_changed=changed_since(args.base, ADDON_SOURCES),
     )
     print(f"platforms={build_matrix(result.platforms)}")
     print(f"test={'true' if result.test else ''}")
@@ -429,10 +538,13 @@ def http_json(url: str, headers: dict[str, str] | None = None, method: str = "GE
 
 
 # Workflows pass unset repository variables as empty strings.
-def npm_published(name: str, version: str) -> bool:
+def npm_package_url(name: str) -> str:
     registry = os.environ.get("NPM_REGISTRY") or "https://registry.npmjs.org"
-    package = registry_name(name, os.environ.get("NPM_SCOPE")).replace("/", "%2F")
-    return http_json(f"{registry}/{package}/{version}") is not None
+    return f"{registry}/{registry_name(name, os.environ.get('NPM_SCOPE')).replace('/', '%2F')}"
+
+
+def npm_published(name: str, version: str) -> bool:
+    return http_json(f"{npm_package_url(name)}/{version}") is not None
 
 
 def python_project() -> str:
@@ -445,19 +557,50 @@ def pypi_published(version: str) -> bool:
     return http_json(f"{index}/pypi/{project}/{version}/json") is not None
 
 
+def libs_repository() -> str:
+    return os.environ.get("LIBS_REPOSITORY") or "simplex-chat/simplex-chat-libs"
+
+
 def libs_api(path: str, method: str = "GET", body: dict | None = None) -> Any:
     """GitHub API of the simplex-chat-libs repository."""
-    repo = os.environ.get("LIBS_REPOSITORY") or "simplex-chat/simplex-chat-libs"
     headers = {"Accept": "application/vnd.github+json"}
     if token := os.environ.get("GH_TOKEN"):
         headers["Authorization"] = f"Bearer {token}"
-    return http_json(f"https://api.github.com/repos/{repo}/{path}", headers, method, body)
+    return http_json(f"https://api.github.com/repos/{libs_repository()}/{path}", headers, method, body)
 
 
 def github_release(tag: str) -> dict | None:
+    """The published release of the tag."""
     return libs_api(f"releases/tags/{tag}")
 
 
+def libs_releases() -> list[dict]:
+    """The releases of simplex-chat-libs; drafts are listed only with a token that can push to it."""
+    releases: list[dict] = []
+    page = 1
+    while batch := libs_api(f"releases?per_page=100&page={page}"):
+        releases += batch
+        page += 1
+    return releases
+
+
+def download(tag: str, asset: str, directory: str, sha256: str) -> Path:
+    """Saves the asset of the release to the directory; exits when its SHA-256 differs."""
+    url = f"https://github.com/{libs_repository()}/releases/download/{tag}/{asset}"
+    target = Path(directory) / asset
+    target.parent.mkdir(parents=True, exist_ok=True)
+    digest = hashlib.sha256()
+    request = urllib.request.Request(url, headers={"User-Agent": "simplex-chat-release"})
+    with urllib.request.urlopen(request, timeout=600) as response, target.open("wb") as file:
+        while chunk := response.read(DOWNLOAD_CHUNK):
+            file.write(chunk)
+            digest.update(chunk)
+    if digest.hexdigest() != sha256:
+        target.unlink()
+        sys.exit(f"SHA-256 of {asset} of {tag} is {digest.hexdigest()}, expected {sha256}")
+    return target
+
+
 def read_releases() -> tuple[dict, str | None]:
     """The releases and the blob of the file, which a change must name."""
     file = libs_api(f"contents/{RELEASES_FILE}")
@@ -495,6 +638,13 @@ def commit_exists(commit: str) -> bool:
     return found.returncode == 0
 
 
+def fetch_commit(commit: str) -> str:
+    """The commit, fetched from origin when the checkout lacks it."""
+    if not commit_exists(commit):
+        git("fetch", "-q", "--depth=1", "origin", commit)
+    return commit
+
+
 def npm_key(directory: str) -> str:
     return f"npm/{read_json(f'{directory}/package.json')['name']}"
 
@@ -517,21 +667,113 @@ def npm_package(directory: str, releases: dict) -> Package:
     )
 
 
-def prepare(args: argparse.Namespace) -> None:
-    cabal_version = read_field("simplex-chat.cabal", r"^version:\s*(\S+)")
-    libs_tag = f"core-{cabal_version}"
-    addon = current_addon_tag()
-    python_version, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE))
+def types_changed(releases: dict, branch: str) -> bool:
+    """Whether the types sources differ from those of the previous core release on the branch."""
+    previous = previous_core_commit(releases, branch)
+    return previous is None or changed_since(fetch_commit(previous), TYPES_SOURCES)
+
+
+def types_problem(version: str, releases: dict, branch: str) -> str | None:
+    """Why the types can't be released with core <version>, or None when they can."""
+    if not types_changed(releases, branch):
+        return None
+    types = read_json(f"{TYPES_DIR}/package.json")
+    if recorded_commit(releases, npm_key(TYPES_DIR), types["version"]) or npm_published(types["name"], types["version"]):
+        return f"{types['name']} changed, and its version {types['version']} is published"
+    if types.get("simplexChatCore") != version:
+        return f"simplexChatCore of {types['name']} is {types.get('simplexChatCore')}, expected {version}"
+    return None
+
+
+def claim_core(version: str, commit: str, branch: str) -> None:
     releases, _ = read_releases()
+    highest = highest_recorded(releases, core_key(branch))
+    if highest and version_key(version) < version_key(highest):
+        sys.exit(f"core {version} is lower than core {highest} recorded for {branch}")
+    if recorded := recorded_core_commit(releases, version):
+        print(f"claim={'resume' if recorded == commit else 'skip'}")
+        return
+    if problem := types_problem(version, releases, branch):
+        sys.exit(problem)
+    libs_commit = record_releases([(core_key(branch), version)], commit)
+    releases, _ = read_releases()
+    if recorded_core_commit(releases, version) == commit:
+        print("claim=new")
+        print(f"commit={libs_commit}")
+    else:
+        print("claim=skip")
+
+
+def current_branch() -> str:
+    return git("rev-parse", "--abbrev-ref", "HEAD")
+
+
+def core_release(version: str, branch: str, types_version: str | None) -> None:
+    releases, _ = read_releases()
+    highest = highest_recorded(releases, core_key(branch))
+    if highest and version_key(version) <= version_key(highest):
+        sys.exit(f"core {version} is not higher than core {highest} recorded for {branch}")
+    (ROOT / CABAL_FILE).write_text(CABAL_VERSION.sub(rf"\g<1>{version}", read_text(CABAL_FILE), count=1))
+    print(f"{CABAL_FILE}: {version}")
+    if not types_version and not types_changed(releases, branch):
+        return
+    types = read_json(f"{TYPES_DIR}/package.json")
+    if not types_version:
+        previous = previous_core_commit(releases, branch)
+        released = json.loads(git("show", f"{previous}:{TYPES_DIR}/package.json")) if previous else types
+        types_version = bump_npm_version(released["version"])
+    readme, found = TYPES_README_CORE.subn(f"SimpleX Chat core: {version}", read_text(TYPES_README), count=1)
+    if not found:
+        sys.exit(f"{TYPES_README} has no line 'SimpleX Chat core: <version>'")
+    write_json(f"{TYPES_DIR}/package.json", {**types, "version": types_version, "simplexChatCore": version})
+    (ROOT / TYPES_README).write_text(readme)
+    print(f"{types['name']}: {types_version}")
+
+
+def pinned_tags(pins: Pins) -> list[str]:
+    return [tag for tag in (pins.libs_release, pins.addon_release) if tag]
+
+
+def published_pin_problems(pins: Pins) -> list[str]:
+    return pin_problems(pins, {tag: github_release(tag) for tag in pinned_tags(pins)})
+
+
+def prepare(args: argparse.Namespace) -> None:
+    packages = frozenset(args.packages or ["nodejs", "python"])
+    branch = args.branch or current_branch()
+    releases, _ = read_releases()
+    libs = libs_releases()
+    libs_tag = latest_libs(libs, recorded_versions(releases, core_key(branch)))
+    if libs_tag is None:
+        sys.exit(f"no published library release recorded for {branch}")
+    core = core_version(libs_tag)
+    if version_key(cabal_version()) > version_key(core):
+        print(f"warning: core-{cabal_version()} is not published, {libs_tag} is pinned", file=sys.stderr)
+    sources_hash = current_addon_hash()
+    addon = addon_release(libs, sources_hash)
+    addon_digests = release_digests(addon, ADDON_ASSETS) if release_status(addon) == "published" else None
+    if "nodejs" in packages and addon_digests is None:
+        sys.exit(f"the add-on release of sources {sources_hash} is not published with every asset")
+    types = read_json(f"{TYPES_DIR}/package.json")
+    types_name = types["name"]
+    if "nodejs" in packages and not npm_published(types_name, types["version"]):
+        sys.exit(f"{types_name} {types['version']} is not published")
+    if "python" in packages:
+        core_commit = recorded_commit(releases, core_key(branch), core)
+        if changed_since(core_commit, PYTHON_TYPES_SOURCES):
+            sys.exit(f"the Python types differ from those of {libs_tag}")
+    nodejs = read_json(f"{NODEJS_DIR}/package.json")
+    python_version, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE))
     result = plan(
         State(
             libs_tag=libs_tag,
-            libs_digests=release_digests(github_release(libs_tag), LIBS_ASSETS),
-            addon_tag=addon,
-            addon_digests=release_digests(github_release(addon), ADDON_ASSETS),
+            libs_digests=release_digests(tag_release(libs, libs_tag), LIBS_ASSETS),
+            addon_tag=addon["tag_name"] if addon else "",
+            addon_digests=addon_digests,
             nodejs_pins=parse_release(read_text(RELEASE_FILE)),
             python_pins=python_pins,
-            types=npm_package(TYPES_DIR, releases),
+            types_dependency=f"^{types['version']}",
+            nodejs_types_dependency=nodejs["dependencies"][types_name],
             nodejs=npm_package(NODEJS_DIR, releases),
             python=package_state(
                 pypi_key(),
@@ -542,24 +784,35 @@ def prepare(args: argparse.Namespace) -> None:
                 releases,
                 lambda: pypi_published(python_version),
             ),
-            types_version=args.types_version,
+            packages=packages,
             nodejs_version=args.nodejs_version,
             python_version=args.python_version,
         )
     )
-    (ROOT / RELEASE_FILE).write_text(render_release(result.nodejs_pins))
-    (ROOT / PYTHON_VERSION_FILE).write_text(render_python_version(result.python_version, result.python_pins))
-    types = read_json(f"{TYPES_DIR}/package.json")
-    write_json(f"{TYPES_DIR}/package.json", {**types, "version": result.types_version})
-    nodejs = read_json(f"{NODEJS_DIR}/package.json")
-    dependencies = nodejs["dependencies"]
-    if result.types_dependency:
-        dependencies = {**dependencies, "@simplex-chat/types": result.types_dependency}
-    write_json(f"{NODEJS_DIR}/package.json", {**nodejs, "version": result.nodejs_version, "dependencies": dependencies})
-    summary = result.title + "\n\n" + "".join(f"- {note}\n" for note in result.notes)
-    if args.summary:
-        Path(args.summary).write_text(summary)
-    print(summary, end="")
+    selected_pins = [
+        pins for name, pins in [("nodejs", result.nodejs_pins), ("python", result.python_pins)] if name in packages
+    ]
+    if problems := [problem for pins in selected_pins for problem in published_pin_problems(pins)]:
+        sys.exit("\n".join(problems))
+    assets = {
+        (tag, asset, pins.sha256[asset])
+        for pins in selected_pins
+        for tag in pinned_tags(pins)
+        for asset in release_assets(tag)
+    }
+    with tempfile.TemporaryDirectory() as directory:
+        for tag, asset, sha256 in sorted(assets):
+            download(tag, asset, directory, sha256)
+            print(f"checked {asset} of {tag}", file=sys.stderr)
+    if "nodejs" in packages:
+        (ROOT / RELEASE_FILE).write_text(render_release(result.nodejs_pins))
+        dependencies = nodejs["dependencies"]
+        if result.types_dependency:
+            dependencies = {**dependencies, types_name: result.types_dependency}
+        write_json(f"{NODEJS_DIR}/package.json", {**nodejs, "version": result.nodejs_version, "dependencies": dependencies})
+    if "python" in packages:
+        (ROOT / PYTHON_VERSION_FILE).write_text(render_python_version(result.python_version, result.python_pins))
+    print(result.title + "\n\n" + "".join(f"- {note}\n" for note in result.notes), end="")
 
 
 def publish_plan() -> None:
@@ -579,7 +832,7 @@ def publish_plan() -> None:
 
 def record(args: argparse.Namespace) -> None:
     versions = [
-        ("core", args.core),
+        (core_key(args.branch), args.core),
         (npm_key(TYPES_DIR), args.types),
         (npm_key(NODEJS_DIR), args.nodejs),
         (pypi_key(), args.python),
@@ -617,35 +870,118 @@ def test_names() -> None:
         )
 
 
-def github_bool(value: str) -> bool:
-    """GitHub expressions pass booleans to the shell as "true", "false" or ""."""
-    return value == "true"
+def print_release_status(tag: str) -> None:
+    print(f"status={release_status(tag_release(libs_releases(), tag))}")
+
+
+def print_addon_release() -> None:
+    sources_hash = current_addon_hash()
+    release = addon_release(libs_releases(), sources_hash)
+    print(f"notes={shlex.quote(addon_notes(sources_hash))}")
+    print(f"status={release_status(release)}")
+    print(f"tag={release['tag_name'] if release else ''}")
+
+
+def publish_release(tag: str) -> None:
+    release = tag_release(libs_releases(), tag)
+    if release is None:
+        sys.exit(f"{tag}: no release")
+    if not release["draft"]:
+        print(f"{tag} is published")
+        return
+    if missing := missing_assets(release):
+        sys.exit(f"{tag}: missing {', '.join(missing)}")
+    libs_api(f"releases/{release['id']}", "PATCH", {"draft": False})
+    print(f"{tag} published")
+
+
+def check_pins() -> None:
+    _, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE))
+    problems = published_pin_problems(parse_release(read_text(RELEASE_FILE))) + published_pin_problems(python_pins)
+    if problems:
+        sys.exit("\n".join(problems))
+    print("the pins match the published releases")
+
+
+def download_asset(tag: str, asset: str, directory: str) -> None:
+    digests = release_digests(github_release(tag), [asset])
+    if digests is None:
+        sys.exit(f"{asset} of {tag} is not published with a digest")
+    download(tag, asset, directory, digests[asset])
+
+
+def download_pinned(asset: str, directory: str) -> None:
+    pins = parse_release(read_text(RELEASE_FILE))
+    tag = next((tag for tag in pinned_tags(pins) if asset in release_assets(tag)), None)
+    if tag is None or asset not in pins.sha256:
+        sys.exit(f"{asset} is not pinned in {RELEASE_FILE}")
+    download(tag, asset, directory, pins.sha256[asset])
 
 
 def main(argv: list[str]) -> None:
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     commands = parser.add_subparsers(dest="command", required=True)
-    commands.add_parser("addon-tag")
+    branch_help = "the branch of the core versions, the checked out branch by default"
+    core_release_parser = commands.add_parser("core-release")
+    core_release_parser.add_argument("version", help="the new core version")
+    core_release_parser.add_argument("--branch", help=branch_help)
+    core_release_parser.add_argument("--types-version", help="this types version instead of a bump when needed")
+    claim_parser = commands.add_parser("claim-core")
+    claim_parser.add_argument("version", help="the core version of simplex-chat.cabal")
+    claim_parser.add_argument("commit", help="the simplex-chat commit to release it from")
+    claim_parser.add_argument("branch", help="the branch of the commit")
+    status_parser = commands.add_parser("release-status")
+    status_parser.add_argument("tag")
+    commands.add_parser("addon-release")
+    latest_libs_parser = commands.add_parser("latest-libs")
+    latest_libs_parser.add_argument("--branch", help=branch_help)
+    publish_release_parser = commands.add_parser("publish-release")
+    publish_release_parser.add_argument("tag")
+    commands.add_parser("check-pins")
+    download_parser = commands.add_parser("download")
+    download_parser.add_argument("tag")
+    download_parser.add_argument("asset")
+    download_parser.add_argument("directory")
+    download_pinned_parser = commands.add_parser("download-pinned")
+    download_pinned_parser.add_argument("asset")
+    download_pinned_parser.add_argument("directory")
     changes_parser = commands.add_parser("changes")
     changes_parser.add_argument("base", help="the commit to compare with")
     build_parser = commands.add_parser("build-plan")
-    build_parser.add_argument("--event", required=True, help="github.event_name")
-    build_parser.add_argument("--base", default="", help="the base commit of a pull request")
-    build_parser.add_argument("--release-missing", type=github_bool, default=False)
-    build_parser.add_argument("--libs-release", default="")
+    build_parser.add_argument("--base", required=True, help="the base commit of the pull request")
     prepare_parser = commands.add_parser("prepare")
-    for package in ["nodejs", "types", "python"]:
+    prepare_parser.add_argument("packages", nargs="*", choices=["nodejs", "python"], help="all when none is listed")
+    prepare_parser.add_argument("--branch", help=branch_help)
+    for package in ["nodejs", "python"]:
         prepare_parser.add_argument(f"--{package}-version", help="this version instead of a bump when needed")
-    prepare_parser.add_argument("--summary", help="file for the PR title and body")
     commands.add_parser("publish-plan")
     record_parser = commands.add_parser("record")
     record_parser.add_argument("--commit", required=True, help="the simplex-chat commit released or published from")
+    record_parser.add_argument("--branch", default="master", help="the branch of the core version")
     for name in ["core", "types", "nodejs", "python"]:
         record_parser.add_argument(f"--{name}", default="", help=f"{name} version, recorded when not empty")
     commands.add_parser("test-names")
     args = parser.parse_args(argv)
-    if args.command == "addon-tag":
-        print(current_addon_tag())
+    if args.command == "core-release":
+        core_release(args.version, args.branch or current_branch(), args.types_version)
+    elif args.command == "claim-core":
+        claim_core(args.version, args.commit, args.branch)
+    elif args.command == "release-status":
+        print_release_status(args.tag)
+    elif args.command == "addon-release":
+        print_addon_release()
+    elif args.command == "latest-libs":
+        releases, _ = read_releases()
+        versions = recorded_versions(releases, core_key(args.branch or current_branch()))
+        print(f"tag={latest_libs(libs_releases(), versions) or ''}")
+    elif args.command == "publish-release":
+        publish_release(args.tag)
+    elif args.command == "check-pins":
+        check_pins()
+    elif args.command == "download":
+        download_asset(args.tag, args.asset, args.directory)
+    elif args.command == "download-pinned":
+        download_pinned(args.asset, args.directory)
     elif args.command == "changes":
         changes(args.base)
     elif args.command == "build-plan":

scripts/ci/test_packages_release.py

@@ -1,7 +1,10 @@
+import argparse
 import base64
 import contextlib
 import email.message
+import hashlib
 import io
+import json
 import subprocess
 import tomllib
 import urllib.error
@@ -14,26 +17,41 @@ from unittest import mock
 import packages_release
 from packages_release import (
     ADDON_ASSETS,
+    CABAL_FILE,
     NODEJS_DIR,
+    PYTHON_DIR,
     PYTHON_VERSION_FILE,
+    RELEASE_FILE,
     TYPES_DIR,
+    TYPES_README,
     ADDON_PLATFORMS,
     BuildPlan,
     LIBS_ASSETS,
     Package,
     Pins,
     State,
-    addon_tag,
+    addon_hash,
+    addon_notes,
+    addon_release,
     build_matrix,
     build_plan,
     bump_npm_version,
     bump_python_version,
     changed_since,
     add_releases,
+    claim_core,
+    core_release,
+    download,
+    highest_recorded,
+    latest_libs,
+    missing_assets,
     package_state,
+    pin_problems,
+    prepare,
     publish_plan,
     record_releases,
     recorded_commit,
+    release_status,
     render_releases,
     dist_tag,
     npm_published,
@@ -45,6 +63,8 @@ from packages_release import (
     release_digests,
     render_python_version,
     render_release,
+    tag_release,
+    types_problem,
 )
 
 
@@ -60,6 +80,10 @@ def github_release(assets: list[str], n: int) -> dict:
     return {"assets": [{"name": name, "digest": f"sha256:{sha(n)}"} for name in assets]}
 
 
+def listed_release(tag: str, assets: list[str], n: int, draft: bool = False, body: str | None = None) -> dict:
+    return {"id": n, "tag_name": tag, "draft": draft, "body": body, **github_release(assets, n)}
+
+
 def state(**overrides) -> State:
     current = State(
         libs_tag="core-1.0.0.1",
@@ -68,7 +92,8 @@ def state(**overrides) -> State:
         addon_digests=digests(ADDON_ASSETS, 2),
         nodejs_pins=Pins("core-1.0.0.1", {**digests(LIBS_ASSETS, 1), **digests(ADDON_ASSETS, 2)}, "nodejs-addon-aaa"),
         python_pins=Pins("core-1.0.0.1", digests(LIBS_ASSETS, 1)),
-        types=Package("0.12.1", changed=False, published=True),
+        types_dependency="^0.12.1",
+        nodejs_types_dependency="^0.12.1",
         nodejs=Package("7.1.0-beta.6.1", changed=False, published=True),
         python=Package("7.1.0b6", changed=False, published=True),
     )
@@ -101,15 +126,15 @@ class VersionTest(unittest.TestCase):
         self.assertEqual(registry_name("simplex-chat", None), "simplex-chat")
 
 
-class AddonTagTest(unittest.TestCase):
+class AddonHashTest(unittest.TestCase):
     def test_depends_on_sources_and_node_addon_api(self):
         files = {"a.cc": b"1", "b.h": b"2"}
-        tag = addon_tag(files, "^8.5.0")
-        self.assertRegex(tag, r"^nodejs-addon-[0-9a-f]{12}$")
-        self.assertEqual(addon_tag(dict(reversed(files.items())), "^8.5.0"), tag)
-        self.assertNotEqual(addon_tag({**files, "b.h": b"3"}, "^8.5.0"), tag)
-        self.assertNotEqual(addon_tag({"a.cc": b"1", "c.h": b"2"}, "^8.5.0"), tag)
-        self.assertNotEqual(addon_tag(files, "^8.6.0"), tag)
+        sources_hash = addon_hash(files, "^8.5.0")
+        self.assertRegex(sources_hash, r"^[0-9a-f]{12}$")
+        self.assertEqual(addon_hash(dict(reversed(files.items())), "^8.5.0"), sources_hash)
+        self.assertNotEqual(addon_hash({**files, "b.h": b"3"}, "^8.5.0"), sources_hash)
+        self.assertNotEqual(addon_hash({"a.cc": b"1", "c.h": b"2"}, "^8.5.0"), sources_hash)
+        self.assertNotEqual(addon_hash(files, "^8.6.0"), sources_hash)
 
 
 class PinFilesTest(unittest.TestCase):
@@ -135,7 +160,7 @@ class ReleaseDigestsTest(unittest.TestCase):
 class PlanTest(unittest.TestCase):
     def test_nothing_changes_when_current(self):
         result = plan(state())
-        self.assertEqual((result.nodejs_changed, result.python_changed, result.types_bumped), (False, False, False))
+        self.assertEqual((result.nodejs_changed, result.python_changed, result.types_dependency), (False, False, None))
         self.assertEqual(result.notes, [])
 
     def test_new_libs_release_updates_both_libraries(self):
@@ -170,17 +195,17 @@ class PlanTest(unittest.TestCase):
             plan(state(python=Package("7.1.0b6", True, True))).title, "packages: release python 7.1.0b6.post1"
         )
 
-    def test_types_bump_updates_the_nodejs_dependency(self):
-        result = plan(state(types=Package("0.12.1", True, True)))
+    def test_new_types_version_updates_the_nodejs_dependency(self):
+        result = plan(state(types_dependency="^0.12.2"))
         self.assertEqual(result.types_dependency, "^0.12.2")
-        self.assertEqual(result.title, "packages: release types 0.12.2, nodejs 7.1.0-beta.6.2")
+        self.assertEqual(result.title, "packages: release nodejs 7.1.0-beta.6.2")
 
     def test_all_changes_combine_into_one_release(self):
         result = plan(
             state(
                 **NEW_LIBS,
                 **NEW_ADDON,
-                types=Package("0.12.1", True, True),
+                types_dependency="^0.12.2",
                 nodejs=Package("7.1.0-beta.6.1", True, True),
                 python=Package("7.1.0b6", True, True),
             )
@@ -188,7 +213,16 @@ class PlanTest(unittest.TestCase):
         self.assertEqual(
             (result.nodejs_pins.libs_release, result.nodejs_pins.addon_release), ("core-1.0.0.2", "nodejs-addon-bbb")
         )
-        self.assertEqual(result.title, "packages: release types 0.12.2, nodejs 7.1.0-beta.6.2, python 7.1.0b6.post1")
+        self.assertEqual(result.title, "packages: release nodejs 7.1.0-beta.6.2, python 7.1.0b6.post1")
+
+    def test_unlisted_package_keeps_its_pins_and_version(self):
+        result = plan(state(**NEW_LIBS, types_dependency="^0.12.2", packages=frozenset({"python"})))
+        self.assertEqual(result.nodejs_pins, state().nodejs_pins)
+        self.assertEqual(
+            (result.nodejs_version, result.nodejs_changed, result.types_dependency), ("7.1.0-beta.6.1", False, None)
+        )
+        self.assertEqual(result.python_pins.libs_release, "core-1.0.0.2")
+        self.assertEqual(result.title, "packages: release python 7.1.0b6.post1")
 
     def test_unpublished_versions_are_not_bumped_again(self):
         result = plan(
@@ -199,29 +233,18 @@ class PlanTest(unittest.TestCase):
         self.assertEqual(result.title, "packages: release nodejs 7.1.0-beta.6.2, python 7.1.0b6.post1")
 
     def test_requested_versions_win(self):
-        result = plan(state(nodejs_version="7.2.0", types_version="0.13.0", python_version="7.2.0"))
-        self.assertEqual(result.types_dependency, "^0.13.0")
-        self.assertEqual(result.title, "packages: release types 0.13.0, nodejs 7.2.0, python 7.2.0")
+        result = plan(state(nodejs_version="7.2.0", python_version="7.2.0"))
+        self.assertEqual(result.title, "packages: release nodejs 7.2.0, python 7.2.0")
 
 
 class BuildPlanTest(unittest.TestCase):
-    def plan(self, event="push", release_missing=False, libs_release="", core=False, addon=False):
-        return build_plan(event, release_missing, libs_release, core, addon)
-
-    def test_push_builds_a_missing_addon_release_and_tests_it(self):
-        self.assertEqual(self.plan(release_missing=True), BuildPlan(ADDON_PLATFORMS, test=True))
-        self.assertEqual(self.plan(), BuildPlan([], test=True))
-
     def test_pull_request_tests_with_the_pinned_libs(self):
-        self.assertEqual(self.plan("pull_request"), BuildPlan(ADDON_PLATFORMS, test=True))
-        self.assertEqual(self.plan("pull_request", addon=True), BuildPlan(ADDON_PLATFORMS, test=True))
+        self.assertEqual(build_plan(core_changed=False, addon_changed=False), BuildPlan(ADDON_PLATFORMS, test=True))
+        self.assertEqual(build_plan(core_changed=False, addon_changed=True), BuildPlan(ADDON_PLATFORMS, test=True))
 
     def test_core_change_leaves_tests_to_build_yml(self):
-        self.assertEqual(self.plan("pull_request", core=True), BuildPlan([], test=False))
-        self.assertEqual(self.plan("pull_request", core=True, addon=True), BuildPlan(ADDON_PLATFORMS, test=False))
-
-    def test_new_library_release_is_tested_on_every_platform(self):
-        self.assertEqual(self.plan(libs_release="core-1.0.0.2"), BuildPlan(ADDON_PLATFORMS, test=True))
+        self.assertEqual(build_plan(core_changed=True, addon_changed=False), BuildPlan([], test=False))
+        self.assertEqual(build_plan(core_changed=True, addon_changed=True), BuildPlan(ADDON_PLATFORMS, test=False))
 
     def test_matrix_names_the_runner_of_each_platform(self):
         self.assertEqual(
@@ -297,6 +320,12 @@ class ReleasesFileTest(unittest.TestCase):
         self.assertIsNone(recorded_commit(self.RELEASES, "npm/simplex-chat", "7.1.0-beta.6.4"))
         self.assertIsNone(recorded_commit({}, "pypi/simplex-chat", "7.1.0b6"))
 
+    def test_highest_recorded_version_is_compared_numerically(self):
+        releases = {"core": {"7.1.0.9": "a", "7.1.0.11": "b"}, "npm": {"@simplex-chat/types": {"0.9.0": "c"}}}
+        self.assertEqual(highest_recorded(releases, "core"), "7.1.0.11")
+        self.assertEqual(highest_recorded(releases, "npm/@simplex-chat/types"), "0.9.0")
+        self.assertIsNone(highest_recorded(releases, "npm/simplex-chat"))
+
     def test_adds_entries_and_keeps_recorded_commits(self):
         updated = add_releases(self.RELEASES, [("npm/simplex-chat", "7.1.0-beta.6.4"), ("core", "7.1.0.11")], "x")
         self.assertEqual(updated["npm"]["simplex-chat"], {"7.1.0-beta.6.3": "n1", "7.1.0-beta.6.4": "x"})
@@ -355,13 +384,10 @@ class RecordReleasesTest(unittest.TestCase):
             self.assertEqual(record_releases([("core", "1")], "other"), "libs0")
 
 
-class PackageStateTest(unittest.TestCase):
+class GitRepoTest(unittest.TestCase):
     def setUp(self):
         self.repo = Path(tempfile.mkdtemp())
         self.git("init", "-q")
-        self.write("pkg/version.txt", 'version = "1"\n')
-        self.write("pkg/src.txt", "a")
-        self.commit("set version")
 
     def git(self, *args):
         subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=self.repo, check=True)
@@ -374,6 +400,19 @@ class PackageStateTest(unittest.TestCase):
         self.git("add", "-A")
         self.git("commit", "-q", "-m", message)
 
+    def head(self):
+        return subprocess.run(
+            ["git", "rev-parse", "HEAD"], cwd=self.repo, capture_output=True, text=True
+        ).stdout.strip()
+
+
+class PackageStateTest(GitRepoTest):
+    def setUp(self):
+        super().setUp()
+        self.write("pkg/version.txt", 'version = "1"\n')
+        self.write("pkg/src.txt", "a")
+        self.commit("set version")
+
     def state(self, releases=None, registry_published=False):
         with mock.patch.object(packages_release, "ROOT", self.repo):
             return package_state(
@@ -383,11 +422,6 @@ class PackageStateTest(unittest.TestCase):
     def changed(self):
         return self.state().changed
 
-    def head(self):
-        return subprocess.run(
-            ["git", "rev-parse", "HEAD"], cwd=self.repo, capture_output=True, text=True
-        ).stdout.strip()
-
     def test_recorded_version_is_published_and_compared_with_its_commit(self):
         self.write("pkg/src.txt", "b")
         self.commit("change published with version 1")
@@ -467,5 +501,303 @@ class PackageStateTest(unittest.TestCase):
         self.assertFalse(self.changed())
 
 
+class ReleasesTest(unittest.TestCase):
+    def test_latest_libs_is_the_highest_complete_published_version(self):
+        releases = [
+            listed_release("core-7.1.0.9", LIBS_ASSETS, 1),
+            listed_release("core-7.1.0.11", LIBS_ASSETS, 2),
+            listed_release("core-7.1.0.12", LIBS_ASSETS[1:], 3),
+            listed_release("core-7.1.0.13", LIBS_ASSETS, 4, draft=True),
+            listed_release("core-test-7.1.0.14", LIBS_ASSETS, 5),
+            listed_release("nodejs-addon-1111111", ADDON_ASSETS, 6),
+        ]
+        versions = {"7.1.0.9", "7.1.0.11", "7.1.0.12", "7.1.0.13", "test-7.1.0.14"}
+        self.assertEqual(latest_libs(releases, versions), "core-7.1.0.11")
+        self.assertEqual(latest_libs(releases, {"7.1.0.9"}), "core-7.1.0.9")
+        self.assertIsNone(latest_libs(releases, {"7.0.0.12"}))
+
+    def test_published_release_is_preferred_to_a_draft(self):
+        draft = listed_release("core-7.1.0.12", LIBS_ASSETS, 1, draft=True)
+        published = listed_release("core-7.1.0.12", LIBS_ASSETS, 2)
+        self.assertEqual(tag_release([draft, published], "core-7.1.0.12"), published)
+        self.assertEqual(tag_release([draft], "core-7.1.0.12"), draft)
+        statuses = [
+            release_status(tag_release(releases, "core-7.1.0.12")) for releases in ([draft, published], [draft], [])
+        ]
+        self.assertEqual(statuses, ["published", "draft", "absent"])
+
+    def test_addon_release_is_found_by_its_notes(self):
+        other = listed_release("nodejs-addon-1111111", ADDON_ASSETS, 1, body=addon_notes("bbb"))
+        draft = listed_release("nodejs-addon-2222222", ADDON_ASSETS, 2, draft=True, body=addon_notes("aaa"))
+        published = listed_release("nodejs-addon-3333333", ADDON_ASSETS, 3, body=addon_notes("aaa"))
+        self.assertEqual(addon_release([other, draft, published], "aaa"), published)
+        self.assertEqual(addon_release([other, draft], "aaa"), draft)
+        self.assertIsNone(addon_release([other], "aaa"))
+
+    def test_missing_assets_of_a_draft(self):
+        self.assertEqual(missing_assets(listed_release("core-7.1.0.12", LIBS_ASSETS[1:], 1)), LIBS_ASSETS[:1])
+        self.assertEqual(missing_assets(listed_release("nodejs-addon-1111111", ADDON_ASSETS, 1)), [])
+
+    def test_pin_problems_list_mismatches_and_unpublished_releases(self):
+        pins = Pins("core-7.1.0.11", {**digests(LIBS_ASSETS, 1), **digests(ADDON_ASSETS, 2)}, "nodejs-addon-1111111")
+        published = {
+            "core-7.1.0.11": github_release(LIBS_ASSETS, 1),
+            "nodejs-addon-1111111": github_release(ADDON_ASSETS, 2),
+        }
+        self.assertEqual(pin_problems(pins, published), [])
+        self.assertEqual(
+            len(pin_problems(pins, {**published, "core-7.1.0.11": github_release(LIBS_ASSETS, 3)})), len(LIBS_ASSETS)
+        )
+        self.assertEqual(
+            pin_problems(pins, {**published, "nodejs-addon-1111111": None}), ["nodejs-addon-1111111 is not published"]
+        )
+
+
+class PublishReleaseTest(unittest.TestCase):
+    def publish(self, releases):
+        api = mock.Mock()
+        with (
+            mock.patch.object(packages_release, "libs_releases", return_value=releases),
+            mock.patch.object(packages_release, "libs_api", api),
+            contextlib.redirect_stdout(io.StringIO()),
+        ):
+            packages_release.publish_release("core-7.1.0.12")
+        return api
+
+    def test_complete_draft_is_published(self):
+        api = self.publish([listed_release("core-7.1.0.12", LIBS_ASSETS, 7, draft=True)])
+        api.assert_called_once_with("releases/7", "PATCH", {"draft": False})
+
+    def test_draft_with_missing_assets_stays_a_draft(self):
+        with self.assertRaises(SystemExit):
+            self.publish([listed_release("core-7.1.0.12", LIBS_ASSETS[1:], 7, draft=True)])
+
+    def test_published_release_is_left_unchanged(self):
+        self.publish([listed_release("core-7.1.0.12", LIBS_ASSETS, 7)]).assert_not_called()
+
+
+class ClaimCoreTest(unittest.TestCase):
+    def claim(self, recorded, after_record=None, problem=None, version="7.1.0.12", branch="master"):
+        reads = iter([(recorded, "blob"), (after_record or recorded, "blob")])
+        self.record = mock.Mock(return_value="libs1")
+        output = io.StringIO()
+        with (
+            mock.patch.object(packages_release, "read_releases", lambda: next(reads)),
+            mock.patch.object(packages_release, "record_releases", self.record),
+            mock.patch.object(packages_release, "types_problem", return_value=problem),
+            contextlib.redirect_stdout(output),
+        ):
+            claim_core(version, "sx1", branch)
+        return output.getvalue().splitlines()
+
+    def test_lower_version_fails(self):
+        with self.assertRaises(SystemExit):
+            self.claim({"core": {"master": {"7.1.0.13": "sx0"}}})
+
+    def test_version_recorded_for_this_commit_resumes(self):
+        self.assertEqual(self.claim({"core": {"master": {"7.1.0.12": "sx1"}}}), ["claim=resume"])
+
+    def test_version_recorded_for_another_commit_is_skipped(self):
+        self.assertEqual(self.claim({"core": {"master": {"7.1.0.11": "sx0", "7.1.0.12": "sx0"}}}), ["claim=skip"])
+
+    def test_version_recorded_on_another_branch_is_skipped(self):
+        recorded = {"core": {"master": {"7.1.0.11": "sx0"}, "stable": {"7.1.0.12": "sx2"}}}
+        self.assertEqual(self.claim(recorded), ["claim=skip"])
+
+    def test_failed_types_check_fails_before_recording(self):
+        with self.assertRaises(SystemExit):
+            self.claim({"core": {"master": {"7.1.0.11": "sx0"}}}, problem="types changed")
+        self.record.assert_not_called()
+
+    def test_unrecorded_version_is_claimed(self):
+        claimed = {"core": {"master": {"7.1.0.11": "sx0", "7.1.0.12": "sx1"}}}
+        self.assertEqual(self.claim({"core": {"master": {"7.1.0.11": "sx0"}}}, claimed), ["claim=new", "commit=libs1"])
+        self.record.assert_called_once_with([("core/master", "7.1.0.12")], "sx1")
+
+    def test_stable_versions_are_compared_with_stable_only(self):
+        recorded = {"core": {"master": {"7.1.0.11": "sx0"}, "stable": {"7.0.0.12": "sx3"}}}
+        claimed = {"core": {**recorded["core"], "stable": {"7.0.0.12": "sx3", "7.0.0.13": "sx1"}}}
+        self.assertEqual(
+            self.claim(recorded, claimed, version="7.0.0.13", branch="stable"), ["claim=new", "commit=libs1"]
+        )
+        self.record.assert_called_once_with([("core/stable", "7.0.0.13")], "sx1")
+        with self.assertRaises(SystemExit):
+            self.claim(recorded, version="7.0.0.11", branch="stable")
+
+    def test_concurrent_claim_of_another_commit_is_skipped(self):
+        claimed = {"core": {"master": {"7.1.0.11": "sx0", "7.1.0.12": "sx2"}}}
+        self.assertEqual(self.claim({"core": {"master": {"7.1.0.11": "sx0"}}}, claimed), ["claim=skip"])
+
+
+class TypesProblemTest(unittest.TestCase):
+    def problem(self, types, changed=True, published=False, releases=None):
+        directory = tempfile.TemporaryDirectory()
+        self.addCleanup(directory.cleanup)
+        root = Path(directory.name)
+        (root / TYPES_DIR).mkdir(parents=True)
+        (root / TYPES_DIR / "package.json").write_text(json.dumps({"name": "@simplex-chat/types", **types}))
+        with (
+            mock.patch.object(packages_release, "ROOT", root),
+            mock.patch.object(packages_release, "types_changed", return_value=changed),
+            mock.patch.object(packages_release, "npm_published", return_value=published),
+        ):
+            return types_problem("7.1.0.12", releases or {}, "master")
+
+    def test_unchanged_types_pass(self):
+        self.assertIsNone(self.problem({"version": "0.12.1", "simplexChatCore": "7.1.0.11"}, changed=False))
+
+    def test_changed_types_with_a_new_version_for_the_core_pass(self):
+        self.assertIsNone(self.problem({"version": "0.12.2", "simplexChatCore": "7.1.0.12"}))
+
+    def test_changed_types_with_a_published_version_fail(self):
+        self.assertIsNotNone(self.problem({"version": "0.12.1", "simplexChatCore": "7.1.0.12"}, published=True))
+        recorded = {"npm": {"@simplex-chat/types": {"0.12.1": "c"}}}
+        self.assertIsNotNone(self.problem({"version": "0.12.1", "simplexChatCore": "7.1.0.12"}, releases=recorded))
+
+    def test_changed_types_for_another_core_fail(self):
+        self.assertIsNotNone(self.problem({"version": "0.12.2", "simplexChatCore": "7.1.0.11"}))
+
+
+class CoreReleaseTest(GitRepoTest):
+    def setUp(self):
+        super().setUp()
+        self.write(CABAL_FILE, "name:           simplex-chat\nversion:        7.1.0.11\n")
+        types = {"name": "@simplex-chat/types", "version": "0.12.1", "simplexChatCore": "7.1.0.11"}
+        self.write(f"{TYPES_DIR}/package.json", json.dumps(types, indent=2) + "\n")
+        self.write(TYPES_README, "# Types\n\nSimpleX Chat core: 7.1.0.11\n")
+        self.write(f"{TYPES_DIR}/src/types.ts", "a")
+        self.commit("types 0.12.1")
+        self.releases = {"core": {"master": {"7.1.0.11": self.head()}}}
+
+    def release(self, version, types_version=None):
+        with (
+            mock.patch.object(packages_release, "ROOT", self.repo),
+            mock.patch.object(packages_release, "read_releases", return_value=(self.releases, "blob")),
+            contextlib.redirect_stdout(io.StringIO()),
+        ):
+            core_release(version, "master", types_version)
+
+    def read(self, file):
+        return (self.repo / file).read_text()
+
+    def types(self):
+        return json.loads(self.read(f"{TYPES_DIR}/package.json"))
+
+    def test_changed_types_get_a_new_version_for_the_core(self):
+        self.write(f"{TYPES_DIR}/src/types.ts", "b")
+        self.commit("types change")
+        self.release("7.1.0.12")
+        self.assertIn("version:        7.1.0.12\n", self.read(CABAL_FILE))
+        self.assertEqual((self.types()["version"], self.types()["simplexChatCore"]), ("0.12.2", "7.1.0.12"))
+        self.assertIn("SimpleX Chat core: 7.1.0.12\n", self.read(TYPES_README))
+
+    def test_unchanged_types_keep_their_version(self):
+        self.release("7.1.0.12")
+        self.assertIn("version:        7.1.0.12\n", self.read(CABAL_FILE))
+        self.assertEqual((self.types()["version"], self.types()["simplexChatCore"]), ("0.12.1", "7.1.0.11"))
+
+    def test_requested_types_version_wins(self):
+        self.release("7.1.0.12", "7.1.0-beta.6.1")
+        self.assertEqual((self.types()["version"], self.types()["simplexChatCore"]), ("7.1.0-beta.6.1", "7.1.0.12"))
+
+    def test_types_version_is_bumped_from_the_previous_core_release(self):
+        self.write(f"{TYPES_DIR}/package.json", json.dumps({**self.types(), "version": "0.12.5"}))
+        self.write(f"{TYPES_DIR}/src/types.ts", "b")
+        self.commit("types change and a manual version")
+        self.release("7.1.0.12")
+        self.assertEqual(self.types()["version"], "0.12.2")
+
+    def test_version_must_be_higher_than_the_recorded_one(self):
+        with self.assertRaises(SystemExit):
+            self.release("7.1.0.11")
+        self.assertIn("version:        7.1.0.11\n", self.read(CABAL_FILE))
+
+
+class PrepareTest(GitRepoTest):
+    def setUp(self):
+        super().setUp()
+        nodejs = {
+            "name": "simplex-chat",
+            "version": "7.1.0-beta.6.1",
+            "dependencies": {"@simplex-chat/types": "^0.12.1"},
+            "devDependencies": {"node-addon-api": "^8.5.0"},
+        }
+        self.write(CABAL_FILE, "version:        1.0.0.2\n")
+        self.write(f"{NODEJS_DIR}/binding.gyp", "{}")
+        self.write(f"{NODEJS_DIR}/cpp/simplex.cc", "add-on")
+        self.write(f"{NODEJS_DIR}/package.json", json.dumps(nodejs))
+        self.write(RELEASE_FILE, render_release(state().nodejs_pins))
+        self.write(f"{TYPES_DIR}/package.json", json.dumps({"name": "@simplex-chat/types", "version": "0.12.1"}))
+        self.write(PYTHON_VERSION_FILE, render_python_version("7.1.0b6", state().python_pins))
+        self.write(f"{PYTHON_DIR}/src/simplex_chat/types/_types.py", "a")
+        self.write(f"{PYTHON_DIR}/pyproject.toml", 'name = "simplex-chat"\n')
+        self.commit("packages")
+        with mock.patch.object(packages_release, "ROOT", self.repo):
+            sources_hash = packages_release.current_addon_hash()
+        self.libs = [
+            listed_release("core-1.0.0.2", LIBS_ASSETS, 3),
+            listed_release("nodejs-addon-1111111", ADDON_ASSETS, 4, body=addon_notes(sources_hash)),
+        ]
+        self.releases = {
+            "core": {"master": {"1.0.0.2": self.head()}},
+            "npm": {"simplex-chat": {"7.1.0-beta.6.1": self.head()}},
+            "pypi": {"simplex-chat": {"7.1.0b6": self.head()}},
+        }
+
+    def prepare(self, *packages):
+        published = {release["tag_name"]: release for release in self.libs}
+        download = mock.Mock()
+        with (
+            mock.patch.object(packages_release, "ROOT", self.repo),
+            mock.patch.object(packages_release, "read_releases", return_value=(self.releases, "blob")),
+            mock.patch.object(packages_release, "libs_releases", return_value=self.libs),
+            mock.patch.object(packages_release, "github_release", published.get),
+            mock.patch.object(packages_release, "npm_published", return_value=True),
+            mock.patch.object(packages_release, "pypi_published", return_value=True),
+            mock.patch.object(packages_release, "download", download),
+            contextlib.redirect_stdout(io.StringIO()),
+            contextlib.redirect_stderr(io.StringIO()),
+        ):
+            prepare(
+                argparse.Namespace(packages=list(packages), branch="master", nodejs_version=None, python_version=None)
+            )
+        return download
+
+    def test_listed_package_is_released_alone(self):
+        download = self.prepare("nodejs")
+        pins = parse_release((self.repo / RELEASE_FILE).read_text())
+        self.assertEqual((pins.libs_release, pins.addon_release), ("core-1.0.0.2", "nodejs-addon-1111111"))
+        self.assertEqual(json.loads((self.repo / NODEJS_DIR / "package.json").read_text())["version"], "7.1.0-beta.6.2")
+        python = parse_python_version((self.repo / PYTHON_VERSION_FILE).read_text())
+        self.assertEqual(python, ("7.1.0b6", state().python_pins))
+        self.assertEqual(download.call_count, len(LIBS_ASSETS) + len(ADDON_ASSETS))
+
+    def test_python_types_newer_than_the_pinned_core_stop_the_release(self):
+        self.write(f"{PYTHON_DIR}/src/simplex_chat/types/_types.py", "b")
+        self.commit("python types change")
+        with self.assertRaises(SystemExit):
+            self.prepare("python")
+        self.assertEqual(parse_python_version((self.repo / PYTHON_VERSION_FILE).read_text())[0], "7.1.0b6")
+
+
+class DownloadTest(unittest.TestCase):
+    def setUp(self):
+        directory = tempfile.TemporaryDirectory()
+        self.addCleanup(directory.cleanup)
+        self.directory = directory.name
+
+    def download(self, sha256):
+        with mock.patch.object(packages_release.urllib.request, "urlopen", return_value=io.BytesIO(b"data")):
+            return download("core-1.0.0.1", "a.zip", self.directory, sha256)
+
+    def test_matching_digest_keeps_the_file(self):
+        self.assertEqual(self.download(hashlib.sha256(b"data").hexdigest()).read_bytes(), b"data")
+
+    def test_mismatching_digest_removes_the_file(self):
+        with self.assertRaises(SystemExit):
+            self.download(sha(1))
+        self.assertFalse((Path(self.directory) / "a.zip").exists())
+
+
 if __name__ == "__main__":
     unittest.main()