#!/usr/bin/env python3 """Release automation for simplex-chat and @simplex-chat/types on npm and simplex-chat on PyPI. core-release set the core version and the types version for a library release claim-core claim the library release of the core version for a commit release-status print whether a release of simplex-chat-libs is published, a draft or absent addon-release print the add-on release of the current add-on sources latest-libs print the newest published library release publish-release publish a draft release that has every asset check-pins check the pins of the packages against the published releases download download a release asset and check its digest download-pinned download a pinned asset and check its pin changes print whether the changes since a commit touch the chat core and the add-on build-plan print the platforms to test the packages on, none when build.yml runs for the pull request prepare update pins and versions of the packages publish-plan print the package versions that are not published yet record record releases and published versions in releases.toml of simplex-chat-libs test-names rename the packages to NPM_SCOPE and PYPI_PROJECT (test repositories) """ from __future__ import annotations import argparse import base64 import copy import hashlib import json import os import re import shlex import subprocess import sys import tempfile import tomllib import urllib.error import urllib.request from collections.abc import Callable, Collection from dataclasses import dataclass, field, replace from typing import Any from pathlib import Path ROOT = Path(__file__).resolve().parents[2] NODEJS_DIR = "packages/simplex-chat-nodejs" TYPES_DIR = "packages/simplex-chat-client/types/typescript" PYTHON_DIR = "packages/simplex-chat-python" RELEASE_FILE = f"{NODEJS_DIR}/src/release.ts" PYTHON_VERSION_FILE = f"{PYTHON_DIR}/src/simplex_chat/_version.py" ADDON_SOURCES = [f"{NODEJS_DIR}/binding.gyp", f"{NODEJS_DIR}/cpp"] CORE_SOURCES = ["src", "simplex-chat.cabal", "cabal.project"] BUILD_WORKFLOW = ".github/workflows/build.yml" CABAL_FILE = "simplex-chat.cabal" CABAL_VERSION = re.compile(r"^(version:\s*)(\S+)", re.M) TYPES_SOURCES = [f"{TYPES_DIR}/src"] TYPES_README = f"{TYPES_DIR}/README.md" TYPES_README_CORE = re.compile(r"^SimpleX Chat core: .*$", re.M) PYTHON_TYPES_SOURCES = [f"{PYTHON_DIR}/src/simplex_chat/types"] ADDON_HASH_LENGTH = 12 CORE_TAG = re.compile(r"^core-(\d+(?:\.\d+)*)$") ADDON_TAG_PREFIX = "nodejs-addon-" LIBS_ASSETS = [ "simplex-chat-libs-linux-aarch64.zip", "simplex-chat-libs-linux-x86_64-postgres.zip", "simplex-chat-libs-linux-x86_64.zip", "simplex-chat-libs-macos-aarch64.zip", "simplex-chat-libs-macos-x86_64.zip", "simplex-chat-libs-windows-x86_64.zip", ] ADDON_RUNNERS = { "linux-aarch64": "ubuntu-22.04-arm", "linux-x86_64": "ubuntu-22.04", "macos-aarch64": "macos-latest", "macos-x86_64": "macos-15-intel", "windows-x86_64": "windows-latest", } ADDON_PLATFORMS = list(ADDON_RUNNERS) ADDON_ASSETS = [f"simplex-chat-nodejs-{platform}.node" for platform in ADDON_PLATFORMS] SHA256_HEX = re.compile(r"^[0-9a-f]{64}$") RELEASES_FILE = "releases.toml" RECORD_ATTEMPTS = 5 DOWNLOAD_CHUNK = 1 << 20 TOML_BARE_KEY = re.compile(r"^[A-Za-z0-9_-]+$") # Pure functions def addon_hash(files: dict[str, bytes], node_addon_api: str) -> str: digest = hashlib.sha256() for name in sorted(files): digest.update(name.encode() + b"\0" + files[name] + b"\0") digest.update(f"node-addon-api@{node_addon_api}".encode()) return digest.hexdigest()[:ADDON_HASH_LENGTH] def addon_notes(sources_hash: str) -> str: return f"sources: {sources_hash}" def version_key(version: str) -> tuple[int, ...]: return tuple(int(part) for part in version.split(".")) def bump_npm_version(version: str) -> str: """A package release of app version 7.1.0-beta.6 is 7.1.0-beta.6.1, then 7.1.0-beta.6.2.""" _, _, prerelease = version.partition("-") if len(prerelease.split(".")) == 2: return f"{version}.1" head, _, last = version.rpartition(".") return f"{head}.{int(last) + 1}" if last.isdigit() else f"{version}.1" def bump_python_version(version: str) -> str: """A package release of app version 7.1.0b6 is 7.1.0b6.post1, then 7.1.0b6.post2.""" post = re.search(r"\.post(\d+)$", version) return f"{version[: post.start()]}.post{int(post[1]) + 1}" if post else f"{version}.post1" def dist_tag(version: str) -> str: return "beta" if "-" in version else "latest" def registry_name(name: str, scope: str | None) -> str: """Package name on npm: test repositories publish under their own scope.""" return f"{scope}/{name.split('/')[-1]}" if scope else name @dataclass(frozen=True) class Pins: libs_release: str sha256: dict[str, str] addon_release: str | None = None def parse_release(source: str) -> Pins: libs = re.search(r'export const LIBS_RELEASE = "([^"]+)"', source) addon = re.search(r'export const ADDON_RELEASE: string \| undefined = (?:"([^"]+)"|undefined)', source) if not libs or not addon: raise ValueError(f"cannot parse {RELEASE_FILE}") hashes = dict(re.findall(r'^ {2}"([^"]+)": "([0-9a-f]{64})",$', source, re.M)) return Pins(libs[1], hashes, addon[1]) def render_release(pins: Pins) -> str: hashes = "".join(f' "{asset}": "{pins.sha256[asset]}",\n' for asset in sorted(pins.sha256)) addon = f'"{pins.addon_release}"' if pins.addon_release else "undefined" return ( "// Written by scripts/ci/packages_release.py from the simplex-chat-libs releases.\n" "\n" f'export const LIBS_RELEASE = "{pins.libs_release}"\n' "\n" f"export const ADDON_RELEASE: string | undefined = {addon}\n" "\n" "export const SHA256: {readonly [asset: string]: string} = {\n" f"{hashes}}}\n" ) def parse_python_version(source: str) -> tuple[str, Pins]: version = re.search(r'^__version__ = "([^"]+)"', source, re.M) libs = re.search(r'^LIBS_RELEASE = "([^"]+)"', source, re.M) if not version or not libs: raise ValueError(f"cannot parse {PYTHON_VERSION_FILE}") hashes = dict(re.findall(r'^ {4}"([^"]+)": "([0-9a-f]{64})",$', source, re.M)) return version[1], Pins(libs[1], hashes) def render_python_version(version: str, pins: Pins) -> str: hashes = "".join(f' "{asset}": "{pins.sha256[asset]}",\n' for asset in sorted(pins.sha256)) return ( '"""Package version and the simplex-chat-libs release it loads.\n' "\n" "Written by scripts/ci/packages_release.py prepare: it bumps the version\n" "and pins new library releases with the SHA-256 of their assets.\n" '"""\n' "\n" f'__version__ = "{version}" # PEP 440 — read by hatchling for wheel metadata\n' f'LIBS_RELEASE = "{pins.libs_release}" # simplex-chat-libs release tag\n' "LIBS_SHA256 = {\n" f"{hashes}}}\n" ) def asset_digests(release: dict) -> dict[str, str]: return {a["name"]: (a.get("digest") or "").removeprefix("sha256:") for a in release["assets"]} def release_digests(release: dict | None, assets: list[str]) -> dict[str, str] | None: """Asset digests of a GitHub release, or None when the release is missing or incomplete.""" if release is None: return None digests = asset_digests(release) if not all(SHA256_HEX.match(digests.get(asset, "")) for asset in assets): return None return {asset: digests[asset] for asset in assets} def release_assets(tag: str) -> list[str]: if CORE_TAG.match(tag): return LIBS_ASSETS if tag.startswith(ADDON_TAG_PREFIX): return ADDON_ASSETS raise ValueError(f"no asset set for release {tag}") def preferred_release(releases: list[dict]) -> dict | None: """The published release among them, otherwise the first draft.""" return next((r for r in releases if not r["draft"]), next(iter(releases), None)) def release_status(release: dict | None) -> str: if release is None: return "absent" return "draft" if release["draft"] else "published" def tag_release(releases: list[dict], tag: str) -> dict | None: return preferred_release([r for r in releases if r["tag_name"] == tag]) def addon_release(releases: list[dict], sources_hash: str) -> dict | None: notes = addon_notes(sources_hash) return preferred_release( [r for r in releases if r["tag_name"].startswith(ADDON_TAG_PREFIX) and r.get("body") == notes] ) def core_version(tag: str) -> str | None: match = CORE_TAG.match(tag) return match[1] if match else None def latest_libs(releases: list[dict], versions: Collection[str]) -> str | None: """The newest published release of the core versions that has every library asset.""" tags = [ r["tag_name"] for r in releases if not r["draft"] and core_version(r["tag_name"]) in versions and release_digests(r, LIBS_ASSETS) ] return max(tags, key=lambda tag: version_key(core_version(tag)), default=None) def missing_assets(release: dict) -> list[str]: present = {a["name"] for a in release["assets"]} return [asset for asset in release_assets(release["tag_name"]) if asset not in present] def pin_problems(pins: Pins, published: dict[str, dict | None]) -> list[str]: """Differences between the pins and the published releases of the pinned tags.""" problems = [] for tag, release in published.items(): if release is None: problems.append(f"{tag} is not published") continue digests = asset_digests(release) for asset in release_assets(tag): if pins.sha256.get(asset) != digests.get(asset): problems.append(f"{asset} of {tag}: pinned {pins.sha256.get(asset)}, published {digests.get(asset)}") return problems @dataclass(frozen=True) class Package: version: str changed: bool published: bool @dataclass(frozen=True) class State: libs_tag: str libs_digests: dict[str, str] | None addon_tag: str addon_digests: dict[str, str] | None nodejs_pins: Pins python_pins: Pins types_dependency: str nodejs_types_dependency: str nodejs: Package python: Package packages: frozenset[str] = frozenset({"nodejs", "python"}) nodejs_version: str | None = None python_version: str | None = None @dataclass(frozen=True) class Plan: nodejs_pins: Pins python_pins: Pins types_dependency: str | None nodejs_version: str nodejs_changed: bool python_version: str python_changed: bool notes: list[str] = field(default_factory=list) @property def title(self) -> str: parts = [] if self.nodejs_changed: parts.append(f"nodejs {self.nodejs_version}") if self.python_changed: parts.append(f"python {self.python_version}") return f"packages: release {', '.join(parts)}" def plan(state: State) -> Plan: """What a package release changes, from the state of the branch and the releases.""" notes: list[str] = [] def pin(pins: Pins, field_name: str, tag: str, digests: dict[str, str] | None, what: str) -> tuple[Pins, bool]: current = getattr(pins, field_name) if current == tag: return pins, False if digests is None: notes.append(f"{what} release {tag} is missing or incomplete; {current or 'none'} stays pinned") return pins, False notes.append(f"{what}: {tag}") return replace(pins, **{field_name: tag, "sha256": {**pins.sha256, **digests}}), True def next_version(pkg: Package, changed: bool, override: str | None, bump) -> str: if override: return override return bump(pkg.version) if changed and pkg.published else pkg.version nodejs_pins, nodejs_version, nodejs_changed, types_dependency = state.nodejs_pins, state.nodejs.version, False, None if "nodejs" in state.packages: nodejs_pins, nodejs_libs = pin( state.nodejs_pins, "libs_release", state.libs_tag, state.libs_digests, "Node.js libsimplex" ) nodejs_pins, addon = pin(nodejs_pins, "addon_release", state.addon_tag, state.addon_digests, "Node.js add-on") if state.types_dependency != state.nodejs_types_dependency: types_dependency = state.types_dependency notes.append(f"@simplex-chat/types: {types_dependency}") nodejs_changed = nodejs_libs or addon or types_dependency is not None or state.nodejs.changed nodejs_version = next_version(state.nodejs, nodejs_changed, state.nodejs_version, bump_npm_version) python_pins, python_version, python_changed = state.python_pins, state.python.version, False if "python" in state.packages: python_pins, python_libs = pin( state.python_pins, "libs_release", state.libs_tag, state.libs_digests, "Python libsimplex" ) python_changed = python_libs or state.python.changed python_version = next_version(state.python, python_changed, state.python_version, bump_python_version) for name, old, new in [ ("simplex-chat (npm)", state.nodejs.version, nodejs_version), ("simplex-chat (PyPI)", state.python.version, python_version), ]: if old != new: notes.append(f"{name}: {old} → {new}") return Plan( nodejs_pins=nodejs_pins, python_pins=python_pins, types_dependency=types_dependency, nodejs_version=nodejs_version, nodejs_changed=nodejs_changed or nodejs_version != state.nodejs.version, python_version=python_version, python_changed=python_changed or python_version != state.python.version, notes=notes, ) def pull_request_paths(workflow: str) -> list[str]: found = re.search(r'^ pull_request:\n paths:\n((?: - ".*"\n)+)', workflow, re.M) if not found: raise ValueError(f"no pull_request paths in {BUILD_WORKFLOW}") return re.findall(r'^ - "(.*)"$', found[1], re.M) def build_matrix(platforms: list[str]) -> str: return json.dumps([{"platform": platform, "os": ADDON_RUNNERS[platform]} for platform in platforms]) def core_key(branch: str) -> str: return f"core/{branch}" def recorded_versions(releases: dict, key: str) -> dict[str, str]: """The recorded versions and their simplex-chat commits; key is "core/", "npm/" or "pypi/".""" table = releases for part in key.split("/", 1): table = table.get(part, {}) return {version: commit for version, commit in table.items() if isinstance(commit, str)} def recorded_commit(releases: dict, key: str, version: str) -> str | None: return recorded_versions(releases, key).get(version) def highest_recorded(releases: dict, key: str) -> str | None: return max(recorded_versions(releases, key), key=version_key, default=None) def recorded_core_commit(releases: dict, version: str) -> str | None: """The commit recorded for the core version on any branch: a release tag is shared by the branches.""" for branch in releases.get("core", {}): if commit := recorded_commit(releases, core_key(branch), version): return commit return None def previous_core_commit(releases: dict, branch: str) -> str | None: """The commit of the highest core version recorded for the branch.""" previous = highest_recorded(releases, core_key(branch)) return recorded_commit(releases, core_key(branch), previous) if previous else None def add_releases(releases: dict, entries: list[tuple[str, str]], commit: str) -> dict: """A recorded version keeps its first commit: re-running a release does not rewrite history.""" result = copy.deepcopy(releases) for key, version in entries: table = result for part in key.split("/", 1): table = table.setdefault(part, {}) table.setdefault(version, commit) return result def render_releases(releases: dict) -> str: def toml_key(key: str) -> str: return key if TOML_BARE_KEY.match(key) else json.dumps(key) sections: list[str] = [] # in the order of recording: entries are only added, so that order is chronological def walk(path: list[str], table: dict) -> None: values = [(k, v) for k, v in table.items() if isinstance(v, str)] if values: lines = [f"[{'.'.join(toml_key(part) for part in path)}]"] lines += [f"{toml_key(version)} = {json.dumps(commit)}" for version, commit in values] sections.append("\n".join(lines) + "\n") for name, child in table.items(): if isinstance(child, dict): walk([*path, name], child) walk([], releases) return "\n".join(sections) # Repository and registry access def git(*args: str) -> str: return subprocess.run(["git", *args], cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip() def read_text(file: str) -> str: return (ROOT / file).read_text() def read_field(file: str, pattern: str) -> str: found = re.search(pattern, read_text(file), re.M) if not found: raise ValueError(f"no match for {pattern} in {file}") return found[1] def read_json(file: str) -> dict: return json.loads(read_text(file)) def write_json(file: str, value: dict) -> None: (ROOT / file).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n") def current_addon_hash() -> str: files = { path.relative_to(ROOT).as_posix(): path.read_bytes() for source in ADDON_SOURCES for path in ([ROOT / source] if (ROOT / source).is_file() else sorted((ROOT / source).rglob("*"))) if path.is_file() } return addon_hash(files, read_json(f"{NODEJS_DIR}/package.json")["devDependencies"]["node-addon-api"]) def cabal_version() -> str: return CABAL_VERSION.search(read_text(CABAL_FILE))[2] def changed_since(base: str, sources: list[str]) -> bool: diff = subprocess.run(["git", "diff", "--quiet", base, "HEAD", "--", *sources], cwd=ROOT) return diff.returncode != 0 def changes(base: str) -> None: print(f"core={'true' if changed_since(base, CORE_SOURCES) else ''}") print(f"addon={'true' if changed_since(base, ADDON_SOURCES) else ''}") def print_build_plan(args: argparse.Namespace) -> None: build_paths = [f":(glob){path}" for path in pull_request_paths(read_text(BUILD_WORKFLOW))] platforms = [] if changed_since(args.base, build_paths) else ADDON_PLATFORMS print(f"platforms={build_matrix(platforms)}") def package_state( key: str, version: str, directory: str, version_file: str, version_pattern: str, releases: dict, registry_published: Callable[[], bool], ) -> Package: """A recorded version is published from its recorded commit; the registry is asked about other versions, which are compared with the commit that set them.""" recorded = recorded_commit(releases, key, version) base = ( recorded if recorded and commit_exists(recorded) else git("log", "-1", "--format=%H", "-G", version_pattern, "--", version_file) ) changed = changed_since(base, [directory, f":!{directory}/tests"]) return Package(version, changed, recorded is not None or registry_published()) def http_json(url: str, headers: dict[str, str] | None = None, method: str = "GET", body: dict | None = None) -> Any: """The JSON response, or None for 404.""" request = urllib.request.Request( url, headers={"User-Agent": "simplex-chat-release", **(headers or {})}, method=method, data=json.dumps(body).encode() if body is not None else None, ) try: with urllib.request.urlopen(request, timeout=60) as response: return json.load(response) except urllib.error.HTTPError as e: if e.code == 404: return None raise # Workflows pass unset repository variables as empty strings. def npm_package_url(name: str) -> str: registry = os.environ.get("NPM_REGISTRY") or "https://registry.npmjs.org" return f"{registry}/{registry_name(name, os.environ.get('NPM_SCOPE')).replace('/', '%2F')}" def npm_published(name: str, version: str) -> bool: return http_json(f"{npm_package_url(name)}/{version}") is not None def python_project() -> str: return read_field(f"{PYTHON_DIR}/pyproject.toml", r'^name = "([^"]+)"') def pypi_published(version: str) -> bool: index = os.environ.get("PYPI_URL") or "https://pypi.org" project = os.environ.get("PYPI_PROJECT") or python_project() return http_json(f"{index}/pypi/{project}/{version}/json") is not None def libs_repository() -> str: return os.environ.get("LIBS_REPOSITORY") or "simplex-chat/simplex-chat-libs" def libs_api(path: str, method: str = "GET", body: dict | None = None) -> Any: """GitHub API of the simplex-chat-libs repository.""" headers = {"Accept": "application/vnd.github+json"} if token := os.environ.get("GH_TOKEN"): headers["Authorization"] = f"Bearer {token}" return http_json(f"https://api.github.com/repos/{libs_repository()}/{path}", headers, method, body) def github_release(tag: str) -> dict | None: """The published release of the tag.""" return libs_api(f"releases/tags/{tag}") def libs_releases() -> list[dict]: """The releases of simplex-chat-libs; drafts are listed only with a token that can push to it.""" releases: list[dict] = [] page = 1 while batch := libs_api(f"releases?per_page=100&page={page}"): releases += batch page += 1 return releases def download(tag: str, asset: str, directory: str, sha256: str) -> Path: """Saves the asset of the release to the directory; exits when its SHA-256 differs.""" url = f"https://github.com/{libs_repository()}/releases/download/{tag}/{asset}" target = Path(directory) / asset target.parent.mkdir(parents=True, exist_ok=True) digest = hashlib.sha256() request = urllib.request.Request(url, headers={"User-Agent": "simplex-chat-release"}) with urllib.request.urlopen(request, timeout=600) as response, target.open("wb") as file: while chunk := response.read(DOWNLOAD_CHUNK): file.write(chunk) digest.update(chunk) if digest.hexdigest() != sha256: target.unlink() sys.exit(f"SHA-256 of {asset} of {tag} is {digest.hexdigest()}, expected {sha256}") return target def read_releases() -> tuple[dict, str | None]: """The releases and the blob of the file, which a change must name.""" file = libs_api(f"contents/{RELEASES_FILE}") if file is None: return {}, None return tomllib.loads(base64.b64decode(file["content"]).decode()), file["sha"] def record_releases(entries: list[tuple[str, str]], commit: str) -> str: """Adds the entries in one commit of simplex-chat-libs and returns that commit.""" message = ", ".join(f"{key} {version}" for key, version in entries) attempt = 1 while True: releases, blob = read_releases() updated = add_releases(releases, entries, commit) if blob and updated == releases: return libs_api(f"commits?path={RELEASES_FILE}&per_page=1")[0]["sha"] body = { "message": message, "content": base64.b64encode(render_releases(updated).encode()).decode(), "committer": {"name": "SimpleX Chat", "email": "build@simplex.chat"}, **({"sha": blob} if blob else {}), } try: return libs_api(f"contents/{RELEASES_FILE}", "PUT", body)["commit"]["sha"] except urllib.error.HTTPError as e: # 409 and 422: another writer changed or created the file since it was read if e.code not in (409, 422) or attempt == RECORD_ATTEMPTS: raise attempt += 1 def commit_exists(commit: str) -> bool: found = subprocess.run(["git", "cat-file", "-e", f"{commit}^{{commit}}"], cwd=ROOT, capture_output=True) return found.returncode == 0 def fetch_commit(commit: str) -> str: """The commit, fetched from origin when the checkout lacks it.""" if not commit_exists(commit): git("fetch", "-q", "--depth=1", "origin", commit) return commit def npm_key(directory: str) -> str: return f"npm/{read_json(f'{directory}/package.json')['name']}" def pypi_key() -> str: return f"pypi/{python_project()}" def npm_package(directory: str, releases: dict) -> Package: package = read_json(f"{directory}/package.json") name, version = package["name"], package["version"] return package_state( npm_key(directory), version, directory, f"{directory}/package.json", '"version":', releases, lambda: npm_published(name, version), ) def types_changed(releases: dict, branch: str) -> bool: """Whether the types sources differ from those of the previous core release on the branch.""" previous = previous_core_commit(releases, branch) return previous is None or changed_since(fetch_commit(previous), TYPES_SOURCES) def types_problem(version: str, releases: dict, branch: str) -> str | None: """Why the types can't be released with core , or None when they can.""" if not types_changed(releases, branch): return None types = read_json(f"{TYPES_DIR}/package.json") if recorded_commit(releases, npm_key(TYPES_DIR), types["version"]) or npm_published(types["name"], types["version"]): return f"{types['name']} changed, and its version {types['version']} is published" if types.get("simplexChatCore") != version: return f"simplexChatCore of {types['name']} is {types.get('simplexChatCore')}, expected {version}" return None def claim_core(version: str, commit: str, branch: str) -> None: releases, _ = read_releases() highest = highest_recorded(releases, core_key(branch)) if highest and version_key(version) < version_key(highest): sys.exit(f"core {version} is lower than core {highest} recorded for {branch}") if recorded := recorded_core_commit(releases, version): print(f"claim={'resume' if recorded == commit else 'skip'}") return if problem := types_problem(version, releases, branch): sys.exit(problem) libs_commit = record_releases([(core_key(branch), version)], commit) releases, _ = read_releases() if recorded_core_commit(releases, version) == commit: print("claim=new") print(f"commit={libs_commit}") else: print("claim=skip") def current_branch() -> str: return git("rev-parse", "--abbrev-ref", "HEAD") def core_release(version: str, branch: str, types_version: str | None) -> None: releases, _ = read_releases() highest = highest_recorded(releases, core_key(branch)) if highest and version_key(version) <= version_key(highest): sys.exit(f"core {version} is not higher than core {highest} recorded for {branch}") (ROOT / CABAL_FILE).write_text(CABAL_VERSION.sub(rf"\g<1>{version}", read_text(CABAL_FILE), count=1)) print(f"{CABAL_FILE}: {version}") if not types_version and not types_changed(releases, branch): return types = read_json(f"{TYPES_DIR}/package.json") if not types_version: previous = previous_core_commit(releases, branch) released = json.loads(git("show", f"{previous}:{TYPES_DIR}/package.json")) if previous else types types_version = bump_npm_version(released["version"]) readme, found = TYPES_README_CORE.subn(f"SimpleX Chat core: {version}", read_text(TYPES_README), count=1) if not found: sys.exit(f"{TYPES_README} has no line 'SimpleX Chat core: '") write_json(f"{TYPES_DIR}/package.json", {**types, "version": types_version, "simplexChatCore": version}) (ROOT / TYPES_README).write_text(readme) print(f"{types['name']}: {types_version}") def pinned_tags(pins: Pins) -> list[str]: return [tag for tag in (pins.libs_release, pins.addon_release) if tag] def published_pin_problems(pins: Pins) -> list[str]: return pin_problems(pins, {tag: github_release(tag) for tag in pinned_tags(pins)}) def prepare(args: argparse.Namespace) -> None: packages = frozenset(args.packages or ["nodejs", "python"]) branch = args.branch or current_branch() releases, _ = read_releases() libs = libs_releases() libs_tag = latest_libs(libs, recorded_versions(releases, core_key(branch))) if libs_tag is None: sys.exit(f"no published library release recorded for {branch}") core = core_version(libs_tag) if version_key(cabal_version()) > version_key(core): print(f"warning: core-{cabal_version()} is not published, {libs_tag} is pinned", file=sys.stderr) sources_hash = current_addon_hash() addon = addon_release(libs, sources_hash) addon_digests = release_digests(addon, ADDON_ASSETS) if release_status(addon) == "published" else None if "nodejs" in packages and addon_digests is None: sys.exit(f"the add-on release of sources {sources_hash} is not published with every asset") types = read_json(f"{TYPES_DIR}/package.json") types_name = types["name"] if "nodejs" in packages and not npm_published(types_name, types["version"]): sys.exit(f"{types_name} {types['version']} is not published") if "python" in packages: core_commit = recorded_commit(releases, core_key(branch), core) if changed_since(core_commit, PYTHON_TYPES_SOURCES): sys.exit(f"the Python types differ from those of {libs_tag}") nodejs = read_json(f"{NODEJS_DIR}/package.json") python_version, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE)) result = plan( State( libs_tag=libs_tag, libs_digests=release_digests(tag_release(libs, libs_tag), LIBS_ASSETS), addon_tag=addon["tag_name"] if addon else "", addon_digests=addon_digests, nodejs_pins=parse_release(read_text(RELEASE_FILE)), python_pins=python_pins, types_dependency=f"^{types['version']}", nodejs_types_dependency=nodejs["dependencies"][types_name], nodejs=npm_package(NODEJS_DIR, releases), python=package_state( pypi_key(), python_version, PYTHON_DIR, PYTHON_VERSION_FILE, "^__version__ =", releases, lambda: pypi_published(python_version), ), packages=packages, nodejs_version=args.nodejs_version, python_version=args.python_version, ) ) selected_pins = [ pins for name, pins in [("nodejs", result.nodejs_pins), ("python", result.python_pins)] if name in packages ] if problems := [problem for pins in selected_pins for problem in published_pin_problems(pins)]: sys.exit("\n".join(problems)) assets = { (tag, asset, pins.sha256[asset]) for pins in selected_pins for tag in pinned_tags(pins) for asset in release_assets(tag) } with tempfile.TemporaryDirectory() as directory: for tag, asset, sha256 in sorted(assets): download(tag, asset, directory, sha256) print(f"checked {asset} of {tag}", file=sys.stderr) if "nodejs" in packages: (ROOT / RELEASE_FILE).write_text(render_release(result.nodejs_pins)) dependencies = nodejs["dependencies"] if result.types_dependency: dependencies = {**dependencies, types_name: result.types_dependency} write_json(f"{NODEJS_DIR}/package.json", {**nodejs, "version": result.nodejs_version, "dependencies": dependencies}) if "python" in packages: (ROOT / PYTHON_VERSION_FILE).write_text(render_python_version(result.python_version, result.python_pins)) print(result.title + "\n\n" + "".join(f"- {note}\n" for note in result.notes), end="") def publish_plan() -> None: """A recorded version is published: npm and PyPI list a new version minutes after it is published.""" releases, _ = read_releases() for key, directory in [("types", TYPES_DIR), ("nodejs", NODEJS_DIR)]: package = read_json(f"{directory}/package.json") published = recorded_commit(releases, npm_key(directory), package["version"]) or npm_published( package["name"], package["version"] ) print(f"{key}_version={'' if published else package['version']}") print(f"{key}_dist_tag={dist_tag(package['version'])}") version, _ = parse_python_version(read_text(PYTHON_VERSION_FILE)) published = recorded_commit(releases, pypi_key(), version) or pypi_published(version) print(f"python_version={'' if published else version}") def record(args: argparse.Namespace) -> None: versions = [ (core_key(args.branch), args.core), (npm_key(TYPES_DIR), args.types), (npm_key(NODEJS_DIR), args.nodejs), (pypi_key(), args.python), ] entries = [(key, version) for key, version in versions if version] if not entries: sys.exit("nothing to record") print(record_releases(entries, args.commit)) def test_names() -> None: if scope := os.environ.get("NPM_SCOPE"): # npm provenance requires the repository of the publishing workflow. url = f"git+https://github.com/{os.environ['GITHUB_REPOSITORY']}.git" types = read_json(f"{TYPES_DIR}/package.json") write_json( f"{TYPES_DIR}/package.json", {**types, "name": registry_name(types["name"], scope), "repository": {**types["repository"], "url": url}}, ) nodejs = read_json(f"{NODEJS_DIR}/package.json") alias = f"npm:{registry_name(types['name'], scope)}@{nodejs['dependencies'][types['name']]}" write_json( f"{NODEJS_DIR}/package.json", { **nodejs, "name": registry_name(nodejs["name"], scope), "repository": {**nodejs["repository"], "url": url}, "dependencies": {**nodejs["dependencies"], types["name"]: alias}, }, ) if project := os.environ.get("PYPI_PROJECT"): pyproject = ROOT / PYTHON_DIR / "pyproject.toml" pyproject.write_text( re.sub(r'^name = "[^"]+"', f'name = "{project}"', pyproject.read_text(), count=1, flags=re.M) ) def print_release_status(tag: str) -> None: print(f"status={release_status(tag_release(libs_releases(), tag))}") def print_addon_release() -> None: sources_hash = current_addon_hash() release = addon_release(libs_releases(), sources_hash) print(f"notes={shlex.quote(addon_notes(sources_hash))}") print(f"status={release_status(release)}") print(f"tag={release['tag_name'] if release else ''}") def publish_release(tag: str) -> None: release = tag_release(libs_releases(), tag) if release is None: sys.exit(f"{tag}: no release") if not release["draft"]: print(f"{tag} is published") return if missing := missing_assets(release): sys.exit(f"{tag}: missing {', '.join(missing)}") libs_api(f"releases/{release['id']}", "PATCH", {"draft": False}) print(f"{tag} published") def check_pins() -> None: _, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE)) problems = published_pin_problems(parse_release(read_text(RELEASE_FILE))) + published_pin_problems(python_pins) if problems: sys.exit("\n".join(problems)) print("the pins match the published releases") def download_asset(tag: str, asset: str, directory: str) -> None: digests = release_digests(github_release(tag), [asset]) if digests is None: sys.exit(f"{asset} of {tag} is not published with a digest") download(tag, asset, directory, digests[asset]) def download_pinned(asset: str, directory: str) -> None: pins = parse_release(read_text(RELEASE_FILE)) tag = next((tag for tag in pinned_tags(pins) if asset in release_assets(tag)), None) if tag is None or asset not in pins.sha256: sys.exit(f"{asset} is not pinned in {RELEASE_FILE}") download(tag, asset, directory, pins.sha256[asset]) def main(argv: list[str]) -> None: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) commands = parser.add_subparsers(dest="command", required=True) branch_help = "the branch of the core versions, the checked out branch by default" core_release_parser = commands.add_parser("core-release") core_release_parser.add_argument("version", help="the new core version") core_release_parser.add_argument("--branch", help=branch_help) core_release_parser.add_argument("--types-version", help="this types version instead of a bump when needed") claim_parser = commands.add_parser("claim-core") claim_parser.add_argument("version", help="the core version of simplex-chat.cabal") claim_parser.add_argument("commit", help="the simplex-chat commit to release it from") claim_parser.add_argument("branch", help="the branch of the commit") status_parser = commands.add_parser("release-status") status_parser.add_argument("tag") commands.add_parser("addon-release") latest_libs_parser = commands.add_parser("latest-libs") latest_libs_parser.add_argument("--branch", help=branch_help) publish_release_parser = commands.add_parser("publish-release") publish_release_parser.add_argument("tag") commands.add_parser("check-pins") download_parser = commands.add_parser("download") download_parser.add_argument("tag") download_parser.add_argument("asset") download_parser.add_argument("directory") download_pinned_parser = commands.add_parser("download-pinned") download_pinned_parser.add_argument("asset") download_pinned_parser.add_argument("directory") changes_parser = commands.add_parser("changes") changes_parser.add_argument("base", help="the commit to compare with") build_parser = commands.add_parser("build-plan") build_parser.add_argument("--base", required=True, help="the base commit of the pull request") prepare_parser = commands.add_parser("prepare") prepare_parser.add_argument("packages", nargs="*", choices=["nodejs", "python"], help="all when none is listed") prepare_parser.add_argument("--branch", help=branch_help) for package in ["nodejs", "python"]: prepare_parser.add_argument(f"--{package}-version", help="this version instead of a bump when needed") commands.add_parser("publish-plan") record_parser = commands.add_parser("record") record_parser.add_argument("--commit", required=True, help="the simplex-chat commit released or published from") record_parser.add_argument("--branch", default="master", help="the branch of the core version") for name in ["core", "types", "nodejs", "python"]: record_parser.add_argument(f"--{name}", default="", help=f"{name} version, recorded when not empty") commands.add_parser("test-names") args = parser.parse_args(argv) if args.command == "core-release": core_release(args.version, args.branch or current_branch(), args.types_version) elif args.command == "claim-core": claim_core(args.version, args.commit, args.branch) elif args.command == "release-status": print_release_status(args.tag) elif args.command == "addon-release": print_addon_release() elif args.command == "latest-libs": releases, _ = read_releases() versions = recorded_versions(releases, core_key(args.branch or current_branch())) print(f"tag={latest_libs(libs_releases(), versions) or ''}") elif args.command == "publish-release": publish_release(args.tag) elif args.command == "check-pins": check_pins() elif args.command == "download": download_asset(args.tag, args.asset, args.directory) elif args.command == "download-pinned": download_pinned(args.asset, args.directory) elif args.command == "changes": changes(args.base) elif args.command == "build-plan": print_build_plan(args) elif args.command == "prepare": prepare(args) elif args.command == "publish-plan": publish_plan() elif args.command == "record": record(args) else: test_names() if __name__ == "__main__": main(sys.argv[1:])