mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-10-21 18:43:22 +00:00
EvgenyandEvgeny @ SimpleX Chat
a766d0af0f
* ci: package releases with drafts in simplex-chat-libs * test release * core: 7.1.0.12 * packages: release nodejs and python libraries * fixes --------- Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>
1001 lines
40 KiB
Python
1001 lines
40 KiB
Python
#!/usr/bin/env python3
|
|
"""Release automation for simplex-chat and @simplex-chat/types on npm and simplex-chat on PyPI.
|
|
|
|
core-release set the core version and the types version for a library release
|
|
claim-core claim the library release of the core version for a commit
|
|
release-status print whether a release of simplex-chat-libs is published, a draft or absent
|
|
addon-release print the add-on release of the current add-on sources
|
|
latest-libs print the newest published library release
|
|
publish-release publish a draft release that has every asset
|
|
check-pins check the pins of the packages against the published releases
|
|
download download a release asset and check its digest
|
|
download-pinned download a pinned asset and check its pin
|
|
changes print whether the changes since a commit touch the chat core and the add-on
|
|
build-plan print the platforms to build the add-on on and whether to run the package tests
|
|
prepare update pins and versions of the packages
|
|
publish-plan print the package versions that are not published yet
|
|
record record releases and published versions in releases.toml of simplex-chat-libs
|
|
test-names rename the packages to NPM_SCOPE and PYPI_PROJECT (test repositories)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import tomllib
|
|
import urllib.error
|
|
import urllib.request
|
|
from collections.abc import Callable, Collection
|
|
from dataclasses import dataclass, field, replace
|
|
from typing import Any
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
NODEJS_DIR = "packages/simplex-chat-nodejs"
|
|
TYPES_DIR = "packages/simplex-chat-client/types/typescript"
|
|
PYTHON_DIR = "packages/simplex-chat-python"
|
|
RELEASE_FILE = f"{NODEJS_DIR}/src/release.ts"
|
|
PYTHON_VERSION_FILE = f"{PYTHON_DIR}/src/simplex_chat/_version.py"
|
|
ADDON_SOURCES = [f"{NODEJS_DIR}/binding.gyp", f"{NODEJS_DIR}/cpp"]
|
|
CORE_SOURCES = ["src", "simplex-chat.cabal", "cabal.project"]
|
|
CABAL_FILE = "simplex-chat.cabal"
|
|
CABAL_VERSION = re.compile(r"^(version:\s*)(\S+)", re.M)
|
|
TYPES_SOURCES = [f"{TYPES_DIR}/src"]
|
|
TYPES_README = f"{TYPES_DIR}/README.md"
|
|
TYPES_README_CORE = re.compile(r"^SimpleX Chat core: .*$", re.M)
|
|
PYTHON_TYPES_SOURCES = [f"{PYTHON_DIR}/src/simplex_chat/types"]
|
|
ADDON_HASH_LENGTH = 12
|
|
CORE_TAG = re.compile(r"^core-(\d+(?:\.\d+)*)$")
|
|
ADDON_TAG_PREFIX = "nodejs-addon-"
|
|
LIBS_ASSETS = [
|
|
"simplex-chat-libs-linux-aarch64.zip",
|
|
"simplex-chat-libs-linux-x86_64-postgres.zip",
|
|
"simplex-chat-libs-linux-x86_64.zip",
|
|
"simplex-chat-libs-macos-aarch64.zip",
|
|
"simplex-chat-libs-macos-x86_64.zip",
|
|
"simplex-chat-libs-windows-x86_64.zip",
|
|
]
|
|
ADDON_RUNNERS = {
|
|
"linux-aarch64": "ubuntu-22.04-arm",
|
|
"linux-x86_64": "ubuntu-22.04",
|
|
"macos-aarch64": "macos-latest",
|
|
"macos-x86_64": "macos-15-intel",
|
|
"windows-x86_64": "windows-latest",
|
|
}
|
|
ADDON_PLATFORMS = list(ADDON_RUNNERS)
|
|
ADDON_ASSETS = [f"simplex-chat-nodejs-{platform}.node" for platform in ADDON_PLATFORMS]
|
|
SHA256_HEX = re.compile(r"^[0-9a-f]{64}$")
|
|
RELEASES_FILE = "releases.toml"
|
|
RECORD_ATTEMPTS = 5
|
|
DOWNLOAD_CHUNK = 1 << 20
|
|
TOML_BARE_KEY = re.compile(r"^[A-Za-z0-9_-]+$")
|
|
|
|
|
|
# Pure functions
|
|
|
|
|
|
def addon_hash(files: dict[str, bytes], node_addon_api: str) -> str:
|
|
digest = hashlib.sha256()
|
|
for name in sorted(files):
|
|
digest.update(name.encode() + b"\0" + files[name] + b"\0")
|
|
digest.update(f"node-addon-api@{node_addon_api}".encode())
|
|
return digest.hexdigest()[:ADDON_HASH_LENGTH]
|
|
|
|
|
|
def addon_notes(sources_hash: str) -> str:
|
|
return f"sources: {sources_hash}"
|
|
|
|
|
|
def version_key(version: str) -> tuple[int, ...]:
|
|
return tuple(int(part) for part in version.split("."))
|
|
|
|
|
|
def bump_npm_version(version: str) -> str:
|
|
"""A package release of app version 7.1.0-beta.6 is 7.1.0-beta.6.1, then 7.1.0-beta.6.2."""
|
|
_, _, prerelease = version.partition("-")
|
|
if len(prerelease.split(".")) == 2:
|
|
return f"{version}.1"
|
|
head, _, last = version.rpartition(".")
|
|
return f"{head}.{int(last) + 1}" if last.isdigit() else f"{version}.1"
|
|
|
|
|
|
def bump_python_version(version: str) -> str:
|
|
"""A package release of app version 7.1.0b6 is 7.1.0b6.post1, then 7.1.0b6.post2."""
|
|
post = re.search(r"\.post(\d+)$", version)
|
|
return f"{version[: post.start()]}.post{int(post[1]) + 1}" if post else f"{version}.post1"
|
|
|
|
|
|
def dist_tag(version: str) -> str:
|
|
return "beta" if "-" in version else "latest"
|
|
|
|
|
|
def registry_name(name: str, scope: str | None) -> str:
|
|
"""Package name on npm: test repositories publish under their own scope."""
|
|
return f"{scope}/{name.split('/')[-1]}" if scope else name
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Pins:
|
|
libs_release: str
|
|
sha256: dict[str, str]
|
|
addon_release: str | None = None
|
|
|
|
|
|
def parse_release(source: str) -> Pins:
|
|
libs = re.search(r'export const LIBS_RELEASE = "([^"]+)"', source)
|
|
addon = re.search(r'export const ADDON_RELEASE: string \| undefined = (?:"([^"]+)"|undefined)', source)
|
|
if not libs or not addon:
|
|
raise ValueError(f"cannot parse {RELEASE_FILE}")
|
|
hashes = dict(re.findall(r'^ {2}"([^"]+)": "([0-9a-f]{64})",$', source, re.M))
|
|
return Pins(libs[1], hashes, addon[1])
|
|
|
|
|
|
def render_release(pins: Pins) -> str:
|
|
hashes = "".join(f' "{asset}": "{pins.sha256[asset]}",\n' for asset in sorted(pins.sha256))
|
|
addon = f'"{pins.addon_release}"' if pins.addon_release else "undefined"
|
|
return (
|
|
"// Written by scripts/ci/packages_release.py from the simplex-chat-libs releases.\n"
|
|
"\n"
|
|
f'export const LIBS_RELEASE = "{pins.libs_release}"\n'
|
|
"\n"
|
|
f"export const ADDON_RELEASE: string | undefined = {addon}\n"
|
|
"\n"
|
|
"export const SHA256: {readonly [asset: string]: string} = {\n"
|
|
f"{hashes}}}\n"
|
|
)
|
|
|
|
|
|
def parse_python_version(source: str) -> tuple[str, Pins]:
|
|
version = re.search(r'^__version__ = "([^"]+)"', source, re.M)
|
|
libs = re.search(r'^LIBS_RELEASE = "([^"]+)"', source, re.M)
|
|
if not version or not libs:
|
|
raise ValueError(f"cannot parse {PYTHON_VERSION_FILE}")
|
|
hashes = dict(re.findall(r'^ {4}"([^"]+)": "([0-9a-f]{64})",$', source, re.M))
|
|
return version[1], Pins(libs[1], hashes)
|
|
|
|
|
|
def render_python_version(version: str, pins: Pins) -> str:
|
|
hashes = "".join(f' "{asset}": "{pins.sha256[asset]}",\n' for asset in sorted(pins.sha256))
|
|
return (
|
|
'"""Package version and the simplex-chat-libs release it loads.\n'
|
|
"\n"
|
|
"Written by scripts/ci/packages_release.py prepare: it bumps the version\n"
|
|
"and pins new library releases with the SHA-256 of their assets.\n"
|
|
'"""\n'
|
|
"\n"
|
|
f'__version__ = "{version}" # PEP 440 — read by hatchling for wheel metadata\n'
|
|
f'LIBS_RELEASE = "{pins.libs_release}" # simplex-chat-libs release tag\n'
|
|
"LIBS_SHA256 = {\n"
|
|
f"{hashes}}}\n"
|
|
)
|
|
|
|
|
|
def asset_digests(release: dict) -> dict[str, str]:
|
|
return {a["name"]: (a.get("digest") or "").removeprefix("sha256:") for a in release["assets"]}
|
|
|
|
|
|
def release_digests(release: dict | None, assets: list[str]) -> dict[str, str] | None:
|
|
"""Asset digests of a GitHub release, or None when the release is missing or incomplete."""
|
|
if release is None:
|
|
return None
|
|
digests = asset_digests(release)
|
|
if not all(SHA256_HEX.match(digests.get(asset, "")) for asset in assets):
|
|
return None
|
|
return {asset: digests[asset] for asset in assets}
|
|
|
|
|
|
def release_assets(tag: str) -> list[str]:
|
|
if CORE_TAG.match(tag):
|
|
return LIBS_ASSETS
|
|
if tag.startswith(ADDON_TAG_PREFIX):
|
|
return ADDON_ASSETS
|
|
raise ValueError(f"no asset set for release {tag}")
|
|
|
|
|
|
def preferred_release(releases: list[dict]) -> dict | None:
|
|
"""The published release among them, otherwise the first draft."""
|
|
return next((r for r in releases if not r["draft"]), next(iter(releases), None))
|
|
|
|
|
|
def release_status(release: dict | None) -> str:
|
|
if release is None:
|
|
return "absent"
|
|
return "draft" if release["draft"] else "published"
|
|
|
|
|
|
def tag_release(releases: list[dict], tag: str) -> dict | None:
|
|
return preferred_release([r for r in releases if r["tag_name"] == tag])
|
|
|
|
|
|
def addon_release(releases: list[dict], sources_hash: str) -> dict | None:
|
|
notes = addon_notes(sources_hash)
|
|
return preferred_release(
|
|
[r for r in releases if r["tag_name"].startswith(ADDON_TAG_PREFIX) and r.get("body") == notes]
|
|
)
|
|
|
|
|
|
def core_version(tag: str) -> str | None:
|
|
match = CORE_TAG.match(tag)
|
|
return match[1] if match else None
|
|
|
|
|
|
def latest_libs(releases: list[dict], versions: Collection[str]) -> str | None:
|
|
"""The newest published release of the core versions that has every library asset."""
|
|
tags = [
|
|
r["tag_name"]
|
|
for r in releases
|
|
if not r["draft"] and core_version(r["tag_name"]) in versions and release_digests(r, LIBS_ASSETS)
|
|
]
|
|
return max(tags, key=lambda tag: version_key(core_version(tag)), default=None)
|
|
|
|
|
|
def missing_assets(release: dict) -> list[str]:
|
|
present = {a["name"] for a in release["assets"]}
|
|
return [asset for asset in release_assets(release["tag_name"]) if asset not in present]
|
|
|
|
|
|
def pin_problems(pins: Pins, published: dict[str, dict | None]) -> list[str]:
|
|
"""Differences between the pins and the published releases of the pinned tags."""
|
|
problems = []
|
|
for tag, release in published.items():
|
|
if release is None:
|
|
problems.append(f"{tag} is not published")
|
|
continue
|
|
digests = asset_digests(release)
|
|
for asset in release_assets(tag):
|
|
if pins.sha256.get(asset) != digests.get(asset):
|
|
problems.append(f"{asset} of {tag}: pinned {pins.sha256.get(asset)}, published {digests.get(asset)}")
|
|
return problems
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Package:
|
|
version: str
|
|
changed: bool
|
|
published: bool
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class State:
|
|
libs_tag: str
|
|
libs_digests: dict[str, str] | None
|
|
addon_tag: str
|
|
addon_digests: dict[str, str] | None
|
|
nodejs_pins: Pins
|
|
python_pins: Pins
|
|
types_dependency: str
|
|
nodejs_types_dependency: str
|
|
nodejs: Package
|
|
python: Package
|
|
packages: frozenset[str] = frozenset({"nodejs", "python"})
|
|
nodejs_version: str | None = None
|
|
python_version: str | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Plan:
|
|
nodejs_pins: Pins
|
|
python_pins: Pins
|
|
types_dependency: str | None
|
|
nodejs_version: str
|
|
nodejs_changed: bool
|
|
python_version: str
|
|
python_changed: bool
|
|
notes: list[str] = field(default_factory=list)
|
|
|
|
@property
|
|
def title(self) -> str:
|
|
parts = []
|
|
if self.nodejs_changed:
|
|
parts.append(f"nodejs {self.nodejs_version}")
|
|
if self.python_changed:
|
|
parts.append(f"python {self.python_version}")
|
|
return f"packages: release {', '.join(parts)}"
|
|
|
|
|
|
def plan(state: State) -> Plan:
|
|
"""What a package release changes, from the state of the branch and the releases."""
|
|
notes: list[str] = []
|
|
|
|
def pin(pins: Pins, field_name: str, tag: str, digests: dict[str, str] | None, what: str) -> tuple[Pins, bool]:
|
|
current = getattr(pins, field_name)
|
|
if current == tag:
|
|
return pins, False
|
|
if digests is None:
|
|
notes.append(f"{what} release {tag} is missing or incomplete; {current or 'none'} stays pinned")
|
|
return pins, False
|
|
notes.append(f"{what}: {tag}")
|
|
return replace(pins, **{field_name: tag, "sha256": {**pins.sha256, **digests}}), True
|
|
|
|
def next_version(pkg: Package, changed: bool, override: str | None, bump) -> str:
|
|
if override:
|
|
return override
|
|
return bump(pkg.version) if changed and pkg.published else pkg.version
|
|
|
|
nodejs_pins, nodejs_version, nodejs_changed, types_dependency = state.nodejs_pins, state.nodejs.version, False, None
|
|
if "nodejs" in state.packages:
|
|
nodejs_pins, nodejs_libs = pin(
|
|
state.nodejs_pins, "libs_release", state.libs_tag, state.libs_digests, "Node.js libsimplex"
|
|
)
|
|
nodejs_pins, addon = pin(nodejs_pins, "addon_release", state.addon_tag, state.addon_digests, "Node.js add-on")
|
|
if state.types_dependency != state.nodejs_types_dependency:
|
|
types_dependency = state.types_dependency
|
|
notes.append(f"@simplex-chat/types: {types_dependency}")
|
|
nodejs_changed = nodejs_libs or addon or types_dependency is not None or state.nodejs.changed
|
|
nodejs_version = next_version(state.nodejs, nodejs_changed, state.nodejs_version, bump_npm_version)
|
|
python_pins, python_version, python_changed = state.python_pins, state.python.version, False
|
|
if "python" in state.packages:
|
|
python_pins, python_libs = pin(
|
|
state.python_pins, "libs_release", state.libs_tag, state.libs_digests, "Python libsimplex"
|
|
)
|
|
python_changed = python_libs or state.python.changed
|
|
python_version = next_version(state.python, python_changed, state.python_version, bump_python_version)
|
|
for name, old, new in [
|
|
("simplex-chat (npm)", state.nodejs.version, nodejs_version),
|
|
("simplex-chat (PyPI)", state.python.version, python_version),
|
|
]:
|
|
if old != new:
|
|
notes.append(f"{name}: {old} → {new}")
|
|
return Plan(
|
|
nodejs_pins=nodejs_pins,
|
|
python_pins=python_pins,
|
|
types_dependency=types_dependency,
|
|
nodejs_version=nodejs_version,
|
|
nodejs_changed=nodejs_changed or nodejs_version != state.nodejs.version,
|
|
python_version=python_version,
|
|
python_changed=python_changed or python_version != state.python.version,
|
|
notes=notes,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BuildPlan:
|
|
platforms: list[str]
|
|
test: bool
|
|
|
|
|
|
def build_plan(core_changed: bool, addon_changed: bool) -> BuildPlan:
|
|
"""The package tests of a pull request that changes the core run in build.yml, with its own libsimplex."""
|
|
if core_changed:
|
|
return BuildPlan(ADDON_PLATFORMS if addon_changed else [], test=False)
|
|
return BuildPlan(ADDON_PLATFORMS, test=True)
|
|
|
|
|
|
def build_matrix(platforms: list[str]) -> str:
|
|
return json.dumps([{"platform": platform, "os": ADDON_RUNNERS[platform]} for platform in platforms])
|
|
|
|
|
|
def core_key(branch: str) -> str:
|
|
return f"core/{branch}"
|
|
|
|
|
|
def recorded_versions(releases: dict, key: str) -> dict[str, str]:
|
|
"""The recorded versions and their simplex-chat commits; key is "core/<branch>", "npm/<name>" or "pypi/<name>"."""
|
|
table = releases
|
|
for part in key.split("/", 1):
|
|
table = table.get(part, {})
|
|
return {version: commit for version, commit in table.items() if isinstance(commit, str)}
|
|
|
|
|
|
def recorded_commit(releases: dict, key: str, version: str) -> str | None:
|
|
return recorded_versions(releases, key).get(version)
|
|
|
|
|
|
def highest_recorded(releases: dict, key: str) -> str | None:
|
|
return max(recorded_versions(releases, key), key=version_key, default=None)
|
|
|
|
|
|
def recorded_core_commit(releases: dict, version: str) -> str | None:
|
|
"""The commit recorded for the core version on any branch: a release tag is shared by the branches."""
|
|
for branch in releases.get("core", {}):
|
|
if commit := recorded_commit(releases, core_key(branch), version):
|
|
return commit
|
|
return None
|
|
|
|
|
|
def previous_core_commit(releases: dict, branch: str) -> str | None:
|
|
"""The commit of the highest core version recorded for the branch."""
|
|
previous = highest_recorded(releases, core_key(branch))
|
|
return recorded_commit(releases, core_key(branch), previous) if previous else None
|
|
|
|
|
|
def add_releases(releases: dict, entries: list[tuple[str, str]], commit: str) -> dict:
|
|
"""A recorded version keeps its first commit: re-running a release does not rewrite history."""
|
|
result = copy.deepcopy(releases)
|
|
for key, version in entries:
|
|
table = result
|
|
for part in key.split("/", 1):
|
|
table = table.setdefault(part, {})
|
|
table.setdefault(version, commit)
|
|
return result
|
|
|
|
|
|
def render_releases(releases: dict) -> str:
|
|
def toml_key(key: str) -> str:
|
|
return key if TOML_BARE_KEY.match(key) else json.dumps(key)
|
|
|
|
sections: list[str] = []
|
|
|
|
# in the order of recording: entries are only added, so that order is chronological
|
|
def walk(path: list[str], table: dict) -> None:
|
|
values = [(k, v) for k, v in table.items() if isinstance(v, str)]
|
|
if values:
|
|
lines = [f"[{'.'.join(toml_key(part) for part in path)}]"]
|
|
lines += [f"{toml_key(version)} = {json.dumps(commit)}" for version, commit in values]
|
|
sections.append("\n".join(lines) + "\n")
|
|
for name, child in table.items():
|
|
if isinstance(child, dict):
|
|
walk([*path, name], child)
|
|
|
|
walk([], releases)
|
|
return "\n".join(sections)
|
|
|
|
|
|
# Repository and registry access
|
|
|
|
|
|
def git(*args: str) -> str:
|
|
return subprocess.run(["git", *args], cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip()
|
|
|
|
|
|
def read_text(file: str) -> str:
|
|
return (ROOT / file).read_text()
|
|
|
|
|
|
def read_field(file: str, pattern: str) -> str:
|
|
found = re.search(pattern, read_text(file), re.M)
|
|
if not found:
|
|
raise ValueError(f"no match for {pattern} in {file}")
|
|
return found[1]
|
|
|
|
|
|
def read_json(file: str) -> dict:
|
|
return json.loads(read_text(file))
|
|
|
|
|
|
def write_json(file: str, value: dict) -> None:
|
|
(ROOT / file).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n")
|
|
|
|
|
|
def current_addon_hash() -> str:
|
|
files = {
|
|
path.relative_to(ROOT).as_posix(): path.read_bytes()
|
|
for source in ADDON_SOURCES
|
|
for path in ([ROOT / source] if (ROOT / source).is_file() else sorted((ROOT / source).rglob("*")))
|
|
if path.is_file()
|
|
}
|
|
return addon_hash(files, read_json(f"{NODEJS_DIR}/package.json")["devDependencies"]["node-addon-api"])
|
|
|
|
|
|
def cabal_version() -> str:
|
|
return CABAL_VERSION.search(read_text(CABAL_FILE))[2]
|
|
|
|
|
|
def changed_since(base: str, sources: list[str]) -> bool:
|
|
diff = subprocess.run(["git", "diff", "--quiet", base, "HEAD", "--", *sources], cwd=ROOT)
|
|
return diff.returncode != 0
|
|
|
|
|
|
def changes(base: str) -> None:
|
|
print(f"core={'true' if changed_since(base, CORE_SOURCES) else ''}")
|
|
print(f"addon={'true' if changed_since(base, ADDON_SOURCES) else ''}")
|
|
|
|
|
|
def print_build_plan(args: argparse.Namespace) -> None:
|
|
result = build_plan(
|
|
core_changed=changed_since(args.base, CORE_SOURCES),
|
|
addon_changed=changed_since(args.base, ADDON_SOURCES),
|
|
)
|
|
print(f"platforms={build_matrix(result.platforms)}")
|
|
print(f"test={'true' if result.test else ''}")
|
|
|
|
|
|
def package_state(
|
|
key: str,
|
|
version: str,
|
|
directory: str,
|
|
version_file: str,
|
|
version_pattern: str,
|
|
releases: dict,
|
|
registry_published: Callable[[], bool],
|
|
) -> Package:
|
|
"""A recorded version is published from its recorded commit; the registry is asked about other versions,
|
|
which are compared with the commit that set them."""
|
|
recorded = recorded_commit(releases, key, version)
|
|
base = (
|
|
recorded
|
|
if recorded and commit_exists(recorded)
|
|
else git("log", "-1", "--format=%H", "-G", version_pattern, "--", version_file)
|
|
)
|
|
changed = changed_since(base, [directory, f":!{directory}/tests"])
|
|
return Package(version, changed, recorded is not None or registry_published())
|
|
|
|
|
|
def http_json(url: str, headers: dict[str, str] | None = None, method: str = "GET", body: dict | None = None) -> Any:
|
|
"""The JSON response, or None for 404."""
|
|
request = urllib.request.Request(
|
|
url,
|
|
headers={"User-Agent": "simplex-chat-release", **(headers or {})},
|
|
method=method,
|
|
data=json.dumps(body).encode() if body is not None else None,
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=60) as response:
|
|
return json.load(response)
|
|
except urllib.error.HTTPError as e:
|
|
if e.code == 404:
|
|
return None
|
|
raise
|
|
|
|
|
|
# Workflows pass unset repository variables as empty strings.
|
|
def npm_package_url(name: str) -> str:
|
|
registry = os.environ.get("NPM_REGISTRY") or "https://registry.npmjs.org"
|
|
return f"{registry}/{registry_name(name, os.environ.get('NPM_SCOPE')).replace('/', '%2F')}"
|
|
|
|
|
|
def npm_published(name: str, version: str) -> bool:
|
|
return http_json(f"{npm_package_url(name)}/{version}") is not None
|
|
|
|
|
|
def python_project() -> str:
|
|
return read_field(f"{PYTHON_DIR}/pyproject.toml", r'^name = "([^"]+)"')
|
|
|
|
|
|
def pypi_published(version: str) -> bool:
|
|
index = os.environ.get("PYPI_URL") or "https://pypi.org"
|
|
project = os.environ.get("PYPI_PROJECT") or python_project()
|
|
return http_json(f"{index}/pypi/{project}/{version}/json") is not None
|
|
|
|
|
|
def libs_repository() -> str:
|
|
return os.environ.get("LIBS_REPOSITORY") or "simplex-chat/simplex-chat-libs"
|
|
|
|
|
|
def libs_api(path: str, method: str = "GET", body: dict | None = None) -> Any:
|
|
"""GitHub API of the simplex-chat-libs repository."""
|
|
headers = {"Accept": "application/vnd.github+json"}
|
|
if token := os.environ.get("GH_TOKEN"):
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
return http_json(f"https://api.github.com/repos/{libs_repository()}/{path}", headers, method, body)
|
|
|
|
|
|
def github_release(tag: str) -> dict | None:
|
|
"""The published release of the tag."""
|
|
return libs_api(f"releases/tags/{tag}")
|
|
|
|
|
|
def libs_releases() -> list[dict]:
|
|
"""The releases of simplex-chat-libs; drafts are listed only with a token that can push to it."""
|
|
releases: list[dict] = []
|
|
page = 1
|
|
while batch := libs_api(f"releases?per_page=100&page={page}"):
|
|
releases += batch
|
|
page += 1
|
|
return releases
|
|
|
|
|
|
def download(tag: str, asset: str, directory: str, sha256: str) -> Path:
|
|
"""Saves the asset of the release to the directory; exits when its SHA-256 differs."""
|
|
url = f"https://github.com/{libs_repository()}/releases/download/{tag}/{asset}"
|
|
target = Path(directory) / asset
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
digest = hashlib.sha256()
|
|
request = urllib.request.Request(url, headers={"User-Agent": "simplex-chat-release"})
|
|
with urllib.request.urlopen(request, timeout=600) as response, target.open("wb") as file:
|
|
while chunk := response.read(DOWNLOAD_CHUNK):
|
|
file.write(chunk)
|
|
digest.update(chunk)
|
|
if digest.hexdigest() != sha256:
|
|
target.unlink()
|
|
sys.exit(f"SHA-256 of {asset} of {tag} is {digest.hexdigest()}, expected {sha256}")
|
|
return target
|
|
|
|
|
|
def read_releases() -> tuple[dict, str | None]:
|
|
"""The releases and the blob of the file, which a change must name."""
|
|
file = libs_api(f"contents/{RELEASES_FILE}")
|
|
if file is None:
|
|
return {}, None
|
|
return tomllib.loads(base64.b64decode(file["content"]).decode()), file["sha"]
|
|
|
|
|
|
def record_releases(entries: list[tuple[str, str]], commit: str) -> str:
|
|
"""Adds the entries in one commit of simplex-chat-libs and returns that commit."""
|
|
message = ", ".join(f"{key} {version}" for key, version in entries)
|
|
attempt = 1
|
|
while True:
|
|
releases, blob = read_releases()
|
|
updated = add_releases(releases, entries, commit)
|
|
if blob and updated == releases:
|
|
return libs_api(f"commits?path={RELEASES_FILE}&per_page=1")[0]["sha"]
|
|
body = {
|
|
"message": message,
|
|
"content": base64.b64encode(render_releases(updated).encode()).decode(),
|
|
"committer": {"name": "SimpleX Chat", "email": "build@simplex.chat"},
|
|
**({"sha": blob} if blob else {}),
|
|
}
|
|
try:
|
|
return libs_api(f"contents/{RELEASES_FILE}", "PUT", body)["commit"]["sha"]
|
|
except urllib.error.HTTPError as e:
|
|
# 409 and 422: another writer changed or created the file since it was read
|
|
if e.code not in (409, 422) or attempt == RECORD_ATTEMPTS:
|
|
raise
|
|
attempt += 1
|
|
|
|
|
|
def commit_exists(commit: str) -> bool:
|
|
found = subprocess.run(["git", "cat-file", "-e", f"{commit}^{{commit}}"], cwd=ROOT, capture_output=True)
|
|
return found.returncode == 0
|
|
|
|
|
|
def fetch_commit(commit: str) -> str:
|
|
"""The commit, fetched from origin when the checkout lacks it."""
|
|
if not commit_exists(commit):
|
|
git("fetch", "-q", "--depth=1", "origin", commit)
|
|
return commit
|
|
|
|
|
|
def npm_key(directory: str) -> str:
|
|
return f"npm/{read_json(f'{directory}/package.json')['name']}"
|
|
|
|
|
|
def pypi_key() -> str:
|
|
return f"pypi/{python_project()}"
|
|
|
|
|
|
def npm_package(directory: str, releases: dict) -> Package:
|
|
package = read_json(f"{directory}/package.json")
|
|
name, version = package["name"], package["version"]
|
|
return package_state(
|
|
npm_key(directory),
|
|
version,
|
|
directory,
|
|
f"{directory}/package.json",
|
|
'"version":',
|
|
releases,
|
|
lambda: npm_published(name, version),
|
|
)
|
|
|
|
|
|
def types_changed(releases: dict, branch: str) -> bool:
|
|
"""Whether the types sources differ from those of the previous core release on the branch."""
|
|
previous = previous_core_commit(releases, branch)
|
|
return previous is None or changed_since(fetch_commit(previous), TYPES_SOURCES)
|
|
|
|
|
|
def types_problem(version: str, releases: dict, branch: str) -> str | None:
|
|
"""Why the types can't be released with core <version>, or None when they can."""
|
|
if not types_changed(releases, branch):
|
|
return None
|
|
types = read_json(f"{TYPES_DIR}/package.json")
|
|
if recorded_commit(releases, npm_key(TYPES_DIR), types["version"]) or npm_published(types["name"], types["version"]):
|
|
return f"{types['name']} changed, and its version {types['version']} is published"
|
|
if types.get("simplexChatCore") != version:
|
|
return f"simplexChatCore of {types['name']} is {types.get('simplexChatCore')}, expected {version}"
|
|
return None
|
|
|
|
|
|
def claim_core(version: str, commit: str, branch: str) -> None:
|
|
releases, _ = read_releases()
|
|
highest = highest_recorded(releases, core_key(branch))
|
|
if highest and version_key(version) < version_key(highest):
|
|
sys.exit(f"core {version} is lower than core {highest} recorded for {branch}")
|
|
if recorded := recorded_core_commit(releases, version):
|
|
print(f"claim={'resume' if recorded == commit else 'skip'}")
|
|
return
|
|
if problem := types_problem(version, releases, branch):
|
|
sys.exit(problem)
|
|
libs_commit = record_releases([(core_key(branch), version)], commit)
|
|
releases, _ = read_releases()
|
|
if recorded_core_commit(releases, version) == commit:
|
|
print("claim=new")
|
|
print(f"commit={libs_commit}")
|
|
else:
|
|
print("claim=skip")
|
|
|
|
|
|
def current_branch() -> str:
|
|
return git("rev-parse", "--abbrev-ref", "HEAD")
|
|
|
|
|
|
def core_release(version: str, branch: str, types_version: str | None) -> None:
|
|
releases, _ = read_releases()
|
|
highest = highest_recorded(releases, core_key(branch))
|
|
if highest and version_key(version) <= version_key(highest):
|
|
sys.exit(f"core {version} is not higher than core {highest} recorded for {branch}")
|
|
(ROOT / CABAL_FILE).write_text(CABAL_VERSION.sub(rf"\g<1>{version}", read_text(CABAL_FILE), count=1))
|
|
print(f"{CABAL_FILE}: {version}")
|
|
if not types_version and not types_changed(releases, branch):
|
|
return
|
|
types = read_json(f"{TYPES_DIR}/package.json")
|
|
if not types_version:
|
|
previous = previous_core_commit(releases, branch)
|
|
released = json.loads(git("show", f"{previous}:{TYPES_DIR}/package.json")) if previous else types
|
|
types_version = bump_npm_version(released["version"])
|
|
readme, found = TYPES_README_CORE.subn(f"SimpleX Chat core: {version}", read_text(TYPES_README), count=1)
|
|
if not found:
|
|
sys.exit(f"{TYPES_README} has no line 'SimpleX Chat core: <version>'")
|
|
write_json(f"{TYPES_DIR}/package.json", {**types, "version": types_version, "simplexChatCore": version})
|
|
(ROOT / TYPES_README).write_text(readme)
|
|
print(f"{types['name']}: {types_version}")
|
|
|
|
|
|
def pinned_tags(pins: Pins) -> list[str]:
|
|
return [tag for tag in (pins.libs_release, pins.addon_release) if tag]
|
|
|
|
|
|
def published_pin_problems(pins: Pins) -> list[str]:
|
|
return pin_problems(pins, {tag: github_release(tag) for tag in pinned_tags(pins)})
|
|
|
|
|
|
def prepare(args: argparse.Namespace) -> None:
|
|
packages = frozenset(args.packages or ["nodejs", "python"])
|
|
branch = args.branch or current_branch()
|
|
releases, _ = read_releases()
|
|
libs = libs_releases()
|
|
libs_tag = latest_libs(libs, recorded_versions(releases, core_key(branch)))
|
|
if libs_tag is None:
|
|
sys.exit(f"no published library release recorded for {branch}")
|
|
core = core_version(libs_tag)
|
|
if version_key(cabal_version()) > version_key(core):
|
|
print(f"warning: core-{cabal_version()} is not published, {libs_tag} is pinned", file=sys.stderr)
|
|
sources_hash = current_addon_hash()
|
|
addon = addon_release(libs, sources_hash)
|
|
addon_digests = release_digests(addon, ADDON_ASSETS) if release_status(addon) == "published" else None
|
|
if "nodejs" in packages and addon_digests is None:
|
|
sys.exit(f"the add-on release of sources {sources_hash} is not published with every asset")
|
|
types = read_json(f"{TYPES_DIR}/package.json")
|
|
types_name = types["name"]
|
|
if "nodejs" in packages and not npm_published(types_name, types["version"]):
|
|
sys.exit(f"{types_name} {types['version']} is not published")
|
|
if "python" in packages:
|
|
core_commit = recorded_commit(releases, core_key(branch), core)
|
|
if changed_since(core_commit, PYTHON_TYPES_SOURCES):
|
|
sys.exit(f"the Python types differ from those of {libs_tag}")
|
|
nodejs = read_json(f"{NODEJS_DIR}/package.json")
|
|
python_version, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE))
|
|
result = plan(
|
|
State(
|
|
libs_tag=libs_tag,
|
|
libs_digests=release_digests(tag_release(libs, libs_tag), LIBS_ASSETS),
|
|
addon_tag=addon["tag_name"] if addon else "",
|
|
addon_digests=addon_digests,
|
|
nodejs_pins=parse_release(read_text(RELEASE_FILE)),
|
|
python_pins=python_pins,
|
|
types_dependency=f"^{types['version']}",
|
|
nodejs_types_dependency=nodejs["dependencies"][types_name],
|
|
nodejs=npm_package(NODEJS_DIR, releases),
|
|
python=package_state(
|
|
pypi_key(),
|
|
python_version,
|
|
PYTHON_DIR,
|
|
PYTHON_VERSION_FILE,
|
|
"^__version__ =",
|
|
releases,
|
|
lambda: pypi_published(python_version),
|
|
),
|
|
packages=packages,
|
|
nodejs_version=args.nodejs_version,
|
|
python_version=args.python_version,
|
|
)
|
|
)
|
|
selected_pins = [
|
|
pins for name, pins in [("nodejs", result.nodejs_pins), ("python", result.python_pins)] if name in packages
|
|
]
|
|
if problems := [problem for pins in selected_pins for problem in published_pin_problems(pins)]:
|
|
sys.exit("\n".join(problems))
|
|
assets = {
|
|
(tag, asset, pins.sha256[asset])
|
|
for pins in selected_pins
|
|
for tag in pinned_tags(pins)
|
|
for asset in release_assets(tag)
|
|
}
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
for tag, asset, sha256 in sorted(assets):
|
|
download(tag, asset, directory, sha256)
|
|
print(f"checked {asset} of {tag}", file=sys.stderr)
|
|
if "nodejs" in packages:
|
|
(ROOT / RELEASE_FILE).write_text(render_release(result.nodejs_pins))
|
|
dependencies = nodejs["dependencies"]
|
|
if result.types_dependency:
|
|
dependencies = {**dependencies, types_name: result.types_dependency}
|
|
write_json(f"{NODEJS_DIR}/package.json", {**nodejs, "version": result.nodejs_version, "dependencies": dependencies})
|
|
if "python" in packages:
|
|
(ROOT / PYTHON_VERSION_FILE).write_text(render_python_version(result.python_version, result.python_pins))
|
|
print(result.title + "\n\n" + "".join(f"- {note}\n" for note in result.notes), end="")
|
|
|
|
|
|
def publish_plan() -> None:
|
|
"""A recorded version is published: npm and PyPI list a new version minutes after it is published."""
|
|
releases, _ = read_releases()
|
|
for key, directory in [("types", TYPES_DIR), ("nodejs", NODEJS_DIR)]:
|
|
package = read_json(f"{directory}/package.json")
|
|
published = recorded_commit(releases, npm_key(directory), package["version"]) or npm_published(
|
|
package["name"], package["version"]
|
|
)
|
|
print(f"{key}_version={'' if published else package['version']}")
|
|
print(f"{key}_dist_tag={dist_tag(package['version'])}")
|
|
version, _ = parse_python_version(read_text(PYTHON_VERSION_FILE))
|
|
published = recorded_commit(releases, pypi_key(), version) or pypi_published(version)
|
|
print(f"python_version={'' if published else version}")
|
|
|
|
|
|
def record(args: argparse.Namespace) -> None:
|
|
versions = [
|
|
(core_key(args.branch), args.core),
|
|
(npm_key(TYPES_DIR), args.types),
|
|
(npm_key(NODEJS_DIR), args.nodejs),
|
|
(pypi_key(), args.python),
|
|
]
|
|
entries = [(key, version) for key, version in versions if version]
|
|
if not entries:
|
|
sys.exit("nothing to record")
|
|
print(record_releases(entries, args.commit))
|
|
|
|
|
|
def test_names() -> None:
|
|
if scope := os.environ.get("NPM_SCOPE"):
|
|
# npm provenance requires the repository of the publishing workflow.
|
|
url = f"git+https://github.com/{os.environ['GITHUB_REPOSITORY']}.git"
|
|
types = read_json(f"{TYPES_DIR}/package.json")
|
|
write_json(
|
|
f"{TYPES_DIR}/package.json",
|
|
{**types, "name": registry_name(types["name"], scope), "repository": {**types["repository"], "url": url}},
|
|
)
|
|
nodejs = read_json(f"{NODEJS_DIR}/package.json")
|
|
alias = f"npm:{registry_name(types['name'], scope)}@{nodejs['dependencies'][types['name']]}"
|
|
write_json(
|
|
f"{NODEJS_DIR}/package.json",
|
|
{
|
|
**nodejs,
|
|
"name": registry_name(nodejs["name"], scope),
|
|
"repository": {**nodejs["repository"], "url": url},
|
|
"dependencies": {**nodejs["dependencies"], types["name"]: alias},
|
|
},
|
|
)
|
|
if project := os.environ.get("PYPI_PROJECT"):
|
|
pyproject = ROOT / PYTHON_DIR / "pyproject.toml"
|
|
pyproject.write_text(
|
|
re.sub(r'^name = "[^"]+"', f'name = "{project}"', pyproject.read_text(), count=1, flags=re.M)
|
|
)
|
|
|
|
|
|
def print_release_status(tag: str) -> None:
|
|
print(f"status={release_status(tag_release(libs_releases(), tag))}")
|
|
|
|
|
|
def print_addon_release() -> None:
|
|
sources_hash = current_addon_hash()
|
|
release = addon_release(libs_releases(), sources_hash)
|
|
print(f"notes={shlex.quote(addon_notes(sources_hash))}")
|
|
print(f"status={release_status(release)}")
|
|
print(f"tag={release['tag_name'] if release else ''}")
|
|
|
|
|
|
def publish_release(tag: str) -> None:
|
|
release = tag_release(libs_releases(), tag)
|
|
if release is None:
|
|
sys.exit(f"{tag}: no release")
|
|
if not release["draft"]:
|
|
print(f"{tag} is published")
|
|
return
|
|
if missing := missing_assets(release):
|
|
sys.exit(f"{tag}: missing {', '.join(missing)}")
|
|
libs_api(f"releases/{release['id']}", "PATCH", {"draft": False})
|
|
print(f"{tag} published")
|
|
|
|
|
|
def check_pins() -> None:
|
|
_, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE))
|
|
problems = published_pin_problems(parse_release(read_text(RELEASE_FILE))) + published_pin_problems(python_pins)
|
|
if problems:
|
|
sys.exit("\n".join(problems))
|
|
print("the pins match the published releases")
|
|
|
|
|
|
def download_asset(tag: str, asset: str, directory: str) -> None:
|
|
digests = release_digests(github_release(tag), [asset])
|
|
if digests is None:
|
|
sys.exit(f"{asset} of {tag} is not published with a digest")
|
|
download(tag, asset, directory, digests[asset])
|
|
|
|
|
|
def download_pinned(asset: str, directory: str) -> None:
|
|
pins = parse_release(read_text(RELEASE_FILE))
|
|
tag = next((tag for tag in pinned_tags(pins) if asset in release_assets(tag)), None)
|
|
if tag is None or asset not in pins.sha256:
|
|
sys.exit(f"{asset} is not pinned in {RELEASE_FILE}")
|
|
download(tag, asset, directory, pins.sha256[asset])
|
|
|
|
|
|
def main(argv: list[str]) -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
branch_help = "the branch of the core versions, the checked out branch by default"
|
|
core_release_parser = commands.add_parser("core-release")
|
|
core_release_parser.add_argument("version", help="the new core version")
|
|
core_release_parser.add_argument("--branch", help=branch_help)
|
|
core_release_parser.add_argument("--types-version", help="this types version instead of a bump when needed")
|
|
claim_parser = commands.add_parser("claim-core")
|
|
claim_parser.add_argument("version", help="the core version of simplex-chat.cabal")
|
|
claim_parser.add_argument("commit", help="the simplex-chat commit to release it from")
|
|
claim_parser.add_argument("branch", help="the branch of the commit")
|
|
status_parser = commands.add_parser("release-status")
|
|
status_parser.add_argument("tag")
|
|
commands.add_parser("addon-release")
|
|
latest_libs_parser = commands.add_parser("latest-libs")
|
|
latest_libs_parser.add_argument("--branch", help=branch_help)
|
|
publish_release_parser = commands.add_parser("publish-release")
|
|
publish_release_parser.add_argument("tag")
|
|
commands.add_parser("check-pins")
|
|
download_parser = commands.add_parser("download")
|
|
download_parser.add_argument("tag")
|
|
download_parser.add_argument("asset")
|
|
download_parser.add_argument("directory")
|
|
download_pinned_parser = commands.add_parser("download-pinned")
|
|
download_pinned_parser.add_argument("asset")
|
|
download_pinned_parser.add_argument("directory")
|
|
changes_parser = commands.add_parser("changes")
|
|
changes_parser.add_argument("base", help="the commit to compare with")
|
|
build_parser = commands.add_parser("build-plan")
|
|
build_parser.add_argument("--base", required=True, help="the base commit of the pull request")
|
|
prepare_parser = commands.add_parser("prepare")
|
|
prepare_parser.add_argument("packages", nargs="*", choices=["nodejs", "python"], help="all when none is listed")
|
|
prepare_parser.add_argument("--branch", help=branch_help)
|
|
for package in ["nodejs", "python"]:
|
|
prepare_parser.add_argument(f"--{package}-version", help="this version instead of a bump when needed")
|
|
commands.add_parser("publish-plan")
|
|
record_parser = commands.add_parser("record")
|
|
record_parser.add_argument("--commit", required=True, help="the simplex-chat commit released or published from")
|
|
record_parser.add_argument("--branch", default="master", help="the branch of the core version")
|
|
for name in ["core", "types", "nodejs", "python"]:
|
|
record_parser.add_argument(f"--{name}", default="", help=f"{name} version, recorded when not empty")
|
|
commands.add_parser("test-names")
|
|
args = parser.parse_args(argv)
|
|
if args.command == "core-release":
|
|
core_release(args.version, args.branch or current_branch(), args.types_version)
|
|
elif args.command == "claim-core":
|
|
claim_core(args.version, args.commit, args.branch)
|
|
elif args.command == "release-status":
|
|
print_release_status(args.tag)
|
|
elif args.command == "addon-release":
|
|
print_addon_release()
|
|
elif args.command == "latest-libs":
|
|
releases, _ = read_releases()
|
|
versions = recorded_versions(releases, core_key(args.branch or current_branch()))
|
|
print(f"tag={latest_libs(libs_releases(), versions) or ''}")
|
|
elif args.command == "publish-release":
|
|
publish_release(args.tag)
|
|
elif args.command == "check-pins":
|
|
check_pins()
|
|
elif args.command == "download":
|
|
download_asset(args.tag, args.asset, args.directory)
|
|
elif args.command == "download-pinned":
|
|
download_pinned(args.asset, args.directory)
|
|
elif args.command == "changes":
|
|
changes(args.base)
|
|
elif args.command == "build-plan":
|
|
print_build_plan(args)
|
|
elif args.command == "prepare":
|
|
prepare(args)
|
|
elif args.command == "publish-plan":
|
|
publish_plan()
|
|
elif args.command == "record":
|
|
record(args)
|
|
else:
|
|
test_names()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv[1:])
|