mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-10-17 15:10:28 +00:00
nodejs: lazy per-backend libsimplex download (#7543)
* ci: drop dlltool from nodejs libs release * nodejs: download libsimplex into the user cache * nodejs: load libsimplex at runtime per backend * support bot: select db backend with --db * python: match backend switch error text * nodejs: download prebuilt add-on at runtime * ci: build nodejs add-on for libs release * calculator bot: load libs and add-on in image * support bot: document add-on override * nodejs: keep worker test output free of colors * nodejs: verify sha-256 of downloaded assets * python: pin core libs release with sha-256 * ci: add packages release script * ci: release nodejs add-on and open release PR * ci: publish packages after release PR * nodejs: keep npm from building the add-on * calculator bot: build the add-on in the image * ci: merge package release workflows * ci: open release PR when publishing fails * ci: build the add-on with node-addon-api only * ci: skip publishing in a called packages workflow * ci: update package release actions to Node 24 * nodejs: fix receive test timeout * ci: test packages on pull requests * nodejs: reuse the database in the close stress test * ci: keep failed package tests from hanging * ci: skip preset server tests * ci: plan package builds from the changes * ci: test packages with a given libsimplex * ci: test packages with libsimplex of core changes
This commit is contained in:
sh
@@ -66,8 +66,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=libsimplex /libs /opt/simplex/libs
|
||||
# Without it, npm re-running the library's preinstall downloads the released libs.
|
||||
ENV SIMPLEX_LIBS_DIR=/opt/simplex/libs
|
||||
# The library loads libsimplex and the add-on built below instead of downloading the released ones.
|
||||
ENV SIMPLEX_LIBS_DIR=/opt/simplex/libs \
|
||||
SIMPLEX_ADDON_PATH=/src/packages/simplex-chat-nodejs/build/Release/simplex.node
|
||||
|
||||
WORKDIR /src
|
||||
COPY packages/simplex-chat-client/types/typescript packages/simplex-chat-client/types/typescript
|
||||
@@ -79,6 +80,7 @@ COPY packages/simplex-chat-nodejs packages/simplex-chat-nodejs
|
||||
RUN cd packages/simplex-chat-nodejs && \
|
||||
npm install --no-audit --no-fund && \
|
||||
npm install --no-save --no-audit --no-fund ../simplex-chat-client/types/typescript && \
|
||||
npx node-gyp rebuild && \
|
||||
npx tsc && \
|
||||
cp src/simplex.* dist/
|
||||
|
||||
@@ -105,7 +107,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates dumb-init libffi8 libgmp10 libssl3 zlib1g && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=build /opt/simplex/libs /opt/simplex/libs
|
||||
COPY --from=build --chown=node:node /src /src
|
||||
ENV SIMPLEX_LIBS_DIR=/opt/simplex/libs \
|
||||
SIMPLEX_ADDON_PATH=/src/packages/simplex-chat-nodejs/build/Release/simplex.node
|
||||
|
||||
# A named volume takes this ownership; a bind mount must be owned by uid 1000.
|
||||
RUN mkdir -p /src/apps/simplex-calculator-bot/data && \
|
||||
|
||||
@@ -0,0 +1,529 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Release automation for simplex-chat and @simplex-chat/types on npm and simplex-chat on PyPI.
|
||||
|
||||
addon-tag print the add-on release tag for the current sources
|
||||
changes print whether the changes since a commit touch the chat core and the add-on
|
||||
build-plan print the platforms to build the add-on on and whether to run the package tests
|
||||
prepare update pins and versions for a release PR
|
||||
publish-plan print the package versions that are not published yet
|
||||
test-names rename the packages to NPM_SCOPE and PYPI_PROJECT (test repositories)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field, replace
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
NODEJS_DIR = "packages/simplex-chat-nodejs"
|
||||
TYPES_DIR = "packages/simplex-chat-client/types/typescript"
|
||||
PYTHON_DIR = "packages/simplex-chat-python"
|
||||
RELEASE_FILE = f"{NODEJS_DIR}/src/release.ts"
|
||||
PYTHON_VERSION_FILE = f"{PYTHON_DIR}/src/simplex_chat/_version.py"
|
||||
ADDON_SOURCES = [f"{NODEJS_DIR}/binding.gyp", f"{NODEJS_DIR}/cpp"]
|
||||
CORE_SOURCES = ["src", "simplex-chat.cabal", "cabal.project"]
|
||||
ADDON_TAG_HASH_LENGTH = 12
|
||||
LIBS_ASSETS = [
|
||||
"simplex-chat-libs-linux-aarch64.zip",
|
||||
"simplex-chat-libs-linux-x86_64-postgres.zip",
|
||||
"simplex-chat-libs-linux-x86_64.zip",
|
||||
"simplex-chat-libs-macos-aarch64.zip",
|
||||
"simplex-chat-libs-macos-x86_64.zip",
|
||||
"simplex-chat-libs-windows-x86_64.zip",
|
||||
]
|
||||
ADDON_RUNNERS = {
|
||||
"linux-aarch64": "ubuntu-22.04-arm",
|
||||
"linux-x86_64": "ubuntu-22.04",
|
||||
"macos-aarch64": "macos-latest",
|
||||
"macos-x86_64": "macos-15-intel",
|
||||
"windows-x86_64": "windows-latest",
|
||||
}
|
||||
ADDON_PLATFORMS = list(ADDON_RUNNERS)
|
||||
# build.yml builds libsimplex of pull requests on this platform only
|
||||
CORE_TEST_PLATFORM = "linux-x86_64"
|
||||
ADDON_ASSETS = [f"simplex-chat-nodejs-{platform}.node" for platform in ADDON_PLATFORMS]
|
||||
SHA256_HEX = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
# Pure functions
|
||||
|
||||
|
||||
def addon_tag(files: dict[str, bytes], node_addon_api: str) -> str:
|
||||
digest = hashlib.sha256()
|
||||
for name in sorted(files):
|
||||
digest.update(name.encode() + b"\0" + files[name] + b"\0")
|
||||
digest.update(f"node-addon-api@{node_addon_api}".encode())
|
||||
return f"nodejs-addon-{digest.hexdigest()[:ADDON_TAG_HASH_LENGTH]}"
|
||||
|
||||
|
||||
def bump_npm_version(version: str) -> str:
|
||||
"""A package release of app version 7.1.0-beta.6 is 7.1.0-beta.6.1, then 7.1.0-beta.6.2."""
|
||||
_, _, prerelease = version.partition("-")
|
||||
if len(prerelease.split(".")) == 2:
|
||||
return f"{version}.1"
|
||||
head, _, last = version.rpartition(".")
|
||||
return f"{head}.{int(last) + 1}" if last.isdigit() else f"{version}.1"
|
||||
|
||||
|
||||
def bump_python_version(version: str) -> str:
|
||||
"""A package release of app version 7.1.0b6 is 7.1.0b6.post1, then 7.1.0b6.post2."""
|
||||
post = re.search(r"\.post(\d+)$", version)
|
||||
return f"{version[: post.start()]}.post{int(post[1]) + 1}" if post else f"{version}.post1"
|
||||
|
||||
|
||||
def dist_tag(version: str) -> str:
|
||||
return "beta" if "-" in version else "latest"
|
||||
|
||||
|
||||
def registry_name(name: str, scope: str | None) -> str:
|
||||
"""Package name on npm: test repositories publish under their own scope."""
|
||||
return f"{scope}/{name.split('/')[-1]}" if scope else name
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Pins:
|
||||
libs_release: str
|
||||
sha256: dict[str, str]
|
||||
addon_release: str | None = None
|
||||
|
||||
|
||||
def parse_release(source: str) -> Pins:
|
||||
libs = re.search(r'export const LIBS_RELEASE = "([^"]+)"', source)
|
||||
addon = re.search(r'export const ADDON_RELEASE: string \| undefined = (?:"([^"]+)"|undefined)', source)
|
||||
if not libs or not addon:
|
||||
raise ValueError(f"cannot parse {RELEASE_FILE}")
|
||||
hashes = dict(re.findall(r'^ {2}"([^"]+)": "([0-9a-f]{64})",$', source, re.M))
|
||||
return Pins(libs[1], hashes, addon[1])
|
||||
|
||||
|
||||
def render_release(pins: Pins) -> str:
|
||||
hashes = "".join(f' "{asset}": "{pins.sha256[asset]}",\n' for asset in sorted(pins.sha256))
|
||||
addon = f'"{pins.addon_release}"' if pins.addon_release else "undefined"
|
||||
return (
|
||||
"// Written by scripts/ci/packages_release.py from the simplex-chat-libs releases.\n"
|
||||
"\n"
|
||||
f'export const LIBS_RELEASE = "{pins.libs_release}"\n'
|
||||
"\n"
|
||||
f"export const ADDON_RELEASE: string | undefined = {addon}\n"
|
||||
"\n"
|
||||
"export const SHA256: {readonly [asset: string]: string} = {\n"
|
||||
f"{hashes}}}\n"
|
||||
)
|
||||
|
||||
|
||||
def parse_python_version(source: str) -> tuple[str, Pins]:
|
||||
version = re.search(r'^__version__ = "([^"]+)"', source, re.M)
|
||||
libs = re.search(r'^LIBS_RELEASE = "([^"]+)"', source, re.M)
|
||||
if not version or not libs:
|
||||
raise ValueError(f"cannot parse {PYTHON_VERSION_FILE}")
|
||||
hashes = dict(re.findall(r'^ {4}"([^"]+)": "([0-9a-f]{64})",$', source, re.M))
|
||||
return version[1], Pins(libs[1], hashes)
|
||||
|
||||
|
||||
def render_python_version(version: str, pins: Pins) -> str:
|
||||
hashes = "".join(f' "{asset}": "{pins.sha256[asset]}",\n' for asset in sorted(pins.sha256))
|
||||
return (
|
||||
'"""Package version and the simplex-chat-libs release it loads.\n'
|
||||
"\n"
|
||||
"Written by scripts/ci/packages_release.py: the release PR bumps the version\n"
|
||||
"and pins new library releases with the SHA-256 of their assets.\n"
|
||||
'"""\n'
|
||||
"\n"
|
||||
f'__version__ = "{version}" # PEP 440 — read by hatchling for wheel metadata\n'
|
||||
f'LIBS_RELEASE = "{pins.libs_release}" # simplex-chat-libs release tag\n'
|
||||
"LIBS_SHA256 = {\n"
|
||||
f"{hashes}}}\n"
|
||||
)
|
||||
|
||||
|
||||
def release_digests(release: dict | None, assets: list[str]) -> dict[str, str] | None:
|
||||
"""Asset digests of a GitHub release, or None when the release is missing or incomplete."""
|
||||
if release is None:
|
||||
return None
|
||||
digests = {a["name"]: (a.get("digest") or "").removeprefix("sha256:") for a in release["assets"]}
|
||||
if not all(SHA256_HEX.match(digests.get(asset, "")) for asset in assets):
|
||||
return None
|
||||
return {asset: digests[asset] for asset in assets}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Package:
|
||||
version: str
|
||||
changed: bool
|
||||
published: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class State:
|
||||
libs_tag: str
|
||||
libs_digests: dict[str, str] | None
|
||||
addon_tag: str
|
||||
addon_digests: dict[str, str] | None
|
||||
nodejs_pins: Pins
|
||||
python_pins: Pins
|
||||
types: Package
|
||||
nodejs: Package
|
||||
python: Package
|
||||
types_version: str | None = None
|
||||
nodejs_version: str | None = None
|
||||
python_version: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Plan:
|
||||
nodejs_pins: Pins
|
||||
python_pins: Pins
|
||||
types_version: str
|
||||
types_bumped: bool
|
||||
nodejs_version: str
|
||||
nodejs_changed: bool
|
||||
python_version: str
|
||||
python_changed: bool
|
||||
notes: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def types_dependency(self) -> str | None:
|
||||
return f"^{self.types_version}" if self.types_bumped else None
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
parts = []
|
||||
if self.types_bumped:
|
||||
parts.append(f"types {self.types_version}")
|
||||
if self.nodejs_changed:
|
||||
parts.append(f"nodejs {self.nodejs_version}")
|
||||
if self.python_changed:
|
||||
parts.append(f"python {self.python_version}")
|
||||
return f"packages: release {', '.join(parts)}"
|
||||
|
||||
|
||||
def plan(state: State) -> Plan:
|
||||
"""What a release PR changes, from the state of the branch and the releases."""
|
||||
notes: list[str] = []
|
||||
|
||||
def pin(pins: Pins, field_name: str, tag: str, digests: dict[str, str] | None, what: str) -> tuple[Pins, bool]:
|
||||
current = getattr(pins, field_name)
|
||||
if current == tag:
|
||||
return pins, False
|
||||
if digests is None:
|
||||
notes.append(f"{what} release {tag} is missing or incomplete; {current or 'none'} stays pinned")
|
||||
return pins, False
|
||||
notes.append(f"{what}: {tag}")
|
||||
return replace(pins, **{field_name: tag, "sha256": {**pins.sha256, **digests}}), True
|
||||
|
||||
nodejs_pins, nodejs_libs = pin(
|
||||
state.nodejs_pins, "libs_release", state.libs_tag, state.libs_digests, "Node.js libsimplex"
|
||||
)
|
||||
nodejs_pins, addon = pin(nodejs_pins, "addon_release", state.addon_tag, state.addon_digests, "Node.js add-on")
|
||||
python_pins, python_libs = pin(
|
||||
state.python_pins, "libs_release", state.libs_tag, state.libs_digests, "Python libsimplex"
|
||||
)
|
||||
|
||||
def next_version(pkg: Package, changed: bool, override: str | None, bump) -> str:
|
||||
if override:
|
||||
return override
|
||||
return bump(pkg.version) if changed and pkg.published else pkg.version
|
||||
|
||||
types_version = next_version(state.types, state.types.changed, state.types_version, bump_npm_version)
|
||||
types_bumped = types_version != state.types.version
|
||||
nodejs_changed = nodejs_libs or addon or types_bumped or state.nodejs.changed
|
||||
nodejs_version = next_version(state.nodejs, nodejs_changed, state.nodejs_version, bump_npm_version)
|
||||
python_changed = python_libs or state.python.changed
|
||||
python_version = next_version(state.python, python_changed, state.python_version, bump_python_version)
|
||||
for name, old, new in [
|
||||
("@simplex-chat/types", state.types.version, types_version),
|
||||
("simplex-chat (npm)", state.nodejs.version, nodejs_version),
|
||||
("simplex-chat (PyPI)", state.python.version, python_version),
|
||||
]:
|
||||
if old != new:
|
||||
notes.append(f"{name}: {old} → {new}")
|
||||
return Plan(
|
||||
nodejs_pins=nodejs_pins,
|
||||
python_pins=python_pins,
|
||||
types_version=types_version,
|
||||
types_bumped=types_bumped,
|
||||
nodejs_version=nodejs_version,
|
||||
nodejs_changed=nodejs_changed or nodejs_version != state.nodejs.version,
|
||||
python_version=python_version,
|
||||
python_changed=python_changed or python_version != state.python.version,
|
||||
notes=notes,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BuildPlan:
|
||||
platforms: list[str]
|
||||
test: bool
|
||||
|
||||
|
||||
def build_plan(
|
||||
event: str, release_missing: bool, libs_artifact: bool, libs_release: str, core_changed: bool, addon_changed: bool
|
||||
) -> BuildPlan:
|
||||
"""Each package test runs once, against the libsimplex of the change: a pull request that changes the core
|
||||
is tested by build.yml with its own libsimplex (libs_artifact), a new library release with that release."""
|
||||
if libs_artifact:
|
||||
return BuildPlan([CORE_TEST_PLATFORM], test=True)
|
||||
if libs_release:
|
||||
return BuildPlan(ADDON_PLATFORMS, test=True)
|
||||
if event == "pull_request":
|
||||
if core_changed:
|
||||
return BuildPlan(ADDON_PLATFORMS if addon_changed else [], test=False)
|
||||
return BuildPlan(ADDON_PLATFORMS, test=True)
|
||||
return BuildPlan(ADDON_PLATFORMS if release_missing else [], test=True)
|
||||
|
||||
|
||||
def build_matrix(platforms: list[str]) -> str:
|
||||
return json.dumps([{"platform": platform, "os": ADDON_RUNNERS[platform]} for platform in platforms])
|
||||
|
||||
|
||||
# Repository and registry access
|
||||
|
||||
|
||||
def git(*args: str) -> str:
|
||||
return subprocess.run(["git", *args], cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def read_text(file: str) -> str:
|
||||
return (ROOT / file).read_text()
|
||||
|
||||
|
||||
def read_field(file: str, pattern: str) -> str:
|
||||
found = re.search(pattern, read_text(file), re.M)
|
||||
if not found:
|
||||
raise ValueError(f"no match for {pattern} in {file}")
|
||||
return found[1]
|
||||
|
||||
|
||||
def read_json(file: str) -> dict:
|
||||
return json.loads(read_text(file))
|
||||
|
||||
|
||||
def write_json(file: str, value: dict) -> None:
|
||||
(ROOT / file).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def current_addon_tag() -> str:
|
||||
files = {
|
||||
path.relative_to(ROOT).as_posix(): path.read_bytes()
|
||||
for source in ADDON_SOURCES
|
||||
for path in ([ROOT / source] if (ROOT / source).is_file() else sorted((ROOT / source).rglob("*")))
|
||||
if path.is_file()
|
||||
}
|
||||
return addon_tag(files, read_json(f"{NODEJS_DIR}/package.json")["devDependencies"]["node-addon-api"])
|
||||
|
||||
|
||||
def changed_since(base: str, sources: list[str]) -> bool:
|
||||
diff = subprocess.run(["git", "diff", "--quiet", base, "HEAD", "--", *sources], cwd=ROOT)
|
||||
return diff.returncode != 0
|
||||
|
||||
|
||||
def changes(base: str) -> None:
|
||||
print(f"core={'true' if changed_since(base, CORE_SOURCES) else ''}")
|
||||
print(f"addon={'true' if changed_since(base, ADDON_SOURCES) else ''}")
|
||||
|
||||
|
||||
def print_build_plan(args: argparse.Namespace) -> None:
|
||||
pull_request = args.event == "pull_request" and not args.libs_artifact
|
||||
result = build_plan(
|
||||
args.event,
|
||||
release_missing=args.release_missing,
|
||||
libs_artifact=args.libs_artifact,
|
||||
libs_release=args.libs_release,
|
||||
core_changed=pull_request and changed_since(args.base, CORE_SOURCES),
|
||||
addon_changed=pull_request and changed_since(args.base, ADDON_SOURCES),
|
||||
)
|
||||
print(f"platforms={build_matrix(result.platforms)}")
|
||||
print(f"test={'true' if result.test else ''}")
|
||||
|
||||
|
||||
def publish_tag(package: str, version: str) -> str:
|
||||
"""Tag of the commit a version was published from (packages.yml)."""
|
||||
return f"{package}-v{version}"
|
||||
|
||||
|
||||
def published_tag(package: str, version: str) -> str | None:
|
||||
"""The publish tag of the version, when it exists: registries list new versions with a delay."""
|
||||
tag = f"refs/tags/{publish_tag(package, version)}"
|
||||
found = subprocess.run(["git", "rev-parse", "--verify", "--quiet", tag], cwd=ROOT, capture_output=True)
|
||||
return tag if found.returncode == 0 else None
|
||||
|
||||
|
||||
def changed_since_publish(package: str, version: str, directory: str, version_file: str, version_pattern: str) -> bool:
|
||||
"""Shipped files changed since the version was published, or since it was set when it was not published yet."""
|
||||
tag = published_tag(package, version)
|
||||
base = tag or git("log", "-1", "--format=%H", "-G", version_pattern, "--", version_file)
|
||||
diff = subprocess.run(["git", "diff", "--quiet", base, "HEAD", "--", directory, f":!{directory}/tests"], cwd=ROOT)
|
||||
return diff.returncode != 0
|
||||
|
||||
|
||||
def http_json(url: str, headers: dict[str, str] | None = None) -> dict | None:
|
||||
"""The JSON at `url`, or None for 404."""
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "simplex-chat-release", **(headers or {})})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
return json.load(response)
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
return None
|
||||
raise
|
||||
|
||||
|
||||
def npm_published(name: str, version: str) -> bool:
|
||||
registry = os.environ.get("NPM_REGISTRY", "https://registry.npmjs.org")
|
||||
package = registry_name(name, os.environ.get("NPM_SCOPE")).replace("/", "%2F")
|
||||
return http_json(f"{registry}/{package}/{version}") is not None
|
||||
|
||||
|
||||
def python_project() -> str:
|
||||
return read_field(f"{PYTHON_DIR}/pyproject.toml", r'^name = "([^"]+)"')
|
||||
|
||||
|
||||
def pypi_published(version: str) -> bool:
|
||||
index = os.environ.get("PYPI_URL", "https://pypi.org")
|
||||
project = os.environ.get("PYPI_PROJECT") or python_project()
|
||||
return http_json(f"{index}/pypi/{project}/{version}/json") is not None
|
||||
|
||||
|
||||
def github_release(tag: str) -> dict | None:
|
||||
repo = os.environ.get("LIBS_REPOSITORY", "simplex-chat/simplex-chat-libs")
|
||||
headers = {"Accept": "application/vnd.github+json"}
|
||||
if token := os.environ.get("GH_TOKEN"):
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return http_json(f"https://api.github.com/repos/{repo}/releases/tags/{tag}", headers)
|
||||
|
||||
|
||||
def npm_package(key: str, directory: str) -> Package:
|
||||
package = read_json(f"{directory}/package.json")
|
||||
changed = changed_since_publish(key, package["version"], directory, f"{directory}/package.json", '"version":')
|
||||
published = bool(published_tag(key, package["version"])) or npm_published(package["name"], package["version"])
|
||||
return Package(package["version"], changed, published)
|
||||
|
||||
|
||||
def prepare(args: argparse.Namespace) -> None:
|
||||
cabal_version = read_field("simplex-chat.cabal", r"^version:\s*(\S+)")
|
||||
libs_tag = f"core-{cabal_version}"
|
||||
addon = current_addon_tag()
|
||||
python_version, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE))
|
||||
python_changed = changed_since_publish("python", python_version, PYTHON_DIR, PYTHON_VERSION_FILE, "^__version__ =")
|
||||
result = plan(
|
||||
State(
|
||||
libs_tag=libs_tag,
|
||||
libs_digests=release_digests(github_release(libs_tag), LIBS_ASSETS),
|
||||
addon_tag=addon,
|
||||
addon_digests=release_digests(github_release(addon), ADDON_ASSETS),
|
||||
nodejs_pins=parse_release(read_text(RELEASE_FILE)),
|
||||
python_pins=python_pins,
|
||||
types=npm_package("types", TYPES_DIR),
|
||||
nodejs=npm_package("nodejs", NODEJS_DIR),
|
||||
python=Package(
|
||||
python_version,
|
||||
python_changed,
|
||||
bool(published_tag("python", python_version)) or pypi_published(python_version),
|
||||
),
|
||||
types_version=args.types_version,
|
||||
nodejs_version=args.nodejs_version,
|
||||
python_version=args.python_version,
|
||||
)
|
||||
)
|
||||
(ROOT / RELEASE_FILE).write_text(render_release(result.nodejs_pins))
|
||||
(ROOT / PYTHON_VERSION_FILE).write_text(render_python_version(result.python_version, result.python_pins))
|
||||
types = read_json(f"{TYPES_DIR}/package.json")
|
||||
write_json(f"{TYPES_DIR}/package.json", {**types, "version": result.types_version})
|
||||
nodejs = read_json(f"{NODEJS_DIR}/package.json")
|
||||
dependencies = nodejs["dependencies"]
|
||||
if result.types_dependency:
|
||||
dependencies = {**dependencies, "@simplex-chat/types": result.types_dependency}
|
||||
write_json(f"{NODEJS_DIR}/package.json", {**nodejs, "version": result.nodejs_version, "dependencies": dependencies})
|
||||
summary = result.title + "\n\n" + "".join(f"- {note}\n" for note in result.notes)
|
||||
if args.summary:
|
||||
Path(args.summary).write_text(summary)
|
||||
print(summary, end="")
|
||||
|
||||
|
||||
def publish_plan() -> None:
|
||||
for key, directory in [("types", TYPES_DIR), ("nodejs", NODEJS_DIR)]:
|
||||
package = read_json(f"{directory}/package.json")
|
||||
pending = not published_tag(key, package["version"]) and not npm_published(package["name"], package["version"])
|
||||
print(f"{key}_version={package['version'] if pending else ''}")
|
||||
print(f"{key}_dist_tag={dist_tag(package['version'])}")
|
||||
version, _ = parse_python_version(read_text(PYTHON_VERSION_FILE))
|
||||
pending = not published_tag("python", version) and not pypi_published(version)
|
||||
print(f"python_version={version if pending else ''}")
|
||||
|
||||
|
||||
def test_names() -> None:
|
||||
if scope := os.environ.get("NPM_SCOPE"):
|
||||
# npm provenance requires the repository of the publishing workflow.
|
||||
url = f"git+https://github.com/{os.environ['GITHUB_REPOSITORY']}.git"
|
||||
types = read_json(f"{TYPES_DIR}/package.json")
|
||||
write_json(
|
||||
f"{TYPES_DIR}/package.json",
|
||||
{**types, "name": registry_name(types["name"], scope), "repository": {**types["repository"], "url": url}},
|
||||
)
|
||||
nodejs = read_json(f"{NODEJS_DIR}/package.json")
|
||||
alias = f"npm:{registry_name(types['name'], scope)}@{nodejs['dependencies'][types['name']]}"
|
||||
write_json(
|
||||
f"{NODEJS_DIR}/package.json",
|
||||
{
|
||||
**nodejs,
|
||||
"name": registry_name(nodejs["name"], scope),
|
||||
"repository": {**nodejs["repository"], "url": url},
|
||||
"dependencies": {**nodejs["dependencies"], types["name"]: alias},
|
||||
},
|
||||
)
|
||||
if project := os.environ.get("PYPI_PROJECT"):
|
||||
pyproject = ROOT / PYTHON_DIR / "pyproject.toml"
|
||||
pyproject.write_text(
|
||||
re.sub(r'^name = "[^"]+"', f'name = "{project}"', pyproject.read_text(), count=1, flags=re.M)
|
||||
)
|
||||
|
||||
|
||||
def github_bool(value: str) -> bool:
|
||||
"""GitHub expressions pass booleans to the shell as "true", "false" or ""."""
|
||||
return value == "true"
|
||||
|
||||
|
||||
def main(argv: list[str]) -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
commands.add_parser("addon-tag")
|
||||
changes_parser = commands.add_parser("changes")
|
||||
changes_parser.add_argument("base", help="the commit to compare with")
|
||||
build_parser = commands.add_parser("build-plan")
|
||||
build_parser.add_argument("--event", required=True, help="github.event_name")
|
||||
build_parser.add_argument("--base", default="", help="the base commit of a pull request")
|
||||
build_parser.add_argument("--release-missing", type=github_bool, default=False)
|
||||
build_parser.add_argument("--libs-artifact", type=github_bool, default=False)
|
||||
build_parser.add_argument("--libs-release", default="")
|
||||
prepare_parser = commands.add_parser("prepare")
|
||||
for package in ["nodejs", "types", "python"]:
|
||||
prepare_parser.add_argument(f"--{package}-version", help="this version instead of a bump when needed")
|
||||
prepare_parser.add_argument("--summary", help="file for the PR title and body")
|
||||
commands.add_parser("publish-plan")
|
||||
commands.add_parser("test-names")
|
||||
args = parser.parse_args(argv)
|
||||
if args.command == "addon-tag":
|
||||
print(current_addon_tag())
|
||||
elif args.command == "changes":
|
||||
changes(args.base)
|
||||
elif args.command == "build-plan":
|
||||
print_build_plan(args)
|
||||
elif args.command == "prepare":
|
||||
prepare(args)
|
||||
elif args.command == "publish-plan":
|
||||
publish_plan()
|
||||
else:
|
||||
test_names()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
@@ -0,0 +1,288 @@
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import packages_release
|
||||
from packages_release import (
|
||||
ADDON_ASSETS,
|
||||
ADDON_PLATFORMS,
|
||||
BuildPlan,
|
||||
LIBS_ASSETS,
|
||||
Package,
|
||||
Pins,
|
||||
State,
|
||||
addon_tag,
|
||||
build_matrix,
|
||||
build_plan,
|
||||
bump_npm_version,
|
||||
bump_python_version,
|
||||
changed_since,
|
||||
changed_since_publish,
|
||||
published_tag,
|
||||
dist_tag,
|
||||
parse_python_version,
|
||||
parse_release,
|
||||
plan,
|
||||
registry_name,
|
||||
release_digests,
|
||||
render_python_version,
|
||||
render_release,
|
||||
)
|
||||
|
||||
|
||||
def sha(n: int) -> str:
|
||||
return f"{n:x}" * 64
|
||||
|
||||
|
||||
def digests(assets: list[str], n: int) -> dict[str, str]:
|
||||
return {asset: sha(n) for asset in assets}
|
||||
|
||||
|
||||
def github_release(assets: list[str], n: int) -> dict:
|
||||
return {"assets": [{"name": name, "digest": f"sha256:{sha(n)}"} for name in assets]}
|
||||
|
||||
|
||||
def state(**overrides) -> State:
|
||||
current = State(
|
||||
libs_tag="core-1.0.0.1",
|
||||
libs_digests=digests(LIBS_ASSETS, 1),
|
||||
addon_tag="nodejs-addon-aaa",
|
||||
addon_digests=digests(ADDON_ASSETS, 2),
|
||||
nodejs_pins=Pins("core-1.0.0.1", {**digests(LIBS_ASSETS, 1), **digests(ADDON_ASSETS, 2)}, "nodejs-addon-aaa"),
|
||||
python_pins=Pins("core-1.0.0.1", digests(LIBS_ASSETS, 1)),
|
||||
types=Package("0.12.1", changed=False, published=True),
|
||||
nodejs=Package("7.1.0-beta.6.1", changed=False, published=True),
|
||||
python=Package("7.1.0b6", changed=False, published=True),
|
||||
)
|
||||
return replace(current, **overrides)
|
||||
|
||||
|
||||
NEW_LIBS = {"libs_tag": "core-1.0.0.2", "libs_digests": digests(LIBS_ASSETS, 3)}
|
||||
NEW_ADDON = {"addon_tag": "nodejs-addon-bbb", "addon_digests": digests(ADDON_ASSETS, 4)}
|
||||
|
||||
|
||||
class VersionTest(unittest.TestCase):
|
||||
def test_npm_bump_adds_a_package_release_to_an_app_version(self):
|
||||
self.assertEqual(bump_npm_version("7.1.0-beta.6"), "7.1.0-beta.6.1")
|
||||
self.assertEqual(bump_npm_version("7.1.0-beta.6.1"), "7.1.0-beta.6.2")
|
||||
self.assertEqual(bump_npm_version("0.12.9"), "0.12.10")
|
||||
self.assertEqual(bump_npm_version("7.0.2"), "7.0.3")
|
||||
|
||||
def test_python_bump_uses_post_releases(self):
|
||||
self.assertEqual(bump_python_version("7.1.0b6"), "7.1.0b6.post1")
|
||||
self.assertEqual(bump_python_version("7.1.0b6.post9"), "7.1.0b6.post10")
|
||||
self.assertEqual(bump_python_version("7.0.2"), "7.0.2.post1")
|
||||
|
||||
def test_prereleases_publish_as_beta(self):
|
||||
self.assertEqual(dist_tag("7.1.0-beta.6.2"), "beta")
|
||||
self.assertEqual(dist_tag("7.1.0"), "latest")
|
||||
|
||||
def test_test_scope_renames_npm_packages(self):
|
||||
self.assertEqual(registry_name("@simplex-chat/types", "@test"), "@test/types")
|
||||
self.assertEqual(registry_name("simplex-chat", "@test"), "@test/simplex-chat")
|
||||
self.assertEqual(registry_name("simplex-chat", None), "simplex-chat")
|
||||
|
||||
|
||||
class AddonTagTest(unittest.TestCase):
|
||||
def test_depends_on_sources_and_node_addon_api(self):
|
||||
files = {"a.cc": b"1", "b.h": b"2"}
|
||||
tag = addon_tag(files, "^8.5.0")
|
||||
self.assertRegex(tag, r"^nodejs-addon-[0-9a-f]{12}$")
|
||||
self.assertEqual(addon_tag(dict(reversed(files.items())), "^8.5.0"), tag)
|
||||
self.assertNotEqual(addon_tag({**files, "b.h": b"3"}, "^8.5.0"), tag)
|
||||
self.assertNotEqual(addon_tag({"a.cc": b"1", "c.h": b"2"}, "^8.5.0"), tag)
|
||||
self.assertNotEqual(addon_tag(files, "^8.6.0"), tag)
|
||||
|
||||
|
||||
class PinFilesTest(unittest.TestCase):
|
||||
def test_release_ts_round_trip(self):
|
||||
pins = Pins("core-1.0.0.1", {"b.zip": sha(2), "a.zip": sha(1)}, "nodejs-addon-aaa")
|
||||
self.assertEqual(parse_release(render_release(pins)), pins)
|
||||
self.assertIsNone(parse_release(render_release(replace(pins, addon_release=None))).addon_release)
|
||||
|
||||
def test_python_version_round_trip(self):
|
||||
pins = Pins("core-1.0.0.1", {"b.zip": sha(2), "a.zip": sha(1)})
|
||||
self.assertEqual(parse_python_version(render_python_version("7.1.0b6.post1", pins)), ("7.1.0b6.post1", pins))
|
||||
|
||||
|
||||
class ReleaseDigestsTest(unittest.TestCase):
|
||||
def test_requires_every_asset_with_a_digest(self):
|
||||
self.assertEqual(release_digests(github_release(LIBS_ASSETS, 3), LIBS_ASSETS), digests(LIBS_ASSETS, 3))
|
||||
self.assertIsNone(release_digests(github_release(LIBS_ASSETS[1:], 3), LIBS_ASSETS))
|
||||
self.assertIsNone(release_digests(None, LIBS_ASSETS))
|
||||
no_digests = {"assets": [{"name": name, "digest": None} for name in LIBS_ASSETS]}
|
||||
self.assertIsNone(release_digests(no_digests, LIBS_ASSETS))
|
||||
|
||||
|
||||
class PlanTest(unittest.TestCase):
|
||||
def test_nothing_changes_when_current(self):
|
||||
result = plan(state())
|
||||
self.assertEqual((result.nodejs_changed, result.python_changed, result.types_bumped), (False, False, False))
|
||||
self.assertEqual(result.notes, [])
|
||||
|
||||
def test_new_libs_release_updates_both_libraries(self):
|
||||
result = plan(state(**NEW_LIBS))
|
||||
self.assertEqual(result.nodejs_pins.libs_release, "core-1.0.0.2")
|
||||
self.assertEqual(result.python_pins.libs_release, "core-1.0.0.2")
|
||||
self.assertEqual(result.nodejs_pins.sha256[LIBS_ASSETS[0]], sha(3))
|
||||
self.assertEqual(result.nodejs_pins.sha256[ADDON_ASSETS[0]], sha(2))
|
||||
self.assertEqual(result.python_pins.sha256[LIBS_ASSETS[0]], sha(3))
|
||||
self.assertEqual(result.title, "packages: release nodejs 7.1.0-beta.6.2, python 7.1.0b6.post1")
|
||||
|
||||
def test_new_addon_release_updates_nodejs_only(self):
|
||||
result = plan(state(**NEW_ADDON))
|
||||
self.assertEqual(result.nodejs_pins.addon_release, "nodejs-addon-bbb")
|
||||
self.assertEqual(result.nodejs_pins.sha256[ADDON_ASSETS[0]], sha(4))
|
||||
self.assertEqual(result.title, "packages: release nodejs 7.1.0-beta.6.2")
|
||||
|
||||
def test_incomplete_release_keeps_the_pins(self):
|
||||
result = plan(state(addon_tag="nodejs-addon-bbb", addon_digests=None))
|
||||
self.assertEqual(result.nodejs_pins.addon_release, "nodejs-addon-aaa")
|
||||
self.assertFalse(result.nodejs_changed)
|
||||
self.assertEqual(
|
||||
result.notes,
|
||||
["Node.js add-on release nodejs-addon-bbb is missing or incomplete; nodejs-addon-aaa stays pinned"],
|
||||
)
|
||||
|
||||
def test_source_changes_bump_their_package(self):
|
||||
self.assertEqual(
|
||||
plan(state(nodejs=Package("7.1.0-beta.6.1", True, True))).title, "packages: release nodejs 7.1.0-beta.6.2"
|
||||
)
|
||||
self.assertEqual(
|
||||
plan(state(python=Package("7.1.0b6", True, True))).title, "packages: release python 7.1.0b6.post1"
|
||||
)
|
||||
|
||||
def test_types_bump_updates_the_nodejs_dependency(self):
|
||||
result = plan(state(types=Package("0.12.1", True, True)))
|
||||
self.assertEqual(result.types_dependency, "^0.12.2")
|
||||
self.assertEqual(result.title, "packages: release types 0.12.2, nodejs 7.1.0-beta.6.2")
|
||||
|
||||
def test_all_changes_combine_into_one_release(self):
|
||||
result = plan(
|
||||
state(
|
||||
**NEW_LIBS,
|
||||
**NEW_ADDON,
|
||||
types=Package("0.12.1", True, True),
|
||||
nodejs=Package("7.1.0-beta.6.1", True, True),
|
||||
python=Package("7.1.0b6", True, True),
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
(result.nodejs_pins.libs_release, result.nodejs_pins.addon_release), ("core-1.0.0.2", "nodejs-addon-bbb")
|
||||
)
|
||||
self.assertEqual(result.title, "packages: release types 0.12.2, nodejs 7.1.0-beta.6.2, python 7.1.0b6.post1")
|
||||
|
||||
def test_unpublished_versions_are_not_bumped_again(self):
|
||||
result = plan(
|
||||
state(
|
||||
**NEW_LIBS, nodejs=Package("7.1.0-beta.6.2", True, False), python=Package("7.1.0b6.post1", False, False)
|
||||
)
|
||||
)
|
||||
self.assertEqual(result.title, "packages: release nodejs 7.1.0-beta.6.2, python 7.1.0b6.post1")
|
||||
|
||||
def test_requested_versions_win(self):
|
||||
result = plan(state(nodejs_version="7.2.0", types_version="0.13.0", python_version="7.2.0"))
|
||||
self.assertEqual(result.types_dependency, "^0.13.0")
|
||||
self.assertEqual(result.title, "packages: release types 0.13.0, nodejs 7.2.0, python 7.2.0")
|
||||
|
||||
|
||||
class BuildPlanTest(unittest.TestCase):
|
||||
def plan(self, event="push", release_missing=False, libs_artifact=False, libs_release="", core=False, addon=False):
|
||||
return build_plan(event, release_missing, libs_artifact, libs_release, core, addon)
|
||||
|
||||
def test_push_builds_a_missing_addon_release_and_tests_it(self):
|
||||
self.assertEqual(self.plan(release_missing=True), BuildPlan(ADDON_PLATFORMS, test=True))
|
||||
self.assertEqual(self.plan(), BuildPlan([], test=True))
|
||||
|
||||
def test_pull_request_tests_with_the_pinned_libs(self):
|
||||
self.assertEqual(self.plan("pull_request"), BuildPlan(ADDON_PLATFORMS, test=True))
|
||||
self.assertEqual(self.plan("pull_request", addon=True), BuildPlan(ADDON_PLATFORMS, test=True))
|
||||
|
||||
def test_core_change_leaves_tests_to_build_yml(self):
|
||||
self.assertEqual(self.plan("pull_request", core=True), BuildPlan([], test=False))
|
||||
self.assertEqual(self.plan("pull_request", core=True, addon=True), BuildPlan(ADDON_PLATFORMS, test=False))
|
||||
|
||||
def test_libsimplex_of_the_pull_request_is_tested_on_its_platform(self):
|
||||
self.assertEqual(
|
||||
self.plan("pull_request", libs_artifact=True, core=True), BuildPlan(["linux-x86_64"], test=True)
|
||||
)
|
||||
|
||||
def test_new_library_release_is_tested_on_every_platform(self):
|
||||
self.assertEqual(self.plan(libs_release="core-1.0.0.2"), BuildPlan(ADDON_PLATFORMS, test=True))
|
||||
|
||||
def test_matrix_names_the_runner_of_each_platform(self):
|
||||
self.assertEqual(
|
||||
build_matrix(["linux-x86_64", "windows-x86_64"]),
|
||||
'[{"platform": "linux-x86_64", "os": "ubuntu-22.04"}, {"platform": "windows-x86_64", "os": "windows-latest"}]',
|
||||
)
|
||||
|
||||
|
||||
class ChangedSincePublishTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.repo = Path(tempfile.mkdtemp())
|
||||
self.git("init", "-q")
|
||||
self.write("pkg/version.txt", 'version = "1"\n')
|
||||
self.write("pkg/src.txt", "a")
|
||||
self.commit("set version")
|
||||
|
||||
def git(self, *args):
|
||||
subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=self.repo, check=True)
|
||||
|
||||
def write(self, file, text):
|
||||
(self.repo / file).parent.mkdir(parents=True, exist_ok=True)
|
||||
(self.repo / file).write_text(text)
|
||||
|
||||
def commit(self, message):
|
||||
self.git("add", "-A")
|
||||
self.git("commit", "-q", "-m", message)
|
||||
|
||||
def changed(self):
|
||||
with mock.patch.object(packages_release, "ROOT", self.repo):
|
||||
return changed_since_publish("pkg", "1", "pkg", "pkg/version.txt", "^version =")
|
||||
|
||||
def test_compares_with_the_commit_that_set_the_version(self):
|
||||
self.assertFalse(self.changed())
|
||||
self.write("pkg/src.txt", "b")
|
||||
self.commit("change")
|
||||
self.assertTrue(self.changed())
|
||||
|
||||
def test_compares_with_the_published_commit(self):
|
||||
self.write("pkg/src.txt", "b")
|
||||
self.commit("change")
|
||||
self.git("tag", "pkg-v1")
|
||||
self.assertFalse(self.changed())
|
||||
self.write("pkg/src.txt", "c")
|
||||
self.commit("change after publish")
|
||||
self.assertTrue(self.changed())
|
||||
|
||||
def test_publish_tag_marks_the_version_published(self):
|
||||
with mock.patch.object(packages_release, "ROOT", self.repo):
|
||||
self.assertIsNone(published_tag("pkg", "1"))
|
||||
self.git("tag", "pkg-v1")
|
||||
self.assertEqual(published_tag("pkg", "1"), "refs/tags/pkg-v1")
|
||||
self.assertIsNone(published_tag("pkg", "2"))
|
||||
|
||||
def test_changed_since_looks_only_at_the_sources(self):
|
||||
base = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"], cwd=self.repo, capture_output=True, text=True
|
||||
).stdout.strip()
|
||||
self.write("other/file.txt", "o")
|
||||
self.commit("other change")
|
||||
with mock.patch.object(packages_release, "ROOT", self.repo):
|
||||
self.assertFalse(changed_since(base, ["pkg"]))
|
||||
self.write("pkg/src.txt", "b")
|
||||
self.commit("source change")
|
||||
self.assertTrue(changed_since(base, ["pkg"]))
|
||||
|
||||
def test_ignores_tests_and_other_directories(self):
|
||||
self.write("pkg/tests/t.txt", "t")
|
||||
self.write("other/file.txt", "o")
|
||||
self.commit("tests and other files")
|
||||
self.assertFalse(self.changed())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user