From 489b07ff4f9b517a5aa21815c49cac38777b893e Mon Sep 17 00:00:00 2001 From: sh <37271604+shumvgolove@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:10:31 +0400 Subject: [PATCH] ci: record package releases in simplex-chat-libs (#7719) * ci: use default registries for empty variables * ci: drop package publish tags * ci: record releases in simplex-chat-libs * ci: skip recorded versions in publish plan * nodejs: raise default test timeout for migrations * nodejs: print download progress without console.error * nodejs: use the default timeout in migrating core tests --- .github/workflows/build.yml | 33 +-- .github/workflows/packages.yml | 48 +++- packages/simplex-chat-nodejs/jest.config.js | 2 + packages/simplex-chat-nodejs/src/libs.ts | 4 +- .../simplex-chat-nodejs/tests/core.test.ts | 10 +- scripts/ci/packages_release.py | 221 +++++++++++++---- scripts/ci/test_packages_release.py | 222 ++++++++++++++++-- 7 files changed, 445 insertions(+), 95 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0bd229a576..a3400f3ffa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -132,35 +132,26 @@ jobs: echo "version=$version" >> $GITHUB_OUTPUT fi - - name: Clone libs repo - if: steps.check_version.outputs.version - uses: actions/checkout@v3 - with: - repository: ${{ github.repository }}-libs - token: ${{ secrets.NODEJS_REPO_TOKEN }} - path: libs-repo - - - name: Commit and tag libs release + - name: Record libs release id: libs_commit if: steps.check_version.outputs.version - working-directory: libs-repo + env: + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + LIBS_REPOSITORY: ${{ github.repository }}-libs + VERSION: ${{ steps.check_version.outputs.version }} run: | - tag=core-${{ steps.check_version.outputs.version }} - printf 'version: %s\ncommit: %s\n' "${{ steps.check_version.outputs.version }}" "${{ github.sha }}" > core-release - git config user.name "SimpleX Chat" - git config user.email "build@simplex.chat" - git add core-release - git commit -m "$tag" - git tag "$tag" - git push --atomic origin HEAD "$tag" - echo "tag=$tag" >> $GITHUB_OUTPUT + commit=$(python3 scripts/ci/packages_release.py record --core "$VERSION" --commit "$GITHUB_SHA") + echo "tag=core-$VERSION" >> "$GITHUB_OUTPUT" + echo "commit=$commit" >> "$GITHUB_OUTPUT" + # creates the tag on the commit that recorded the release - name: Create libs release if: steps.check_version.outputs.version uses: softprops/action-gh-release@v2 with: repository: ${{ github.repository }}-libs tag_name: ${{ steps.libs_commit.outputs.tag }} + target_commitish: ${{ steps.libs_commit.outputs.commit }} token: ${{ secrets.NODEJS_REPO_TOKEN }} # ========================= @@ -868,7 +859,7 @@ jobs: if: ${{ !cancelled() && needs.maybe-release.outputs.libs_tag }} # the called workflow can't be granted more than its caller, and its publish job needs these permissions: - contents: write + contents: read id-token: write uses: ./.github/workflows/packages.yml with: @@ -882,7 +873,7 @@ jobs: # build-linux uploads the library before its chat tests run if: ${{ !cancelled() && needs.maybe-release.outputs.core_changed }} permissions: - contents: write + contents: read id-token: write uses: ./.github/workflows/packages.yml with: diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 28011cf804..01c2c1be33 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -2,7 +2,7 @@ name: packages # Releases simplex-chat (npm, PyPI) and @simplex-chat/types (npm) from master and stable: # 1. A missing Node.js add-on release (nodejs-addon-) is built and uploaded to simplex-chat-libs. -# 2. Versions that are not published yet are published, and tagged -v on their commit. +# 2. Versions that are not published yet are published, and recorded in releases.toml of simplex-chat-libs. # npm prerelease versions get dist-tag beta, others latest. # 3. The release PR packages-release/ pins new library and add-on releases with their SHA-256 # and bumps the versions of changed packages. Merging it publishes them (step 2). @@ -208,18 +208,24 @@ jobs: if: needs.addon.outputs.release_branch && contains(github.workflow_ref, '/.github/workflows/packages.yml@') runs-on: ubuntu-latest permissions: - contents: write + contents: read id-token: write + # set by a publish step after it succeeds, so a failed publish still reports the earlier ones + outputs: + types: ${{ steps.types.outputs.version }} + nodejs: ${{ steps.nodejs.outputs.version }} + python: ${{ steps.python.outputs.version }} env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} steps: - name: Checkout Code uses: actions/checkout@v6 - with: - fetch-depth: 0 - name: Versions to publish id: plan + env: + GH_TOKEN: ${{ github.token }} + LIBS_REPOSITORY: ${{ github.repository }}-libs run: python3 scripts/ci/packages_release.py publish-plan >> "$GITHUB_OUTPUT" - name: Setup Node.js @@ -250,6 +256,7 @@ jobs: run: python3 scripts/ci/packages_release.py test-names - name: Publish @simplex-chat/types + id: types if: steps.plan.outputs.types_version working-directory: packages/simplex-chat-client/types/typescript env: @@ -258,9 +265,10 @@ jobs: run: | npm install --no-audit --no-fund npm publish --provenance --access public --tag "$DIST_TAG" - git tag "types-v$VERSION" && git push origin "types-v$VERSION" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: Publish simplex-chat (npm) + id: nodejs if: steps.plan.outputs.nodejs_version working-directory: packages/simplex-chat-nodejs env: @@ -268,7 +276,7 @@ jobs: DIST_TAG: ${{ steps.plan.outputs.nodejs_dist_tag }} run: | npm publish --provenance --access public --tag "$DIST_TAG" - git tag "nodejs-v$VERSION" && git push origin "nodejs-v$VERSION" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: Build and test simplex-chat (PyPI) if: steps.plan.outputs.python_version @@ -286,15 +294,35 @@ jobs: repository-url: ${{ vars.PYPI_UPLOAD_URL || 'https://upload.pypi.org/legacy/' }} password: ${{ secrets.PYPI_API_TOKEN }} - - name: Tag the published PyPI version + - name: Report the published PyPI version + id: python if: steps.plan.outputs.python_version env: VERSION: ${{ steps.plan.outputs.python_version }} - run: git tag "python-v$VERSION" && git push origin "python-v$VERSION" + run: echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + # The only job here that writes to simplex-chat-libs; publish runs npm and pip installs without its token. + record: + needs: publish + if: ${{ !cancelled() && (needs.publish.outputs.types || needs.publish.outputs.nodejs || needs.publish.outputs.python) }} + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v6 + + - name: Record published versions + env: + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + LIBS_REPOSITORY: ${{ github.repository }}-libs + TYPES: ${{ needs.publish.outputs.types }} + NODEJS: ${{ needs.publish.outputs.nodejs }} + PYTHON: ${{ needs.publish.outputs.python }} + run: | + python3 scripts/ci/packages_release.py record --commit "$GITHUB_SHA" \ + --types "$TYPES" --nodejs "$NODEJS" --python "$PYTHON" release: - # After publish, for its version tags. A failed publish leaves its versions unbumped. - needs: [addon, build, publish] + needs: [addon, build] # A merged release PR changes only versions and pins. if: >- !cancelled() && needs.addon.outputs.release_branch && needs.build.result != 'failure' diff --git a/packages/simplex-chat-nodejs/jest.config.js b/packages/simplex-chat-nodejs/jest.config.js index 8f37dfe843..ac94fe9001 100644 --- a/packages/simplex-chat-nodejs/jest.config.js +++ b/packages/simplex-chat-nodejs/jest.config.js @@ -6,6 +6,8 @@ process.env.SIMPLEX_ADDON_PATH ??= path.join(__dirname, "build", "Release", "sim module.exports = { preset: "ts-jest", maxWorkers: 1, + // a test with a new database runs every migration, each committed with a disk flush: seconds on slow CI runners + testTimeout: 30000, testEnvironment: "node", transform: { '^.+\\.ts$': ['ts-jest', { diff --git a/packages/simplex-chat-nodejs/src/libs.ts b/packages/simplex-chat-nodejs/src/libs.ts index b8888ede1d..013e49f5d0 100644 --- a/packages/simplex-chat-nodejs/src/libs.ts +++ b/packages/simplex-chat-nodejs/src/libs.ts @@ -111,7 +111,7 @@ export async function resolveAddonPath( export function installLibs(url: string, sha256: string, target: string, lib: string, timeoutMs: number = REQUEST_TIMEOUT_MS): Promise { return inTempDir(target, async tmp => { - console.error(`Downloading libsimplex from ${url} ...`) + process.stderr.write(`Downloading libsimplex from ${url} ...\n`) const zipPath = path.join(tmp, "libs.zip") await download(url, zipPath, timeoutMs) await verifySha256(zipPath, sha256, url) @@ -124,7 +124,7 @@ export function installLibs(url: string, sha256: string, target: string, lib: st export function installAddon(url: string, sha256: string, target: string, timeoutMs: number = REQUEST_TIMEOUT_MS): Promise { return inTempDir(target, async tmp => { - console.error(`Downloading the Node.js add-on from ${url} ...`) + process.stderr.write(`Downloading the Node.js add-on from ${url} ...\n`) const file = path.join(tmp, path.basename(target)) await download(url, file, timeoutMs) await verifySha256(file, sha256, url) diff --git a/packages/simplex-chat-nodejs/tests/core.test.ts b/packages/simplex-chat-nodejs/tests/core.test.ts index fee903d6c5..608c3939cd 100644 --- a/packages/simplex-chat-nodejs/tests/core.test.ts +++ b/packages/simplex-chat-nodejs/tests/core.test.ts @@ -169,7 +169,7 @@ describe("Core tests", () => { expect(elapsed).toBeLessThan(1800); await stopAndClose(ctrlA); await stopAndClose(ctrlB); - }, 10000); + }); it("should receive events of one controller in order", async () => { const ctrl = await core.chatMigrateInit(dbPath, "key", core.MigrationConfirmation.YesUp); @@ -183,7 +183,7 @@ describe("Core tests", () => { {type: "timedAction", action: "second"} ]); await stopAndClose(ctrl); - }, 10000); + }); it("should close the store while a receive is in flight", async () => { const ctrl = await core.chatMigrateInit(dbPath, "key", core.MigrationConfirmation.YesUp); @@ -201,7 +201,7 @@ describe("Core tests", () => { expect({closed, timerDelayBelow100ms: timerDelay < 100}).toEqual({closed: false, timerDelayBelow100ms: true}); await close; expect(await receives).toEqual([{event: undefined}, {error: "chat receiver stopped"}]); - }, 10000); + }); it("should accept loading libsimplex again from the same path", () => { expect(() => simplex.load(addonPath, libPath)).not.toThrow(); @@ -243,11 +243,11 @@ describe("Core tests", () => { .then(([ctrl]) => simplex.chat_recv_msg_wait(ctrl, 1)) .then((res) => console.log("received " + JSON.stringify(res))); `; - const child = spawnSync(process.execPath, ["-e", script], {cwd: path.join(__dirname, ".."), timeout: 10000, encoding: "utf8"}); + const child = spawnSync(process.execPath, ["-e", script], {cwd: path.join(__dirname, ".."), timeout: 25000, encoding: "utf8"}); if (child.status !== 0 || child.signal !== null) console.log("child stderr:", child.stderr); expect({status: child.status, signal: child.signal, stdout: child.stdout.trim()}) .toEqual({status: 0, signal: null, stdout: 'received ""'}); - }, 15000); + }); it("should not crash when closing stopped controllers repeatedly", async () => { const script = ` diff --git a/scripts/ci/packages_release.py b/scripts/ci/packages_release.py index 5378f311b5..488d35bb8c 100644 --- a/scripts/ci/packages_release.py +++ b/scripts/ci/packages_release.py @@ -6,21 +6,27 @@ changes print whether the changes since a commit touch the chat core and build-plan print the platforms to build the add-on on and whether to run the package tests prepare update pins and versions for a release PR publish-plan print the package versions that are not published yet +record record releases and published versions in releases.toml of simplex-chat-libs test-names rename the packages to NPM_SCOPE and PYPI_PROJECT (test repositories) """ from __future__ import annotations import argparse +import base64 +import copy import hashlib import json import os import re import subprocess import sys +import tomllib import urllib.error import urllib.request +from collections.abc import Callable from dataclasses import dataclass, field, replace +from typing import Any from pathlib import Path ROOT = Path(__file__).resolve().parents[2] @@ -52,6 +58,9 @@ ADDON_PLATFORMS = list(ADDON_RUNNERS) CORE_TEST_PLATFORM = "linux-x86_64" ADDON_ASSETS = [f"simplex-chat-nodejs-{platform}.node" for platform in ADDON_PLATFORMS] SHA256_HEX = re.compile(r"^[0-9a-f]{64}$") +RELEASES_FILE = "releases.toml" +RECORD_ATTEMPTS = 5 +TOML_BARE_KEY = re.compile(r"^[A-Za-z0-9_-]+$") # Pure functions @@ -285,6 +294,47 @@ def build_matrix(platforms: list[str]) -> str: return json.dumps([{"platform": platform, "os": ADDON_RUNNERS[platform]} for platform in platforms]) +def recorded_commit(releases: dict, key: str, version: str) -> str | None: + """The simplex-chat commit recorded for a version; key is "core", "npm/" or "pypi/".""" + table = releases + for part in key.split("/", 1): + table = table.get(part, {}) + commit = table.get(version) + return commit if isinstance(commit, str) else None + + +def add_releases(releases: dict, entries: list[tuple[str, str]], commit: str) -> dict: + """A recorded version keeps its first commit: re-running a release does not rewrite history.""" + result = copy.deepcopy(releases) + for key, version in entries: + table = result + for part in key.split("/", 1): + table = table.setdefault(part, {}) + table.setdefault(version, commit) + return result + + +def render_releases(releases: dict) -> str: + def toml_key(key: str) -> str: + return key if TOML_BARE_KEY.match(key) else json.dumps(key) + + sections: list[str] = [] + + # in the order of recording: entries are only added, so that order is chronological + def walk(path: list[str], table: dict) -> None: + values = [(k, v) for k, v in table.items() if isinstance(v, str)] + if values: + lines = [f"[{'.'.join(toml_key(part) for part in path)}]"] + lines += [f"{toml_key(version)} = {json.dumps(commit)}" for version, commit in values] + sections.append("\n".join(lines) + "\n") + for name, child in table.items(): + if isinstance(child, dict): + walk([*path, name], child) + + walk([], releases) + return "\n".join(sections) + + # Repository and registry access @@ -345,29 +395,35 @@ def print_build_plan(args: argparse.Namespace) -> None: print(f"test={'true' if result.test else ''}") -def publish_tag(package: str, version: str) -> str: - """Tag of the commit a version was published from (packages.yml).""" - return f"{package}-v{version}" +def package_state( + key: str, + version: str, + directory: str, + version_file: str, + version_pattern: str, + releases: dict, + registry_published: Callable[[], bool], +) -> Package: + """A recorded version is published from its recorded commit; the registry is asked about other versions, + which are compared with the commit that set them.""" + recorded = recorded_commit(releases, key, version) + base = ( + recorded + if recorded and commit_exists(recorded) + else git("log", "-1", "--format=%H", "-G", version_pattern, "--", version_file) + ) + changed = changed_since(base, [directory, f":!{directory}/tests"]) + return Package(version, changed, recorded is not None or registry_published()) -def published_tag(package: str, version: str) -> str | None: - """The publish tag of the version, when it exists: registries list new versions with a delay.""" - tag = f"refs/tags/{publish_tag(package, version)}" - found = subprocess.run(["git", "rev-parse", "--verify", "--quiet", tag], cwd=ROOT, capture_output=True) - return tag if found.returncode == 0 else None - - -def changed_since_publish(package: str, version: str, directory: str, version_file: str, version_pattern: str) -> bool: - """Shipped files changed since the version was published, or since it was set when it was not published yet.""" - tag = published_tag(package, version) - base = tag or git("log", "-1", "--format=%H", "-G", version_pattern, "--", version_file) - diff = subprocess.run(["git", "diff", "--quiet", base, "HEAD", "--", directory, f":!{directory}/tests"], cwd=ROOT) - return diff.returncode != 0 - - -def http_json(url: str, headers: dict[str, str] | None = None) -> dict | None: - """The JSON at `url`, or None for 404.""" - request = urllib.request.Request(url, headers={"User-Agent": "simplex-chat-release", **(headers or {})}) +def http_json(url: str, headers: dict[str, str] | None = None, method: str = "GET", body: dict | None = None) -> Any: + """The JSON response, or None for 404.""" + request = urllib.request.Request( + url, + headers={"User-Agent": "simplex-chat-release", **(headers or {})}, + method=method, + data=json.dumps(body).encode() if body is not None else None, + ) try: with urllib.request.urlopen(request, timeout=60) as response: return json.load(response) @@ -377,8 +433,9 @@ def http_json(url: str, headers: dict[str, str] | None = None) -> dict | None: raise +# Workflows pass unset repository variables as empty strings. def npm_published(name: str, version: str) -> bool: - registry = os.environ.get("NPM_REGISTRY", "https://registry.npmjs.org") + registry = os.environ.get("NPM_REGISTRY") or "https://registry.npmjs.org" package = registry_name(name, os.environ.get("NPM_SCOPE")).replace("/", "%2F") return http_json(f"{registry}/{package}/{version}") is not None @@ -388,24 +445,81 @@ def python_project() -> str: def pypi_published(version: str) -> bool: - index = os.environ.get("PYPI_URL", "https://pypi.org") + index = os.environ.get("PYPI_URL") or "https://pypi.org" project = os.environ.get("PYPI_PROJECT") or python_project() return http_json(f"{index}/pypi/{project}/{version}/json") is not None -def github_release(tag: str) -> dict | None: - repo = os.environ.get("LIBS_REPOSITORY", "simplex-chat/simplex-chat-libs") +def libs_api(path: str, method: str = "GET", body: dict | None = None) -> Any: + """GitHub API of the simplex-chat-libs repository.""" + repo = os.environ.get("LIBS_REPOSITORY") or "simplex-chat/simplex-chat-libs" headers = {"Accept": "application/vnd.github+json"} if token := os.environ.get("GH_TOKEN"): headers["Authorization"] = f"Bearer {token}" - return http_json(f"https://api.github.com/repos/{repo}/releases/tags/{tag}", headers) + return http_json(f"https://api.github.com/repos/{repo}/{path}", headers, method, body) -def npm_package(key: str, directory: str) -> Package: +def github_release(tag: str) -> dict | None: + return libs_api(f"releases/tags/{tag}") + + +def read_releases() -> tuple[dict, str | None]: + """The releases and the blob of the file, which a change must name.""" + file = libs_api(f"contents/{RELEASES_FILE}") + if file is None: + return {}, None + return tomllib.loads(base64.b64decode(file["content"]).decode()), file["sha"] + + +def record_releases(entries: list[tuple[str, str]], commit: str) -> str: + """Adds the entries in one commit of simplex-chat-libs and returns that commit.""" + message = ", ".join(f"{key} {version}" for key, version in entries) + attempt = 1 + while True: + releases, blob = read_releases() + updated = add_releases(releases, entries, commit) + if blob and updated == releases: + return libs_api(f"commits?path={RELEASES_FILE}&per_page=1")[0]["sha"] + body = { + "message": message, + "content": base64.b64encode(render_releases(updated).encode()).decode(), + "committer": {"name": "SimpleX Chat", "email": "build@simplex.chat"}, + **({"sha": blob} if blob else {}), + } + try: + return libs_api(f"contents/{RELEASES_FILE}", "PUT", body)["commit"]["sha"] + except urllib.error.HTTPError as e: + # 409 and 422: another writer changed or created the file since it was read + if e.code not in (409, 422) or attempt == RECORD_ATTEMPTS: + raise + attempt += 1 + + +def commit_exists(commit: str) -> bool: + found = subprocess.run(["git", "cat-file", "-e", f"{commit}^{{commit}}"], cwd=ROOT, capture_output=True) + return found.returncode == 0 + + +def npm_key(directory: str) -> str: + return f"npm/{read_json(f'{directory}/package.json')['name']}" + + +def pypi_key() -> str: + return f"pypi/{python_project()}" + + +def npm_package(directory: str, releases: dict) -> Package: package = read_json(f"{directory}/package.json") - changed = changed_since_publish(key, package["version"], directory, f"{directory}/package.json", '"version":') - published = bool(published_tag(key, package["version"])) or npm_published(package["name"], package["version"]) - return Package(package["version"], changed, published) + name, version = package["name"], package["version"] + return package_state( + npm_key(directory), + version, + directory, + f"{directory}/package.json", + '"version":', + releases, + lambda: npm_published(name, version), + ) def prepare(args: argparse.Namespace) -> None: @@ -413,7 +527,7 @@ def prepare(args: argparse.Namespace) -> None: libs_tag = f"core-{cabal_version}" addon = current_addon_tag() python_version, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE)) - python_changed = changed_since_publish("python", python_version, PYTHON_DIR, PYTHON_VERSION_FILE, "^__version__ =") + releases, _ = read_releases() result = plan( State( libs_tag=libs_tag, @@ -422,12 +536,16 @@ def prepare(args: argparse.Namespace) -> None: addon_digests=release_digests(github_release(addon), ADDON_ASSETS), nodejs_pins=parse_release(read_text(RELEASE_FILE)), python_pins=python_pins, - types=npm_package("types", TYPES_DIR), - nodejs=npm_package("nodejs", NODEJS_DIR), - python=Package( + types=npm_package(TYPES_DIR, releases), + nodejs=npm_package(NODEJS_DIR, releases), + python=package_state( + pypi_key(), python_version, - python_changed, - bool(published_tag("python", python_version)) or pypi_published(python_version), + PYTHON_DIR, + PYTHON_VERSION_FILE, + "^__version__ =", + releases, + lambda: pypi_published(python_version), ), types_version=args.types_version, nodejs_version=args.nodejs_version, @@ -450,14 +568,31 @@ def prepare(args: argparse.Namespace) -> None: def publish_plan() -> None: + """A recorded version is published: npm and PyPI list a new version minutes after it is published.""" + releases, _ = read_releases() for key, directory in [("types", TYPES_DIR), ("nodejs", NODEJS_DIR)]: package = read_json(f"{directory}/package.json") - pending = not published_tag(key, package["version"]) and not npm_published(package["name"], package["version"]) - print(f"{key}_version={package['version'] if pending else ''}") + published = recorded_commit(releases, npm_key(directory), package["version"]) or npm_published( + package["name"], package["version"] + ) + print(f"{key}_version={'' if published else package['version']}") print(f"{key}_dist_tag={dist_tag(package['version'])}") version, _ = parse_python_version(read_text(PYTHON_VERSION_FILE)) - pending = not published_tag("python", version) and not pypi_published(version) - print(f"python_version={version if pending else ''}") + published = recorded_commit(releases, pypi_key(), version) or pypi_published(version) + print(f"python_version={'' if published else version}") + + +def record(args: argparse.Namespace) -> None: + versions = [ + ("core", args.core), + (npm_key(TYPES_DIR), args.types), + (npm_key(NODEJS_DIR), args.nodejs), + (pypi_key(), args.python), + ] + entries = [(key, version) for key, version in versions if version] + if not entries: + sys.exit("nothing to record") + print(record_releases(entries, args.commit)) def test_names() -> None: @@ -509,6 +644,10 @@ def main(argv: list[str]) -> None: prepare_parser.add_argument(f"--{package}-version", help="this version instead of a bump when needed") prepare_parser.add_argument("--summary", help="file for the PR title and body") commands.add_parser("publish-plan") + record_parser = commands.add_parser("record") + record_parser.add_argument("--commit", required=True, help="the simplex-chat commit released or published from") + for name in ["core", "types", "nodejs", "python"]: + record_parser.add_argument(f"--{name}", default="", help=f"{name} version, recorded when not empty") commands.add_parser("test-names") args = parser.parse_args(argv) if args.command == "addon-tag": @@ -521,6 +660,8 @@ def main(argv: list[str]) -> None: prepare(args) elif args.command == "publish-plan": publish_plan() + elif args.command == "record": + record(args) else: test_names() diff --git a/scripts/ci/test_packages_release.py b/scripts/ci/test_packages_release.py index 4d0f0ac2f4..634260cabc 100644 --- a/scripts/ci/test_packages_release.py +++ b/scripts/ci/test_packages_release.py @@ -1,4 +1,10 @@ +import base64 +import contextlib +import email.message +import io import subprocess +import tomllib +import urllib.error import tempfile import unittest from dataclasses import replace @@ -8,6 +14,9 @@ from unittest import mock import packages_release from packages_release import ( ADDON_ASSETS, + NODEJS_DIR, + PYTHON_VERSION_FILE, + TYPES_DIR, ADDON_PLATFORMS, BuildPlan, LIBS_ASSETS, @@ -20,12 +29,18 @@ from packages_release import ( bump_npm_version, bump_python_version, changed_since, - changed_since_publish, - published_tag, + add_releases, + package_state, + publish_plan, + record_releases, + recorded_commit, + render_releases, dist_tag, + npm_published, parse_python_version, parse_release, plan, + pypi_published, registry_name, release_digests, render_python_version, @@ -220,7 +235,132 @@ class BuildPlanTest(unittest.TestCase): ) -class ChangedSincePublishTest(unittest.TestCase): +class RegistryUrlTest(unittest.TestCase): + def urls(self, env: dict[str, str]) -> list[str]: + with ( + mock.patch.dict("os.environ", env), + mock.patch.object(packages_release, "http_json", return_value={}) as get, + ): + npm_published("simplex-chat", "1.0.0") + pypi_published("1.0.0") + packages_release.github_release("core-1.0.0.1") + return [call.args[0] for call in get.call_args_list] + + def test_empty_variables_use_the_public_registries(self): + self.assertEqual( + self.urls({"NPM_REGISTRY": "", "NPM_SCOPE": "", "PYPI_URL": "", "PYPI_PROJECT": "", "LIBS_REPOSITORY": ""}), + [ + "https://registry.npmjs.org/simplex-chat/1.0.0", + "https://pypi.org/pypi/simplex-chat/1.0.0/json", + "https://api.github.com/repos/simplex-chat/simplex-chat-libs/releases/tags/core-1.0.0.1", + ], + ) + + def test_variables_select_test_registries(self): + self.assertEqual( + self.urls( + { + "NPM_SCOPE": "@test", + "PYPI_URL": "https://test.pypi.org", + "PYPI_PROJECT": "sx-test", + "LIBS_REPOSITORY": "a/b-libs", + } + ), + [ + "https://registry.npmjs.org/@test%2Fsimplex-chat/1.0.0", + "https://test.pypi.org/pypi/sx-test/1.0.0/json", + "https://api.github.com/repos/a/b-libs/releases/tags/core-1.0.0.1", + ], + ) + + +class ReleasesFileTest(unittest.TestCase): + RELEASES = { + "core": {"7.1.0.11": "c1"}, + "npm": {"simplex-chat": {"7.1.0-beta.6.3": "n1"}, "@simplex-chat/types": {"0.12.2": "t1"}}, + "pypi": {"simplex-chat": {"7.1.0b6": "p1"}}, + } + + def test_renders_toml_that_reads_back(self): + text = render_releases(self.RELEASES) + self.assertEqual(tomllib.loads(text), self.RELEASES) + self.assertIn('[core]\n"7.1.0.11" = "c1"\n', text) + self.assertIn('[npm."@simplex-chat/types"]\n', text) + self.assertIn("[npm.simplex-chat]\n", text) + + def test_keeps_the_order_of_recording(self): + releases = {"npm": {"simplex-chat": {"7.1.0-beta.6.9": "a"}}} + updated = add_releases(releases, [("npm/simplex-chat", "7.1.0-beta.6.10"), ("core", "7.1.0.15")], "b") + self.assertEqual( + render_releases(updated), + '[npm.simplex-chat]\n"7.1.0-beta.6.9" = "a"\n"7.1.0-beta.6.10" = "b"\n\n[core]\n"7.1.0.15" = "b"\n', + ) + + def test_finds_the_recorded_commit(self): + self.assertEqual(recorded_commit(self.RELEASES, "npm/@simplex-chat/types", "0.12.2"), "t1") + self.assertEqual(recorded_commit(self.RELEASES, "core", "7.1.0.11"), "c1") + self.assertIsNone(recorded_commit(self.RELEASES, "npm/simplex-chat", "7.1.0-beta.6.4")) + self.assertIsNone(recorded_commit({}, "pypi/simplex-chat", "7.1.0b6")) + + def test_adds_entries_and_keeps_recorded_commits(self): + updated = add_releases(self.RELEASES, [("npm/simplex-chat", "7.1.0-beta.6.4"), ("core", "7.1.0.11")], "x") + self.assertEqual(updated["npm"]["simplex-chat"], {"7.1.0-beta.6.3": "n1", "7.1.0-beta.6.4": "x"}) + self.assertEqual(updated["core"], {"7.1.0.11": "c1"}) + self.assertEqual(self.RELEASES["npm"]["simplex-chat"], {"7.1.0-beta.6.3": "n1"}) + + +class RecordReleasesTest(unittest.TestCase): + def file(self, releases, blob): + return {"content": base64.b64encode(render_releases(releases).encode()).decode(), "sha": blob} + + def conflict(self): + error = urllib.error.HTTPError("url", 409, "Conflict", email.message.Message(), io.BytesIO()) + self.addCleanup(error.close) + return error + + def test_creates_the_file_and_returns_the_commit(self): + calls = [] + + def api(path, method="GET", body=None): + calls.append((path, method, body)) + return None if method == "GET" else {"commit": {"sha": "libs1"}} + + with mock.patch.object(packages_release, "libs_api", api): + self.assertEqual(record_releases([("core", "7.1.0.12")], "sx1"), "libs1") + _, method, body = calls[1] + self.assertEqual((method, body["message"], "sha" in body), ("PUT", "core 7.1.0.12", False)) + self.assertEqual(tomllib.loads(base64.b64decode(body["content"]).decode()), {"core": {"7.1.0.12": "sx1"}}) + + def test_retries_when_another_writer_changed_the_file(self): + reads = iter([self.file({"core": {"1": "a"}}, "blob1"), self.file({"core": {"1": "a", "2": "b"}}, "blob2")]) + puts = [] + + def api(path, method="GET", body=None): + if method == "GET": + return next(reads) + puts.append(body) + if len(puts) == 1: + raise self.conflict() + return {"commit": {"sha": "libs2"}} + + with mock.patch.object(packages_release, "libs_api", api): + self.assertEqual(record_releases([("npm/simplex-chat", "3")], "sx3"), "libs2") + self.assertEqual([body["sha"] for body in puts], ["blob1", "blob2"]) + self.assertEqual( + tomllib.loads(base64.b64decode(puts[1]["content"]).decode()), + {"core": {"1": "a", "2": "b"}, "npm": {"simplex-chat": {"3": "sx3"}}}, + ) + + def test_recorded_entries_return_the_last_commit_of_the_file(self): + def api(path, method="GET", body=None): + self.assertEqual(method, "GET") + return [{"sha": "libs0"}] if path.startswith("commits") else self.file({"core": {"1": "a"}}, "blob1") + + with mock.patch.object(packages_release, "libs_api", api): + self.assertEqual(record_releases([("core", "1")], "other"), "libs0") + + +class PackageStateTest(unittest.TestCase): def setUp(self): self.repo = Path(tempfile.mkdtemp()) self.git("init", "-q") @@ -239,9 +379,66 @@ class ChangedSincePublishTest(unittest.TestCase): self.git("add", "-A") self.git("commit", "-q", "-m", message) - def changed(self): + def state(self, releases=None, registry_published=False): with mock.patch.object(packages_release, "ROOT", self.repo): - return changed_since_publish("pkg", "1", "pkg", "pkg/version.txt", "^version =") + return package_state( + "npm/pkg", "1", "pkg", "pkg/version.txt", "^version =", releases or {}, lambda: registry_published + ) + + def changed(self): + return self.state().changed + + def head(self): + return subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=self.repo, capture_output=True, text=True + ).stdout.strip() + + def test_recorded_version_is_published_and_compared_with_its_commit(self): + self.write("pkg/src.txt", "b") + self.commit("change published with version 1") + releases = {"npm": {"pkg": {"1": self.head()}}} + self.assertEqual(self.state(releases), Package("1", changed=False, published=True)) + self.write("pkg/src.txt", "c") + self.commit("change after the publish") + self.assertEqual(self.state(releases), Package("1", changed=True, published=True)) + + def test_unrecorded_version_asks_the_registry(self): + self.assertFalse(self.state(registry_published=False).published) + self.assertTrue(self.state(registry_published=True).published) + + def test_publish_plan_skips_recorded_versions_before_the_registries_list_them(self): + self.write(f"{TYPES_DIR}/package.json", '{"name": "@simplex-chat/types", "version": "0.1.0"}') + self.write(f"{NODEJS_DIR}/package.json", '{"name": "simplex-chat", "version": "1.0.0-beta.1.1"}') + self.write(PYTHON_VERSION_FILE, render_python_version("1.0.0b1.post1", Pins("core-1", {}))) + self.write("pyproject.toml", "") + self.commit("packages") + releases = {"npm": {"simplex-chat": {"1.0.0-beta.1.1": "c"}}, "pypi": {"simplex-chat": {"1.0.0b1.post1": "c"}}} + output = io.StringIO() + with ( + mock.patch.object(packages_release, "ROOT", self.repo), + mock.patch.object(packages_release, "read_releases", return_value=(releases, "blob")), + mock.patch.object(packages_release, "python_project", return_value="simplex-chat"), + mock.patch.object(packages_release, "npm_published", return_value=False), + mock.patch.object(packages_release, "pypi_published", return_value=False), + contextlib.redirect_stdout(output), + ): + publish_plan() + self.assertEqual( + output.getvalue().splitlines(), + [ + "types_version=0.1.0", + "types_dist_tag=latest", + "nodejs_version=", + "nodejs_dist_tag=beta", + "python_version=", + ], + ) + + def test_unknown_recorded_commit_falls_back_to_the_version_commit(self): + self.write("pkg/src.txt", "b") + self.commit("change") + state = self.state({"npm": {"pkg": {"1": "0" * 40}}}) + self.assertEqual(state, Package("1", changed=True, published=True)) def test_compares_with_the_commit_that_set_the_version(self): self.assertFalse(self.changed()) @@ -249,21 +446,12 @@ class ChangedSincePublishTest(unittest.TestCase): self.commit("change") self.assertTrue(self.changed()) - def test_compares_with_the_published_commit(self): + def test_new_version_resets_the_comparison(self): self.write("pkg/src.txt", "b") self.commit("change") - self.git("tag", "pkg-v1") + self.write("pkg/version.txt", 'version = "2"\n') + self.commit("set version 2") self.assertFalse(self.changed()) - self.write("pkg/src.txt", "c") - self.commit("change after publish") - self.assertTrue(self.changed()) - - def test_publish_tag_marks_the_version_published(self): - with mock.patch.object(packages_release, "ROOT", self.repo): - self.assertIsNone(published_tag("pkg", "1")) - self.git("tag", "pkg-v1") - self.assertEqual(published_tag("pkg", "1"), "refs/tags/pkg-v1") - self.assertIsNone(published_tag("pkg", "2")) def test_changed_since_looks_only_at_the_sources(self): base = subprocess.run(