ci: record package releases in simplex-chat-libs (#7719)

* ci: use default registries for empty variables

* ci: drop package publish tags

* ci: record releases in simplex-chat-libs

* ci: skip recorded versions in publish plan

* nodejs: raise default test timeout for migrations

* nodejs: print download progress without console.error

* nodejs: use the default timeout in migrating core tests

.github/workflows/build.yml

@@ -132,35 +132,26 @@ jobs:
             echo "version=$version" >> $GITHUB_OUTPUT
           fi
 
-      - name: Clone libs repo
-        if: steps.check_version.outputs.version
-        uses: actions/checkout@v3
-        with:
-          repository: ${{ github.repository }}-libs
-          token: ${{ secrets.NODEJS_REPO_TOKEN }}
-          path: libs-repo
-
-      - name: Commit and tag libs release
+      - name: Record libs release
         id: libs_commit
         if: steps.check_version.outputs.version
-        working-directory: libs-repo
+        env:
+          GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }}
+          LIBS_REPOSITORY: ${{ github.repository }}-libs
+          VERSION: ${{ steps.check_version.outputs.version }}
         run: |
-          tag=core-${{ steps.check_version.outputs.version }}
-          printf 'version: %s\ncommit: %s\n' "${{ steps.check_version.outputs.version }}" "${{ github.sha }}" > core-release
-          git config user.name "SimpleX Chat"
-          git config user.email "build@simplex.chat"
-          git add core-release
-          git commit -m "$tag"
-          git tag "$tag"
-          git push --atomic origin HEAD "$tag"
-          echo "tag=$tag" >> $GITHUB_OUTPUT
+          commit=$(python3 scripts/ci/packages_release.py record --core "$VERSION" --commit "$GITHUB_SHA")
+          echo "tag=core-$VERSION" >> "$GITHUB_OUTPUT"
+          echo "commit=$commit" >> "$GITHUB_OUTPUT"
 
+      # creates the tag on the commit that recorded the release
       - name: Create libs release
         if: steps.check_version.outputs.version
         uses: softprops/action-gh-release@v2
         with:
           repository: ${{ github.repository }}-libs
           tag_name: ${{ steps.libs_commit.outputs.tag }}
+          target_commitish: ${{ steps.libs_commit.outputs.commit }}
           token: ${{ secrets.NODEJS_REPO_TOKEN }}
 
 # =========================
@@ -868,7 +859,7 @@ jobs:
     if: ${{ !cancelled() && needs.maybe-release.outputs.libs_tag }}
     # the called workflow can't be granted more than its caller, and its publish job needs these
     permissions:
-      contents: write
+      contents: read
       id-token: write
     uses: ./.github/workflows/packages.yml
     with:
@@ -882,7 +873,7 @@ jobs:
     # build-linux uploads the library before its chat tests run
     if: ${{ !cancelled() && needs.maybe-release.outputs.core_changed }}
     permissions:
-      contents: write
+      contents: read
       id-token: write
     uses: ./.github/workflows/packages.yml
     with:

.github/workflows/packages.yml

@@ -2,7 +2,7 @@ name: packages
 
 # Releases simplex-chat (npm, PyPI) and @simplex-chat/types (npm) from master and stable:
 # 1. A missing Node.js add-on release (nodejs-addon-<hash of its sources>) is built and uploaded to simplex-chat-libs.
-# 2. Versions that are not published yet are published, and tagged <package>-v<version> on their commit.
+# 2. Versions that are not published yet are published, and recorded in releases.toml of simplex-chat-libs.
 #    npm prerelease versions get dist-tag beta, others latest.
 # 3. The release PR packages-release/<branch> pins new library and add-on releases with their SHA-256
 #    and bumps the versions of changed packages. Merging it publishes them (step 2).
@@ -208,18 +208,24 @@ jobs:
     if: needs.addon.outputs.release_branch && contains(github.workflow_ref, '/.github/workflows/packages.yml@')
     runs-on: ubuntu-latest
     permissions:
-      contents: write
+      contents: read
       id-token: write
+    # set by a publish step after it succeeds, so a failed publish still reports the earlier ones
+    outputs:
+      types: ${{ steps.types.outputs.version }}
+      nodejs: ${{ steps.nodejs.outputs.version }}
+      python: ${{ steps.python.outputs.version }}
     env:
       NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
     steps:
       - name: Checkout Code
         uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
 
       - name: Versions to publish
         id: plan
+        env:
+          GH_TOKEN: ${{ github.token }}
+          LIBS_REPOSITORY: ${{ github.repository }}-libs
         run: python3 scripts/ci/packages_release.py publish-plan >> "$GITHUB_OUTPUT"
 
       - name: Setup Node.js
@@ -250,6 +256,7 @@ jobs:
         run: python3 scripts/ci/packages_release.py test-names
 
       - name: Publish @simplex-chat/types
+        id: types
         if: steps.plan.outputs.types_version
         working-directory: packages/simplex-chat-client/types/typescript
         env:
@@ -258,9 +265,10 @@ jobs:
         run: |
           npm install --no-audit --no-fund
           npm publish --provenance --access public --tag "$DIST_TAG"
-          git tag "types-v$VERSION" && git push origin "types-v$VERSION"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Publish simplex-chat (npm)
+        id: nodejs
         if: steps.plan.outputs.nodejs_version
         working-directory: packages/simplex-chat-nodejs
         env:
@@ -268,7 +276,7 @@ jobs:
           DIST_TAG: ${{ steps.plan.outputs.nodejs_dist_tag }}
         run: |
           npm publish --provenance --access public --tag "$DIST_TAG"
-          git tag "nodejs-v$VERSION" && git push origin "nodejs-v$VERSION"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Build and test simplex-chat (PyPI)
         if: steps.plan.outputs.python_version
@@ -286,15 +294,35 @@ jobs:
           repository-url: ${{ vars.PYPI_UPLOAD_URL || 'https://upload.pypi.org/legacy/' }}
           password: ${{ secrets.PYPI_API_TOKEN }}
 
-      - name: Tag the published PyPI version
+      - name: Report the published PyPI version
+        id: python
         if: steps.plan.outputs.python_version
         env:
           VERSION: ${{ steps.plan.outputs.python_version }}
-        run: git tag "python-v$VERSION" && git push origin "python-v$VERSION"
+        run: echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+  # The only job here that writes to simplex-chat-libs; publish runs npm and pip installs without its token.
+  record:
+    needs: publish
+    if: ${{ !cancelled() && (needs.publish.outputs.types || needs.publish.outputs.nodejs || needs.publish.outputs.python) }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v6
+
+      - name: Record published versions
+        env:
+          GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }}
+          LIBS_REPOSITORY: ${{ github.repository }}-libs
+          TYPES: ${{ needs.publish.outputs.types }}
+          NODEJS: ${{ needs.publish.outputs.nodejs }}
+          PYTHON: ${{ needs.publish.outputs.python }}
+        run: |
+          python3 scripts/ci/packages_release.py record --commit "$GITHUB_SHA" \
+            --types "$TYPES" --nodejs "$NODEJS" --python "$PYTHON"
 
   release:
-    # After publish, for its version tags. A failed publish leaves its versions unbumped.
-    needs: [addon, build, publish]
+    needs: [addon, build]
     # A merged release PR changes only versions and pins.
     if: >-
       !cancelled() && needs.addon.outputs.release_branch && needs.build.result != 'failure'

packages/simplex-chat-nodejs/jest.config.js

@@ -6,6 +6,8 @@ process.env.SIMPLEX_ADDON_PATH ??= path.join(__dirname, "build", "Release", "sim
 module.exports = {
   preset: "ts-jest",
   maxWorkers: 1,
+  // a test with a new database runs every migration, each committed with a disk flush: seconds on slow CI runners
+  testTimeout: 30000,
   testEnvironment: "node",
   transform: {
     '^.+\\.ts$': ['ts-jest', {

packages/simplex-chat-nodejs/src/libs.ts

@@ -111,7 +111,7 @@ export async function resolveAddonPath(
 
 export function installLibs(url: string, sha256: string, target: string, lib: string, timeoutMs: number = REQUEST_TIMEOUT_MS): Promise<void> {
   return inTempDir(target, async tmp => {
-    console.error(`Downloading libsimplex from ${url} ...`)
+    process.stderr.write(`Downloading libsimplex from ${url} ...\n`)
     const zipPath = path.join(tmp, "libs.zip")
     await download(url, zipPath, timeoutMs)
     await verifySha256(zipPath, sha256, url)
@@ -124,7 +124,7 @@ export function installLibs(url: string, sha256: string, target: string, lib: st
 
 export function installAddon(url: string, sha256: string, target: string, timeoutMs: number = REQUEST_TIMEOUT_MS): Promise<void> {
   return inTempDir(target, async tmp => {
-    console.error(`Downloading the Node.js add-on from ${url} ...`)
+    process.stderr.write(`Downloading the Node.js add-on from ${url} ...\n`)
     const file = path.join(tmp, path.basename(target))
     await download(url, file, timeoutMs)
     await verifySha256(file, sha256, url)

packages/simplex-chat-nodejs/tests/core.test.ts

@@ -169,7 +169,7 @@ describe("Core tests", () => {
     expect(elapsed).toBeLessThan(1800);
     await stopAndClose(ctrlA);
     await stopAndClose(ctrlB);
-  }, 10000);
+  });
 
   it("should receive events of one controller in order", async () => {
     const ctrl = await core.chatMigrateInit(dbPath, "key", core.MigrationConfirmation.YesUp);
@@ -183,7 +183,7 @@ describe("Core tests", () => {
       {type: "timedAction", action: "second"}
     ]);
     await stopAndClose(ctrl);
-  }, 10000);
+  });
 
   it("should close the store while a receive is in flight", async () => {
     const ctrl = await core.chatMigrateInit(dbPath, "key", core.MigrationConfirmation.YesUp);
@@ -201,7 +201,7 @@ describe("Core tests", () => {
     expect({closed, timerDelayBelow100ms: timerDelay < 100}).toEqual({closed: false, timerDelayBelow100ms: true});
     await close;
     expect(await receives).toEqual([{event: undefined}, {error: "chat receiver stopped"}]);
-  }, 10000);
+  });
 
   it("should accept loading libsimplex again from the same path", () => {
     expect(() => simplex.load(addonPath, libPath)).not.toThrow();
@@ -243,11 +243,11 @@ describe("Core tests", () => {
         .then(([ctrl]) => simplex.chat_recv_msg_wait(ctrl, 1))
         .then((res) => console.log("received " + JSON.stringify(res)));
     `;
-    const child = spawnSync(process.execPath, ["-e", script], {cwd: path.join(__dirname, ".."), timeout: 10000, encoding: "utf8"});
+    const child = spawnSync(process.execPath, ["-e", script], {cwd: path.join(__dirname, ".."), timeout: 25000, encoding: "utf8"});
     if (child.status !== 0 || child.signal !== null) console.log("child stderr:", child.stderr);
     expect({status: child.status, signal: child.signal, stdout: child.stdout.trim()})
       .toEqual({status: 0, signal: null, stdout: 'received ""'});
-  }, 15000);
+  });
 
   it("should not crash when closing stopped controllers repeatedly", async () => {
     const script = `

scripts/ci/packages_release.py

@@ -6,21 +6,27 @@ changes        print whether the changes since a commit touch the chat core and
 build-plan     print the platforms to build the add-on on and whether to run the package tests
 prepare        update pins and versions for a release PR
 publish-plan   print the package versions that are not published yet
+record         record releases and published versions in releases.toml of simplex-chat-libs
 test-names     rename the packages to NPM_SCOPE and PYPI_PROJECT (test repositories)
 """
 
 from __future__ import annotations
 
 import argparse
+import base64
+import copy
 import hashlib
 import json
 import os
 import re
 import subprocess
 import sys
+import tomllib
 import urllib.error
 import urllib.request
+from collections.abc import Callable
 from dataclasses import dataclass, field, replace
+from typing import Any
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parents[2]
@@ -52,6 +58,9 @@ ADDON_PLATFORMS = list(ADDON_RUNNERS)
 CORE_TEST_PLATFORM = "linux-x86_64"
 ADDON_ASSETS = [f"simplex-chat-nodejs-{platform}.node" for platform in ADDON_PLATFORMS]
 SHA256_HEX = re.compile(r"^[0-9a-f]{64}$")
+RELEASES_FILE = "releases.toml"
+RECORD_ATTEMPTS = 5
+TOML_BARE_KEY = re.compile(r"^[A-Za-z0-9_-]+$")
 
 
 # Pure functions
@@ -285,6 +294,47 @@ def build_matrix(platforms: list[str]) -> str:
     return json.dumps([{"platform": platform, "os": ADDON_RUNNERS[platform]} for platform in platforms])
 
 
+def recorded_commit(releases: dict, key: str, version: str) -> str | None:
+    """The simplex-chat commit recorded for a version; key is "core", "npm/<name>" or "pypi/<name>"."""
+    table = releases
+    for part in key.split("/", 1):
+        table = table.get(part, {})
+    commit = table.get(version)
+    return commit if isinstance(commit, str) else None
+
+
+def add_releases(releases: dict, entries: list[tuple[str, str]], commit: str) -> dict:
+    """A recorded version keeps its first commit: re-running a release does not rewrite history."""
+    result = copy.deepcopy(releases)
+    for key, version in entries:
+        table = result
+        for part in key.split("/", 1):
+            table = table.setdefault(part, {})
+        table.setdefault(version, commit)
+    return result
+
+
+def render_releases(releases: dict) -> str:
+    def toml_key(key: str) -> str:
+        return key if TOML_BARE_KEY.match(key) else json.dumps(key)
+
+    sections: list[str] = []
+
+    # in the order of recording: entries are only added, so that order is chronological
+    def walk(path: list[str], table: dict) -> None:
+        values = [(k, v) for k, v in table.items() if isinstance(v, str)]
+        if values:
+            lines = [f"[{'.'.join(toml_key(part) for part in path)}]"]
+            lines += [f"{toml_key(version)} = {json.dumps(commit)}" for version, commit in values]
+            sections.append("\n".join(lines) + "\n")
+        for name, child in table.items():
+            if isinstance(child, dict):
+                walk([*path, name], child)
+
+    walk([], releases)
+    return "\n".join(sections)
+
+
 # Repository and registry access
 
 
@@ -345,29 +395,35 @@ def print_build_plan(args: argparse.Namespace) -> None:
     print(f"test={'true' if result.test else ''}")
 
 
-def publish_tag(package: str, version: str) -> str:
-    """Tag of the commit a version was published from (packages.yml)."""
-    return f"{package}-v{version}"
+def package_state(
+    key: str,
+    version: str,
+    directory: str,
+    version_file: str,
+    version_pattern: str,
+    releases: dict,
+    registry_published: Callable[[], bool],
+) -> Package:
+    """A recorded version is published from its recorded commit; the registry is asked about other versions,
+    which are compared with the commit that set them."""
+    recorded = recorded_commit(releases, key, version)
+    base = (
+        recorded
+        if recorded and commit_exists(recorded)
+        else git("log", "-1", "--format=%H", "-G", version_pattern, "--", version_file)
+    )
+    changed = changed_since(base, [directory, f":!{directory}/tests"])
+    return Package(version, changed, recorded is not None or registry_published())
 
 
-def published_tag(package: str, version: str) -> str | None:
-    """The publish tag of the version, when it exists: registries list new versions with a delay."""
-    tag = f"refs/tags/{publish_tag(package, version)}"
-    found = subprocess.run(["git", "rev-parse", "--verify", "--quiet", tag], cwd=ROOT, capture_output=True)
-    return tag if found.returncode == 0 else None
-
-
-def changed_since_publish(package: str, version: str, directory: str, version_file: str, version_pattern: str) -> bool:
-    """Shipped files changed since the version was published, or since it was set when it was not published yet."""
-    tag = published_tag(package, version)
-    base = tag or git("log", "-1", "--format=%H", "-G", version_pattern, "--", version_file)
-    diff = subprocess.run(["git", "diff", "--quiet", base, "HEAD", "--", directory, f":!{directory}/tests"], cwd=ROOT)
-    return diff.returncode != 0
-
-
-def http_json(url: str, headers: dict[str, str] | None = None) -> dict | None:
-    """The JSON at `url`, or None for 404."""
-    request = urllib.request.Request(url, headers={"User-Agent": "simplex-chat-release", **(headers or {})})
+def http_json(url: str, headers: dict[str, str] | None = None, method: str = "GET", body: dict | None = None) -> Any:
+    """The JSON response, or None for 404."""
+    request = urllib.request.Request(
+        url,
+        headers={"User-Agent": "simplex-chat-release", **(headers or {})},
+        method=method,
+        data=json.dumps(body).encode() if body is not None else None,
+    )
     try:
         with urllib.request.urlopen(request, timeout=60) as response:
             return json.load(response)
@@ -377,8 +433,9 @@ def http_json(url: str, headers: dict[str, str] | None = None) -> dict | None:
         raise
 
 
+# Workflows pass unset repository variables as empty strings.
 def npm_published(name: str, version: str) -> bool:
-    registry = os.environ.get("NPM_REGISTRY", "https://registry.npmjs.org")
+    registry = os.environ.get("NPM_REGISTRY") or "https://registry.npmjs.org"
     package = registry_name(name, os.environ.get("NPM_SCOPE")).replace("/", "%2F")
     return http_json(f"{registry}/{package}/{version}") is not None
 
@@ -388,24 +445,81 @@ def python_project() -> str:
 
 
 def pypi_published(version: str) -> bool:
-    index = os.environ.get("PYPI_URL", "https://pypi.org")
+    index = os.environ.get("PYPI_URL") or "https://pypi.org"
     project = os.environ.get("PYPI_PROJECT") or python_project()
     return http_json(f"{index}/pypi/{project}/{version}/json") is not None
 
 
-def github_release(tag: str) -> dict | None:
-    repo = os.environ.get("LIBS_REPOSITORY", "simplex-chat/simplex-chat-libs")
+def libs_api(path: str, method: str = "GET", body: dict | None = None) -> Any:
+    """GitHub API of the simplex-chat-libs repository."""
+    repo = os.environ.get("LIBS_REPOSITORY") or "simplex-chat/simplex-chat-libs"
     headers = {"Accept": "application/vnd.github+json"}
     if token := os.environ.get("GH_TOKEN"):
         headers["Authorization"] = f"Bearer {token}"
-    return http_json(f"https://api.github.com/repos/{repo}/releases/tags/{tag}", headers)
+    return http_json(f"https://api.github.com/repos/{repo}/{path}", headers, method, body)
 
 
-def npm_package(key: str, directory: str) -> Package:
+def github_release(tag: str) -> dict | None:
+    return libs_api(f"releases/tags/{tag}")
+
+
+def read_releases() -> tuple[dict, str | None]:
+    """The releases and the blob of the file, which a change must name."""
+    file = libs_api(f"contents/{RELEASES_FILE}")
+    if file is None:
+        return {}, None
+    return tomllib.loads(base64.b64decode(file["content"]).decode()), file["sha"]
+
+
+def record_releases(entries: list[tuple[str, str]], commit: str) -> str:
+    """Adds the entries in one commit of simplex-chat-libs and returns that commit."""
+    message = ", ".join(f"{key} {version}" for key, version in entries)
+    attempt = 1
+    while True:
+        releases, blob = read_releases()
+        updated = add_releases(releases, entries, commit)
+        if blob and updated == releases:
+            return libs_api(f"commits?path={RELEASES_FILE}&per_page=1")[0]["sha"]
+        body = {
+            "message": message,
+            "content": base64.b64encode(render_releases(updated).encode()).decode(),
+            "committer": {"name": "SimpleX Chat", "email": "build@simplex.chat"},
+            **({"sha": blob} if blob else {}),
+        }
+        try:
+            return libs_api(f"contents/{RELEASES_FILE}", "PUT", body)["commit"]["sha"]
+        except urllib.error.HTTPError as e:
+            # 409 and 422: another writer changed or created the file since it was read
+            if e.code not in (409, 422) or attempt == RECORD_ATTEMPTS:
+                raise
+            attempt += 1
+
+
+def commit_exists(commit: str) -> bool:
+    found = subprocess.run(["git", "cat-file", "-e", f"{commit}^{{commit}}"], cwd=ROOT, capture_output=True)
+    return found.returncode == 0
+
+
+def npm_key(directory: str) -> str:
+    return f"npm/{read_json(f'{directory}/package.json')['name']}"
+
+
+def pypi_key() -> str:
+    return f"pypi/{python_project()}"
+
+
+def npm_package(directory: str, releases: dict) -> Package:
     package = read_json(f"{directory}/package.json")
-    changed = changed_since_publish(key, package["version"], directory, f"{directory}/package.json", '"version":')
-    published = bool(published_tag(key, package["version"])) or npm_published(package["name"], package["version"])
-    return Package(package["version"], changed, published)
+    name, version = package["name"], package["version"]
+    return package_state(
+        npm_key(directory),
+        version,
+        directory,
+        f"{directory}/package.json",
+        '"version":',
+        releases,
+        lambda: npm_published(name, version),
+    )
 
 
 def prepare(args: argparse.Namespace) -> None:
@@ -413,7 +527,7 @@ def prepare(args: argparse.Namespace) -> None:
     libs_tag = f"core-{cabal_version}"
     addon = current_addon_tag()
     python_version, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE))
-    python_changed = changed_since_publish("python", python_version, PYTHON_DIR, PYTHON_VERSION_FILE, "^__version__ =")
+    releases, _ = read_releases()
     result = plan(
         State(
             libs_tag=libs_tag,
@@ -422,12 +536,16 @@ def prepare(args: argparse.Namespace) -> None:
             addon_digests=release_digests(github_release(addon), ADDON_ASSETS),
             nodejs_pins=parse_release(read_text(RELEASE_FILE)),
             python_pins=python_pins,
-            types=npm_package("types", TYPES_DIR),
-            nodejs=npm_package("nodejs", NODEJS_DIR),
-            python=Package(
+            types=npm_package(TYPES_DIR, releases),
+            nodejs=npm_package(NODEJS_DIR, releases),
+            python=package_state(
+                pypi_key(),
                 python_version,
-                python_changed,
-                bool(published_tag("python", python_version)) or pypi_published(python_version),
+                PYTHON_DIR,
+                PYTHON_VERSION_FILE,
+                "^__version__ =",
+                releases,
+                lambda: pypi_published(python_version),
             ),
             types_version=args.types_version,
             nodejs_version=args.nodejs_version,
@@ -450,14 +568,31 @@ def prepare(args: argparse.Namespace) -> None:
 
 
 def publish_plan() -> None:
+    """A recorded version is published: npm and PyPI list a new version minutes after it is published."""
+    releases, _ = read_releases()
     for key, directory in [("types", TYPES_DIR), ("nodejs", NODEJS_DIR)]:
         package = read_json(f"{directory}/package.json")
-        pending = not published_tag(key, package["version"]) and not npm_published(package["name"], package["version"])
-        print(f"{key}_version={package['version'] if pending else ''}")
+        published = recorded_commit(releases, npm_key(directory), package["version"]) or npm_published(
+            package["name"], package["version"]
+        )
+        print(f"{key}_version={'' if published else package['version']}")
         print(f"{key}_dist_tag={dist_tag(package['version'])}")
     version, _ = parse_python_version(read_text(PYTHON_VERSION_FILE))
-    pending = not published_tag("python", version) and not pypi_published(version)
-    print(f"python_version={version if pending else ''}")
+    published = recorded_commit(releases, pypi_key(), version) or pypi_published(version)
+    print(f"python_version={'' if published else version}")
+
+
+def record(args: argparse.Namespace) -> None:
+    versions = [
+        ("core", args.core),
+        (npm_key(TYPES_DIR), args.types),
+        (npm_key(NODEJS_DIR), args.nodejs),
+        (pypi_key(), args.python),
+    ]
+    entries = [(key, version) for key, version in versions if version]
+    if not entries:
+        sys.exit("nothing to record")
+    print(record_releases(entries, args.commit))
 
 
 def test_names() -> None:
@@ -509,6 +644,10 @@ def main(argv: list[str]) -> None:
         prepare_parser.add_argument(f"--{package}-version", help="this version instead of a bump when needed")
     prepare_parser.add_argument("--summary", help="file for the PR title and body")
     commands.add_parser("publish-plan")
+    record_parser = commands.add_parser("record")
+    record_parser.add_argument("--commit", required=True, help="the simplex-chat commit released or published from")
+    for name in ["core", "types", "nodejs", "python"]:
+        record_parser.add_argument(f"--{name}", default="", help=f"{name} version, recorded when not empty")
     commands.add_parser("test-names")
     args = parser.parse_args(argv)
     if args.command == "addon-tag":
@@ -521,6 +660,8 @@ def main(argv: list[str]) -> None:
         prepare(args)
     elif args.command == "publish-plan":
         publish_plan()
+    elif args.command == "record":
+        record(args)
     else:
         test_names()
 

scripts/ci/test_packages_release.py

@@ -1,4 +1,10 @@
+import base64
+import contextlib
+import email.message
+import io
 import subprocess
+import tomllib
+import urllib.error
 import tempfile
 import unittest
 from dataclasses import replace
@@ -8,6 +14,9 @@ from unittest import mock
 import packages_release
 from packages_release import (
     ADDON_ASSETS,
+    NODEJS_DIR,
+    PYTHON_VERSION_FILE,
+    TYPES_DIR,
     ADDON_PLATFORMS,
     BuildPlan,
     LIBS_ASSETS,
@@ -20,12 +29,18 @@ from packages_release import (
     bump_npm_version,
     bump_python_version,
     changed_since,
-    changed_since_publish,
-    published_tag,
+    add_releases,
+    package_state,
+    publish_plan,
+    record_releases,
+    recorded_commit,
+    render_releases,
     dist_tag,
+    npm_published,
     parse_python_version,
     parse_release,
     plan,
+    pypi_published,
     registry_name,
     release_digests,
     render_python_version,
@@ -220,7 +235,132 @@ class BuildPlanTest(unittest.TestCase):
         )
 
 
-class ChangedSincePublishTest(unittest.TestCase):
+class RegistryUrlTest(unittest.TestCase):
+    def urls(self, env: dict[str, str]) -> list[str]:
+        with (
+            mock.patch.dict("os.environ", env),
+            mock.patch.object(packages_release, "http_json", return_value={}) as get,
+        ):
+            npm_published("simplex-chat", "1.0.0")
+            pypi_published("1.0.0")
+            packages_release.github_release("core-1.0.0.1")
+        return [call.args[0] for call in get.call_args_list]
+
+    def test_empty_variables_use_the_public_registries(self):
+        self.assertEqual(
+            self.urls({"NPM_REGISTRY": "", "NPM_SCOPE": "", "PYPI_URL": "", "PYPI_PROJECT": "", "LIBS_REPOSITORY": ""}),
+            [
+                "https://registry.npmjs.org/simplex-chat/1.0.0",
+                "https://pypi.org/pypi/simplex-chat/1.0.0/json",
+                "https://api.github.com/repos/simplex-chat/simplex-chat-libs/releases/tags/core-1.0.0.1",
+            ],
+        )
+
+    def test_variables_select_test_registries(self):
+        self.assertEqual(
+            self.urls(
+                {
+                    "NPM_SCOPE": "@test",
+                    "PYPI_URL": "https://test.pypi.org",
+                    "PYPI_PROJECT": "sx-test",
+                    "LIBS_REPOSITORY": "a/b-libs",
+                }
+            ),
+            [
+                "https://registry.npmjs.org/@test%2Fsimplex-chat/1.0.0",
+                "https://test.pypi.org/pypi/sx-test/1.0.0/json",
+                "https://api.github.com/repos/a/b-libs/releases/tags/core-1.0.0.1",
+            ],
+        )
+
+
+class ReleasesFileTest(unittest.TestCase):
+    RELEASES = {
+        "core": {"7.1.0.11": "c1"},
+        "npm": {"simplex-chat": {"7.1.0-beta.6.3": "n1"}, "@simplex-chat/types": {"0.12.2": "t1"}},
+        "pypi": {"simplex-chat": {"7.1.0b6": "p1"}},
+    }
+
+    def test_renders_toml_that_reads_back(self):
+        text = render_releases(self.RELEASES)
+        self.assertEqual(tomllib.loads(text), self.RELEASES)
+        self.assertIn('[core]\n"7.1.0.11" = "c1"\n', text)
+        self.assertIn('[npm."@simplex-chat/types"]\n', text)
+        self.assertIn("[npm.simplex-chat]\n", text)
+
+    def test_keeps_the_order_of_recording(self):
+        releases = {"npm": {"simplex-chat": {"7.1.0-beta.6.9": "a"}}}
+        updated = add_releases(releases, [("npm/simplex-chat", "7.1.0-beta.6.10"), ("core", "7.1.0.15")], "b")
+        self.assertEqual(
+            render_releases(updated),
+            '[npm.simplex-chat]\n"7.1.0-beta.6.9" = "a"\n"7.1.0-beta.6.10" = "b"\n\n[core]\n"7.1.0.15" = "b"\n',
+        )
+
+    def test_finds_the_recorded_commit(self):
+        self.assertEqual(recorded_commit(self.RELEASES, "npm/@simplex-chat/types", "0.12.2"), "t1")
+        self.assertEqual(recorded_commit(self.RELEASES, "core", "7.1.0.11"), "c1")
+        self.assertIsNone(recorded_commit(self.RELEASES, "npm/simplex-chat", "7.1.0-beta.6.4"))
+        self.assertIsNone(recorded_commit({}, "pypi/simplex-chat", "7.1.0b6"))
+
+    def test_adds_entries_and_keeps_recorded_commits(self):
+        updated = add_releases(self.RELEASES, [("npm/simplex-chat", "7.1.0-beta.6.4"), ("core", "7.1.0.11")], "x")
+        self.assertEqual(updated["npm"]["simplex-chat"], {"7.1.0-beta.6.3": "n1", "7.1.0-beta.6.4": "x"})
+        self.assertEqual(updated["core"], {"7.1.0.11": "c1"})
+        self.assertEqual(self.RELEASES["npm"]["simplex-chat"], {"7.1.0-beta.6.3": "n1"})
+
+
+class RecordReleasesTest(unittest.TestCase):
+    def file(self, releases, blob):
+        return {"content": base64.b64encode(render_releases(releases).encode()).decode(), "sha": blob}
+
+    def conflict(self):
+        error = urllib.error.HTTPError("url", 409, "Conflict", email.message.Message(), io.BytesIO())
+        self.addCleanup(error.close)
+        return error
+
+    def test_creates_the_file_and_returns_the_commit(self):
+        calls = []
+
+        def api(path, method="GET", body=None):
+            calls.append((path, method, body))
+            return None if method == "GET" else {"commit": {"sha": "libs1"}}
+
+        with mock.patch.object(packages_release, "libs_api", api):
+            self.assertEqual(record_releases([("core", "7.1.0.12")], "sx1"), "libs1")
+        _, method, body = calls[1]
+        self.assertEqual((method, body["message"], "sha" in body), ("PUT", "core 7.1.0.12", False))
+        self.assertEqual(tomllib.loads(base64.b64decode(body["content"]).decode()), {"core": {"7.1.0.12": "sx1"}})
+
+    def test_retries_when_another_writer_changed_the_file(self):
+        reads = iter([self.file({"core": {"1": "a"}}, "blob1"), self.file({"core": {"1": "a", "2": "b"}}, "blob2")])
+        puts = []
+
+        def api(path, method="GET", body=None):
+            if method == "GET":
+                return next(reads)
+            puts.append(body)
+            if len(puts) == 1:
+                raise self.conflict()
+            return {"commit": {"sha": "libs2"}}
+
+        with mock.patch.object(packages_release, "libs_api", api):
+            self.assertEqual(record_releases([("npm/simplex-chat", "3")], "sx3"), "libs2")
+        self.assertEqual([body["sha"] for body in puts], ["blob1", "blob2"])
+        self.assertEqual(
+            tomllib.loads(base64.b64decode(puts[1]["content"]).decode()),
+            {"core": {"1": "a", "2": "b"}, "npm": {"simplex-chat": {"3": "sx3"}}},
+        )
+
+    def test_recorded_entries_return_the_last_commit_of_the_file(self):
+        def api(path, method="GET", body=None):
+            self.assertEqual(method, "GET")
+            return [{"sha": "libs0"}] if path.startswith("commits") else self.file({"core": {"1": "a"}}, "blob1")
+
+        with mock.patch.object(packages_release, "libs_api", api):
+            self.assertEqual(record_releases([("core", "1")], "other"), "libs0")
+
+
+class PackageStateTest(unittest.TestCase):
     def setUp(self):
         self.repo = Path(tempfile.mkdtemp())
         self.git("init", "-q")
@@ -239,9 +379,66 @@ class ChangedSincePublishTest(unittest.TestCase):
         self.git("add", "-A")
         self.git("commit", "-q", "-m", message)
 
-    def changed(self):
+    def state(self, releases=None, registry_published=False):
         with mock.patch.object(packages_release, "ROOT", self.repo):
-            return changed_since_publish("pkg", "1", "pkg", "pkg/version.txt", "^version =")
+            return package_state(
+                "npm/pkg", "1", "pkg", "pkg/version.txt", "^version =", releases or {}, lambda: registry_published
+            )
+
+    def changed(self):
+        return self.state().changed
+
+    def head(self):
+        return subprocess.run(
+            ["git", "rev-parse", "HEAD"], cwd=self.repo, capture_output=True, text=True
+        ).stdout.strip()
+
+    def test_recorded_version_is_published_and_compared_with_its_commit(self):
+        self.write("pkg/src.txt", "b")
+        self.commit("change published with version 1")
+        releases = {"npm": {"pkg": {"1": self.head()}}}
+        self.assertEqual(self.state(releases), Package("1", changed=False, published=True))
+        self.write("pkg/src.txt", "c")
+        self.commit("change after the publish")
+        self.assertEqual(self.state(releases), Package("1", changed=True, published=True))
+
+    def test_unrecorded_version_asks_the_registry(self):
+        self.assertFalse(self.state(registry_published=False).published)
+        self.assertTrue(self.state(registry_published=True).published)
+
+    def test_publish_plan_skips_recorded_versions_before_the_registries_list_them(self):
+        self.write(f"{TYPES_DIR}/package.json", '{"name": "@simplex-chat/types", "version": "0.1.0"}')
+        self.write(f"{NODEJS_DIR}/package.json", '{"name": "simplex-chat", "version": "1.0.0-beta.1.1"}')
+        self.write(PYTHON_VERSION_FILE, render_python_version("1.0.0b1.post1", Pins("core-1", {})))
+        self.write("pyproject.toml", "")
+        self.commit("packages")
+        releases = {"npm": {"simplex-chat": {"1.0.0-beta.1.1": "c"}}, "pypi": {"simplex-chat": {"1.0.0b1.post1": "c"}}}
+        output = io.StringIO()
+        with (
+            mock.patch.object(packages_release, "ROOT", self.repo),
+            mock.patch.object(packages_release, "read_releases", return_value=(releases, "blob")),
+            mock.patch.object(packages_release, "python_project", return_value="simplex-chat"),
+            mock.patch.object(packages_release, "npm_published", return_value=False),
+            mock.patch.object(packages_release, "pypi_published", return_value=False),
+            contextlib.redirect_stdout(output),
+        ):
+            publish_plan()
+        self.assertEqual(
+            output.getvalue().splitlines(),
+            [
+                "types_version=0.1.0",
+                "types_dist_tag=latest",
+                "nodejs_version=",
+                "nodejs_dist_tag=beta",
+                "python_version=",
+            ],
+        )
+
+    def test_unknown_recorded_commit_falls_back_to_the_version_commit(self):
+        self.write("pkg/src.txt", "b")
+        self.commit("change")
+        state = self.state({"npm": {"pkg": {"1": "0" * 40}}})
+        self.assertEqual(state, Package("1", changed=True, published=True))
 
     def test_compares_with_the_commit_that_set_the_version(self):
         self.assertFalse(self.changed())
@@ -249,21 +446,12 @@ class ChangedSincePublishTest(unittest.TestCase):
         self.commit("change")
         self.assertTrue(self.changed())
 
-    def test_compares_with_the_published_commit(self):
+    def test_new_version_resets_the_comparison(self):
         self.write("pkg/src.txt", "b")
         self.commit("change")
-        self.git("tag", "pkg-v1")
+        self.write("pkg/version.txt", 'version = "2"\n')
+        self.commit("set version 2")
         self.assertFalse(self.changed())
-        self.write("pkg/src.txt", "c")
-        self.commit("change after publish")
-        self.assertTrue(self.changed())
-
-    def test_publish_tag_marks_the_version_published(self):
-        with mock.patch.object(packages_release, "ROOT", self.repo):
-            self.assertIsNone(published_tag("pkg", "1"))
-            self.git("tag", "pkg-v1")
-            self.assertEqual(published_tag("pkg", "1"), "refs/tags/pkg-v1")
-            self.assertIsNone(published_tag("pkg", "2"))
 
     def test_changed_since_looks_only_at_the_sources(self):
         base = subprocess.run(