From a766d0af0f4048ecec96ab367461f80e38f7586f Mon Sep 17 00:00:00 2001 From: Evgeny Date: Mon, 19 Oct 2026 07:50:09 +0100 Subject: [PATCH] ci: package releases with drafts in simplex-chat-libs (#7754) * ci: package releases with drafts in simplex-chat-libs * test release * core: 7.1.0.12 * packages: release nodejs and python libraries * fixes --------- Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com> --- .github/workflows/build.yml | 438 +++++++++++--- .github/workflows/packages.yml | 306 +--------- docs/contributing/RELEASES.md | 153 +++++ .../types/typescript/README.md | 2 + .../types/typescript/package.json | 3 +- packages/simplex-chat-nodejs/package.json | 4 +- packages/simplex-chat-nodejs/src/release.ts | 14 +- packages/simplex-chat-python/README.md | 9 - .../src/simplex_chat/_version.py | 18 +- plans/2026-10-18-package-releases.md | 364 ++++++++++++ scripts/ci/package-tests.sh | 81 +++ scripts/ci/packages_release.py | 548 ++++++++++++++---- scripts/ci/test_packages_release.py | 414 +++++++++++-- simplex-chat.cabal | 2 +- 14 files changed, 1797 insertions(+), 559 deletions(-) create mode 100644 docs/contributing/RELEASES.md create mode 100644 plans/2026-10-18-package-releases.md create mode 100755 scripts/ci/package-tests.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index be1784a237..074a6feb48 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,6 +33,7 @@ env: release: ${{ startsWith(github.ref, 'refs/tags/v') }} branch: ${{ github.base_ref || github.ref_name }} cache_o0: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/stable') }} + release_branch: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/stable') && github.ref_name || '' }} ZSTD_CLEVEL: 9 # This workflow uses custom actions (prepare-build and prepare-release) defined in: @@ -74,7 +75,8 @@ jobs: # ============================= # Create release, but only if it's triggered by tag push. -# Create simplex-chat-libs release core-, but only if the head commit of a push to master or stable changes the version in simplex-chat.cabal. +# On pushes to master or stable: claim the simplex-chat-libs release core- for the branch +# and create its draft, create the draft of a missing add-on release, and find the package versions to publish. # On pull requests and other pushes, this job will always complete. # Secrets: @@ -87,8 +89,15 @@ jobs: maybe-release: runs-on: ubuntu-latest outputs: - libs_tag: ${{ steps.libs_commit.outputs.tag }} + libs_tag: ${{ steps.libs.outputs.tag }} core_changed: ${{ steps.core.outputs.core }} + addon_tag: ${{ steps.addon.outputs.tag }} + addon_draft: ${{ steps.addon.outputs.draft }} + libs_latest: ${{ steps.latest_libs.outputs.tag }} + publish_types: ${{ steps.libs.outputs.tag && steps.publish.outputs.types_version || '' }} + publish_nodejs: ${{ steps.publish.outputs.nodejs_version }} + publish_python: ${{ steps.publish.outputs.python_version }} + publish: ${{ steps.publish.outputs.nodejs_version || steps.publish.outputs.python_version }} steps: - name: Clone project if: github.event_name == 'push' || github.event_name == 'pull_request' @@ -129,38 +138,68 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Check version change - id: check_version - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/stable' + - name: Library release + id: libs + if: env.release_branch shell: bash - run: | - version=$(sed -n 's/^version: *//p' simplex-chat.cabal) - prev_version=$(git show HEAD^1:simplex-chat.cabal | sed -n 's/^version: *//p') - if [ "$version" != "$prev_version" ]; then - echo "version=$version" >> $GITHUB_OUTPUT - fi - - - name: Record libs release - id: libs_commit - if: steps.check_version.outputs.version env: GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} LIBS_REPOSITORY: ${{ github.repository }}-libs - VERSION: ${{ steps.check_version.outputs.version }} run: | - commit=$(python3 scripts/ci/packages_release.py record --core "$VERSION" --commit "$GITHUB_SHA") - echo "tag=core-$VERSION" >> "$GITHUB_OUTPUT" - echo "commit=$commit" >> "$GITHUB_OUTPUT" + version=$(sed -n 's/^version: *//p' simplex-chat.cabal) + tag=core-$version + claim_output=$(python3 scripts/ci/packages_release.py claim-core "$version" "$GITHUB_SHA" "$release_branch") + eval "$claim_output" + if [ "$claim" = skip ]; then + echo "::notice title=Library release::core $version is recorded for another commit" + exit 0 + fi + status_output=$(python3 scripts/ci/packages_release.py release-status "$tag") + eval "$status_output" + if [ "$status" = absent ]; then + gh release create "$tag" --repo "$LIBS_REPOSITORY" --draft --title "$tag" --notes "" ${commit:+--target "$commit"} + fi + if [ "$status" != published ]; then + echo "tag=$tag" >> "$GITHUB_OUTPUT" + fi - # creates the tag on the commit that recorded the release - - name: Create libs release - if: steps.check_version.outputs.version - uses: softprops/action-gh-release@v2 - with: - repository: ${{ github.repository }}-libs - tag_name: ${{ steps.libs_commit.outputs.tag }} - target_commitish: ${{ steps.libs_commit.outputs.commit }} - token: ${{ secrets.NODEJS_REPO_TOKEN }} + - name: Add-on release + id: addon + if: env.release_branch + shell: bash + env: + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + LIBS_REPOSITORY: ${{ github.repository }}-libs + run: | + addon_output=$(python3 scripts/ci/packages_release.py addon-release) + eval "$addon_output" + if [ "$status" = absent ]; then + tag=nodejs-addon-${GITHUB_SHA::7} + gh release create "$tag" --repo "$LIBS_REPOSITORY" --draft --title "$tag" --notes "$notes" + fi + if [ "$status" != published ]; then + echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "draft=true" >> "$GITHUB_OUTPUT" + fi + + - name: Latest library + id: latest_libs + if: steps.addon.outputs.draft + env: + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + LIBS_REPOSITORY: ${{ github.repository }}-libs + run: python3 scripts/ci/packages_release.py latest-libs --branch "$release_branch" >> "$GITHUB_OUTPUT" + + - name: Versions to publish + id: publish + if: env.release_branch + env: + GH_TOKEN: ${{ github.token }} + LIBS_REPOSITORY: ${{ github.repository }}-libs + NPM_SCOPE: ${{ vars.NPM_SCOPE }} + PYPI_PROJECT: ${{ vars.PYPI_PROJECT }} + PYPI_URL: ${{ vars.PYPI_URL }} + run: python3 scripts/ci/packages_release.py publish-plan >> "$GITHUB_OUTPUT" # ========================= # Linux Build @@ -444,12 +483,11 @@ jobs: - name: Upload library if: needs.maybe-release.outputs.libs_tag && matrix.libs - uses: softprops/action-gh-release@v2 - with: - repository: ${{ github.repository }}-libs - tag_name: ${{ needs.maybe-release.outputs.libs_tag }} - files: release-libs/*.zip - token: ${{ secrets.NODEJS_REPO_TOKEN }} + shell: bash + env: + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + TAG: ${{ needs.maybe-release.outputs.libs_tag }} + run: gh release upload "$TAG" release-libs/*.zip --clobber --repo "${{ github.repository }}-libs" - name: Build CLI deb if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true @@ -591,7 +629,7 @@ jobs: - name: Setup Node.js id: setup-node - if: ${{ !cancelled() && needs.maybe-release.outputs.core_changed && steps.build-lib.outcome == 'success' }} + if: ${{ !cancelled() && ((needs.maybe-release.outputs.core_changed && steps.build-lib.outcome == 'success') || (matrix.libs && (needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish))) }} uses: actions/setup-node@v7 with: node-version: 22 @@ -603,35 +641,42 @@ jobs: with: python-version: "3.12" - - name: Build simplex-chat (npm) and add-on - id: build-nodejs - if: ${{ !cancelled() && steps.setup-python.outcome == 'success' }} - shell: bash - working-directory: packages/simplex-chat-nodejs - run: | - (cd ../simplex-chat-client/types/typescript && npm install --no-audit --no-fund && npx tsc) - npm install --ignore-scripts --no-save --no-audit --no-fund ../simplex-chat-client/types/typescript - npx --yes node-gyp rebuild - npx tsc && cp src/simplex.* dist - - - name: Test simplex-chat (npm) - id: test-nodejs - if: ${{ !cancelled() && steps.build-nodejs.outcome == 'success' }} + - name: Package tests (pull request) + if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.core_changed }} timeout-minutes: 30 shell: bash - working-directory: packages/simplex-chat-nodejs env: SIMPLEX_LIBS_DIR: ${{ github.workspace }}/release-libs/simplex-chat-libs-linux-${{ matrix.arch }}/libs - run: npx jest --forceExit --testNamePattern '^(?!.*use preset servers)' + run: | + scripts/ci/package-tests.sh build linux-${{ matrix.arch }} + scripts/ci/package-tests.sh test - - name: Test simplex-chat (PyPI) - if: ${{ !cancelled() && steps.test-nodejs.outcome == 'success' }} + - name: Add-on + if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.addon_draft }} + timeout-minutes: 30 + shell: bash + env: + LIBS_RELEASE: ${{ needs.maybe-release.outputs.libs_latest }} + ADDON_TAG: ${{ needs.maybe-release.outputs.addon_tag }} + LIBS_REPOSITORY: ${{ github.repository }}-libs + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + run: scripts/ci/package-tests.sh addon linux-${{ matrix.arch }} + + - name: Package tests + id: package-tests + if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.publish }} timeout-minutes: 30 shell: bash - working-directory: packages/simplex-chat-python run: | - python -m pip install -e ".[test]" - python -m pytest -q + scripts/ci/package-tests.sh publish linux-${{ matrix.arch }} ${{ needs.maybe-release.outputs.publish_nodejs && 'nodejs' || '' }} ${{ needs.maybe-release.outputs.publish_python && 'python' || '' }} + echo "linux-${{ matrix.arch }}" > "$RUNNER_TEMP/package-tests" + + - name: Package tests result + if: ${{ !cancelled() && steps.package-tests.outcome == 'success' }} + uses: actions/upload-artifact@v7 + with: + name: package-tests-linux-${{ matrix.arch }} + path: ${{ runner.temp }}/package-tests - name: Remove -O2 build directory id: remove-o2 @@ -821,12 +866,11 @@ jobs: (cd postgres-libs && zip -r simplex-chat-libs-linux-x86_64-postgres.zip libs) - name: Upload postgres library - uses: softprops/action-gh-release@v2 - with: - repository: ${{ github.repository }}-libs - tag_name: ${{ needs.maybe-release.outputs.libs_tag }} - files: postgres-libs/*.zip - token: ${{ secrets.NODEJS_REPO_TOKEN }} + shell: bash + env: + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + TAG: ${{ needs.maybe-release.outputs.libs_tag }} + run: gh release upload "$TAG" postgres-libs/*.zip --clobber --repo "${{ github.repository }}-libs" # ========================= # MacOS Build @@ -1084,12 +1128,52 @@ jobs: - name: Upload library if: ${{ !cancelled() && steps.mac_lib_build.outcome == 'success' }} - uses: softprops/action-gh-release@v2 + shell: bash + env: + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + TAG: ${{ needs.maybe-release.outputs.libs_tag }} + run: gh release upload "$TAG" release-libs/*.zip --clobber --repo "${{ github.repository }}-libs" + + - name: Setup Node.js + id: setup-node + if: ${{ !cancelled() && (needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }} + uses: actions/setup-node@v7 with: - repository: ${{ github.repository }}-libs - tag_name: ${{ needs.maybe-release.outputs.libs_tag }} - files: release-libs/*.zip - token: ${{ secrets.NODEJS_REPO_TOKEN }} + node-version: 22 + + - name: Setup Python + id: setup-python + if: ${{ !cancelled() && steps.setup-node.outcome == 'success' }} + uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Add-on + if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.addon_draft }} + timeout-minutes: 30 + shell: bash + env: + LIBS_RELEASE: ${{ needs.maybe-release.outputs.libs_latest }} + ADDON_TAG: ${{ needs.maybe-release.outputs.addon_tag }} + LIBS_REPOSITORY: ${{ github.repository }}-libs + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + run: scripts/ci/package-tests.sh addon macos-${{ matrix.arch }} + + - name: Package tests + id: package-tests + if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.publish }} + timeout-minutes: 30 + shell: bash + run: | + scripts/ci/package-tests.sh publish macos-${{ matrix.arch }} ${{ needs.maybe-release.outputs.publish_nodejs && 'nodejs' || '' }} ${{ needs.maybe-release.outputs.publish_python && 'python' || '' }} + echo "macos-${{ matrix.arch }}" > "$RUNNER_TEMP/package-tests" + + - name: Package tests result + if: ${{ !cancelled() && steps.package-tests.outcome == 'success' }} + uses: actions/upload-artifact@v7 + with: + name: package-tests-macos-${{ matrix.arch }} + path: ${{ runner.temp }}/package-tests - name: Remove -O2 build directory id: remove-o2 @@ -1305,12 +1389,52 @@ jobs: - name: Upload library if: needs.maybe-release.outputs.libs_tag - uses: softprops/action-gh-release@v2 + shell: bash + env: + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + TAG: ${{ needs.maybe-release.outputs.libs_tag }} + run: gh release upload "$TAG" release-libs/*.zip --clobber --repo "${{ github.repository }}-libs" + + - name: Setup Node.js + id: setup-node + if: ${{ !cancelled() && (needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }} + uses: actions/setup-node@v7 with: - repository: ${{ github.repository }}-libs - tag_name: ${{ needs.maybe-release.outputs.libs_tag }} - files: release-libs/*.zip - token: ${{ secrets.NODEJS_REPO_TOKEN }} + node-version: 22 + + - name: Setup Python + id: setup-python + if: ${{ !cancelled() && steps.setup-node.outcome == 'success' }} + uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Add-on + if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.addon_draft }} + timeout-minutes: 30 + shell: bash + env: + LIBS_RELEASE: ${{ needs.maybe-release.outputs.libs_latest }} + ADDON_TAG: ${{ needs.maybe-release.outputs.addon_tag }} + LIBS_REPOSITORY: ${{ github.repository }}-libs + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + run: scripts/ci/package-tests.sh addon windows-x86_64 + + - name: Package tests + id: package-tests + if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.publish }} + timeout-minutes: 30 + shell: bash + run: | + scripts/ci/package-tests.sh publish windows-x86_64 ${{ needs.maybe-release.outputs.publish_nodejs && 'nodejs' || '' }} ${{ needs.maybe-release.outputs.publish_python && 'python' || '' }} + echo "windows-x86_64" > "$RUNNER_TEMP/package-tests" + + - name: Package tests result + if: ${{ !cancelled() && steps.package-tests.outcome == 'success' }} + uses: actions/upload-artifact@v7 + with: + name: package-tests-windows-x86_64 + path: ${{ runner.temp }}/package-tests - name: Upload CLI if: startsWith(github.ref, 'refs/tags/v') @@ -1387,19 +1511,169 @@ jobs: key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }} # ========================= -# Packages release +# Releases and packages # ========================= -# A new library release is tested with the Node.js and Python packages and updates them in a release PR. - - packages: + releases: needs: [maybe-release, build-linux, build-linux-push, build-linux-postgres, build-macos, build-windows] - if: ${{ !cancelled() && needs.maybe-release.outputs.libs_tag }} - # the called workflow can't be granted more than its caller, and its publish job needs these + if: ${{ !cancelled() && (needs.maybe-release.outputs.libs_tag || needs.maybe-release.outputs.addon_draft) }} + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + LIBS_REPOSITORY: ${{ github.repository }}-libs + steps: + - name: Checkout Code + uses: actions/checkout@v6 + + - name: Publish library release + if: needs.maybe-release.outputs.libs_tag + run: python3 scripts/ci/packages_release.py publish-release "${{ needs.maybe-release.outputs.libs_tag }}" + + - name: Publish add-on release + if: ${{ !cancelled() && needs.maybe-release.outputs.addon_draft }} + run: python3 scripts/ci/packages_release.py publish-release "${{ needs.maybe-release.outputs.addon_tag }}" + +# Publishing uses trusted publishing (OIDC) for this workflow on npm and PyPI, +# or the NPM_TOKEN and PYPI_API_TOKEN secrets when they are set. +# Test repositories: variables NPM_SCOPE, PYPI_PROJECT and PYPI_URL / PYPI_UPLOAD_URL (TestPyPI) +# publish under other names. + + publish: + needs: [maybe-release, build-linux, build-linux-push, build-macos, build-windows, releases] + if: ${{ !cancelled() && (needs.maybe-release.outputs.publish || needs.maybe-release.outputs.publish_types) }} + runs-on: ubuntu-latest permissions: contents: read id-token: write - uses: ./.github/workflows/packages.yml - with: - libs_release: ${{ needs.maybe-release.outputs.libs_tag }} - secrets: inherit + # set by a publish step after it succeeds, so a failed publish still reports the earlier ones + outputs: + types: ${{ steps.types.outputs.version }} + nodejs: ${{ steps.nodejs.outputs.version }} + python: ${{ steps.python.outputs.version }} + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + NPM_SCOPE: ${{ vars.NPM_SCOPE }} + PYPI_PROJECT: ${{ vars.PYPI_PROJECT }} + PYPI_URL: ${{ vars.PYPI_URL }} + steps: + - name: Checkout Code + uses: actions/checkout@v6 + + - name: Download package tests results + if: needs.maybe-release.outputs.publish + uses: actions/download-artifact@v8 + with: + pattern: package-tests-* + path: ${{ runner.temp }}/package-tests + + - name: Check package tests and pins + if: needs.maybe-release.outputs.publish + env: + GH_TOKEN: ${{ github.token }} + LIBS_REPOSITORY: ${{ github.repository }}-libs + run: | + passed=$(find "$RUNNER_TEMP/package-tests" -mindepth 1 -maxdepth 1 -type d -name 'package-tests-*' | wc -l) + if [ "$passed" -ne 5 ]; then + echo "::error::package tests passed on $passed of 5 platforms" + exit 1 + fi + python3 scripts/ci/packages_release.py check-pins + + - name: Versions to publish + id: plan + env: + GH_TOKEN: ${{ github.token }} + LIBS_REPOSITORY: ${{ github.repository }}-libs + run: python3 scripts/ci/packages_release.py publish-plan >> "$GITHUB_OUTPUT" + + - name: Setup Node.js + if: steps.plan.outputs.types_version || steps.plan.outputs.nodejs_version + uses: actions/setup-node@v7 + with: + node-version: 24 + registry-url: https://registry.npmjs.org + + - name: Setup Python + if: steps.plan.outputs.python_version + uses: actions/setup-python@v7 + with: + python-version: "3.12" + + # Before the test names: the build installs @simplex-chat/types from this repository. + - name: Build and test simplex-chat (npm) + if: steps.plan.outputs.nodejs_version + working-directory: packages/simplex-chat-nodejs + run: | + (cd ../simplex-chat-client/types/typescript && npm install --no-audit --no-fund && npx tsc) + npm install --ignore-scripts --no-save --no-audit --no-fund ../simplex-chat-client/types/typescript + npx tsc && cp src/simplex.* dist + npx jest tests/libs.test.ts tests/api.unit.test.ts + + - name: Use test package names + if: vars.NPM_SCOPE || vars.PYPI_PROJECT + run: python3 scripts/ci/packages_release.py test-names + + - name: Publish @simplex-chat/types + id: types + if: steps.plan.outputs.types_version && needs.maybe-release.outputs.publish_types && needs.releases.result == 'success' + working-directory: packages/simplex-chat-client/types/typescript + env: + VERSION: ${{ steps.plan.outputs.types_version }} + DIST_TAG: ${{ steps.plan.outputs.types_dist_tag }} + run: | + npm install --no-audit --no-fund + npm publish --provenance --access public --tag "$DIST_TAG" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Publish simplex-chat (npm) + id: nodejs + if: steps.plan.outputs.nodejs_version + working-directory: packages/simplex-chat-nodejs + env: + VERSION: ${{ steps.plan.outputs.nodejs_version }} + DIST_TAG: ${{ steps.plan.outputs.nodejs_dist_tag }} + run: | + npm publish --provenance --access public --tag "$DIST_TAG" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Build and test simplex-chat (PyPI) + if: steps.plan.outputs.python_version + working-directory: packages/simplex-chat-python + run: | + python -m pip install build -e ".[test]" + python -m pytest -q + python -m build + + - name: Publish simplex-chat (PyPI) + if: steps.plan.outputs.python_version + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: packages/simplex-chat-python/dist + repository-url: ${{ vars.PYPI_UPLOAD_URL || 'https://upload.pypi.org/legacy/' }} + password: ${{ secrets.PYPI_API_TOKEN }} + + - name: Report the published PyPI version + id: python + if: steps.plan.outputs.python_version + env: + VERSION: ${{ steps.plan.outputs.python_version }} + run: echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + record: + needs: publish + if: ${{ !cancelled() && (needs.publish.outputs.types || needs.publish.outputs.nodejs || needs.publish.outputs.python) }} + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v6 + + - name: Record published versions + env: + GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} + LIBS_REPOSITORY: ${{ github.repository }}-libs + TYPES: ${{ needs.publish.outputs.types }} + NODEJS: ${{ needs.publish.outputs.nodejs }} + PYTHON: ${{ needs.publish.outputs.python }} + run: | + python3 scripts/ci/packages_release.py record --commit "$GITHUB_SHA" \ + --types "$TYPES" --nodejs "$NODEJS" --python "$PYTHON" diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 2e58fb4d34..e5ed17c4f8 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -1,60 +1,6 @@ name: packages -# Releases simplex-chat (npm, PyPI) and @simplex-chat/types (npm) from master and stable: -# 1. A missing Node.js add-on release (nodejs-addon-) is built and uploaded to simplex-chat-libs. -# 2. Versions that are not published yet are published, and recorded in releases.toml of simplex-chat-libs. -# npm prerelease versions get dist-tag beta, others latest. -# 3. The release PR packages-release/ pins new library and add-on releases with their SHA-256 -# and bumps the versions of changed packages. Merging it publishes them (step 2). -# Package tests run on every platform with the pinned libsimplex. build.yml runs them with the libsimplex it builds: -# for pull requests that change the chat core (linux-x86_64), and for a new library release before its release PR. - -# Publishing uses trusted publishing (OIDC) for this workflow on npm and PyPI, -# or the NPM_TOKEN and PYPI_API_TOKEN secrets when they are set. -# Test repositories: variables NPM_SCOPE, PYPI_PROJECT and PYPI_URL / PYPI_UPLOAD_URL (TestPyPI) -# publish under other names. - -# Secrets: -# ------- -# NODEJS_REPO_TOKEN -# Only select repositories: simplex-chat-libs -# Permissions: -# * Contents (Read and Write) -# RELEASE_PR_TOKEN -# Only select repositories: simplex-chat -# Permissions: -# * Contents (Read and Write) -# * Pull requests (Read and Write) -# A PR opened with GITHUB_TOKEN does not run the checks of other workflows. - on: - workflow_dispatch: - inputs: - nodejs_version: - description: "simplex-chat npm version, bumped when needed by default" - required: false - types_version: - description: "@simplex-chat/types version, bumped when needed by default" - required: false - python_version: - description: "simplex-chat PyPI version, bumped when needed by default" - required: false - # build.yml: a new library release - workflow_call: - inputs: - libs_release: - description: "test with this simplex-chat-libs release" - type: string - default: "" - push: - branches: - - master - - stable - paths: - - "packages/simplex-chat-nodejs/**" - - "packages/simplex-chat-client/types/typescript/**" - - "packages/simplex-chat-python/**" - - ".github/workflows/packages.yml" pull_request: paths: - "packages/simplex-chat-nodejs/**" @@ -79,47 +25,19 @@ jobs: addon: runs-on: ubuntu-latest outputs: - tag: ${{ steps.tag.outputs.tag }} - release_branch: ${{ steps.branch.outputs.release_branch }} - missing: ${{ steps.release.outputs.missing }} platforms: ${{ steps.plan.outputs.platforms }} test: ${{ steps.plan.outputs.test }} steps: - name: Checkout Code uses: actions/checkout@v6 - - name: Add-on release tag - id: tag - run: echo "tag=$(python3 scripts/ci/packages_release.py addon-tag)" >> "$GITHUB_OUTPUT" - - - name: Release branch - id: branch - if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/stable') - run: echo "release_branch=true" >> "$GITHUB_OUTPUT" - - - name: Check the add-on release - id: release - if: steps.branch.outputs.release_branch - env: - GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} - LIBS_REPOSITORY: ${{ github.repository }}-libs - TAG: ${{ steps.tag.outputs.tag }} - run: | - if ! gh release view "$TAG" --repo "$LIBS_REPOSITORY" > /dev/null 2>&1; then - echo "missing=true" >> "$GITHUB_OUTPUT" - fi - - name: Build plan id: plan env: - EVENT: ${{ github.event_name }} BASE: ${{ github.event.pull_request.base.sha }} - RELEASE_MISSING: ${{ steps.release.outputs.missing }} - LIBS_RELEASE: ${{ inputs.libs_release }} run: | - if [ -n "$BASE" ]; then git fetch -q --depth=1 origin "$BASE"; fi - python3 scripts/ci/packages_release.py build-plan --event "$EVENT" --base "$BASE" \ - --release-missing "$RELEASE_MISSING" --libs-release "$LIBS_RELEASE" >> "$GITHUB_OUTPUT" + git fetch -q --depth=1 origin "$BASE" + python3 scripts/ci/packages_release.py build-plan --base "$BASE" >> "$GITHUB_OUTPUT" build: name: "build ${{ matrix.platform }}" @@ -134,7 +52,6 @@ jobs: defaults: run: shell: bash - working-directory: packages/simplex-chat-nodejs steps: - name: Checkout Code uses: actions/checkout@v6 @@ -149,222 +66,9 @@ jobs: with: python-version: "3.12" - # The local types: the dependency can be bumped to a version that is not published yet. - name: Build simplex-chat (npm) and add-on - run: | - (cd ../simplex-chat-client/types/typescript && npm install --no-audit --no-fund && npx tsc) - npm install --ignore-scripts --no-save --no-audit --no-fund ../simplex-chat-client/types/typescript - npx --yes node-gyp rebuild - npx tsc && cp src/simplex.* dist - cp build/Release/simplex.node build/simplex-chat-nodejs-${{ matrix.platform }}.node + run: scripts/ci/package-tests.sh build ${{ matrix.platform }} - - name: Download libsimplex release - if: needs.addon.outputs.test && inputs.libs_release - env: - URL: https://github.com/${{ github.repository }}-libs/releases/download/${{ inputs.libs_release }}/simplex-chat-libs-${{ matrix.platform }}.zip - run: curl -fsSL --create-dirs -o "$RUNNER_TEMP/libsimplex/simplex-chat-libs-${{ matrix.platform }}.zip" "$URL" - - - name: Unpack libsimplex - if: needs.addon.outputs.test && inputs.libs_release - run: python -m zipfile -e "$RUNNER_TEMP/libsimplex/simplex-chat-libs-${{ matrix.platform }}.zip" "$RUNNER_TEMP/libsimplex" - - - name: Test simplex-chat (npm) + - name: Test simplex-chat (npm and PyPI) if: needs.addon.outputs.test - env: - SIMPLEX_LIBS_DIR: ${{ inputs.libs_release && format('{0}/libsimplex/libs', runner.temp) || '' }} - # A failed test can leave chat controllers running, and jest then waits for them. - # Tests on the public preset servers fail in CI when a server is slow to connect. - run: npx jest --forceExit --testNamePattern '^(?!.*use preset servers)' - - - name: Test simplex-chat (PyPI) - if: needs.addon.outputs.test - working-directory: packages/simplex-chat-python - run: | - python -m pip install -e ".[test]" - python -m pytest -q - - - name: Upload add-on artifact - uses: actions/upload-artifact@v7 - with: - name: simplex-nodejs-addon-${{ matrix.platform }} - path: packages/simplex-chat-nodejs/build/simplex-chat-nodejs-${{ matrix.platform }}.node - if-no-files-found: error - - publish: - needs: addon - # Trusted publishing accepts this workflow only when it is not called by another one (build.yml). - if: needs.addon.outputs.release_branch && contains(github.workflow_ref, '/.github/workflows/packages.yml@') - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write - # set by a publish step after it succeeds, so a failed publish still reports the earlier ones - outputs: - types: ${{ steps.types.outputs.version }} - nodejs: ${{ steps.nodejs.outputs.version }} - python: ${{ steps.python.outputs.version }} - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - - name: Versions to publish - id: plan - env: - GH_TOKEN: ${{ github.token }} - LIBS_REPOSITORY: ${{ github.repository }}-libs - run: python3 scripts/ci/packages_release.py publish-plan >> "$GITHUB_OUTPUT" - - - name: Setup Node.js - if: steps.plan.outputs.types_version || steps.plan.outputs.nodejs_version - uses: actions/setup-node@v7 - with: - node-version: 24 - registry-url: https://registry.npmjs.org - - - name: Setup Python - if: steps.plan.outputs.python_version - uses: actions/setup-python@v7 - with: - python-version: "3.12" - - # Before the test names: the build installs @simplex-chat/types from this repository. - - name: Build and test simplex-chat (npm) - if: steps.plan.outputs.nodejs_version - working-directory: packages/simplex-chat-nodejs - run: | - (cd ../simplex-chat-client/types/typescript && npm install --no-audit --no-fund && npx tsc) - npm install --ignore-scripts --no-save --no-audit --no-fund ../simplex-chat-client/types/typescript - npx tsc && cp src/simplex.* dist - npx jest tests/libs.test.ts tests/api.unit.test.ts - - - name: Use test package names - if: vars.NPM_SCOPE || vars.PYPI_PROJECT - run: python3 scripts/ci/packages_release.py test-names - - - name: Publish @simplex-chat/types - id: types - if: steps.plan.outputs.types_version - working-directory: packages/simplex-chat-client/types/typescript - env: - VERSION: ${{ steps.plan.outputs.types_version }} - DIST_TAG: ${{ steps.plan.outputs.types_dist_tag }} - run: | - npm install --no-audit --no-fund - npm publish --provenance --access public --tag "$DIST_TAG" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - - name: Publish simplex-chat (npm) - id: nodejs - if: steps.plan.outputs.nodejs_version - working-directory: packages/simplex-chat-nodejs - env: - VERSION: ${{ steps.plan.outputs.nodejs_version }} - DIST_TAG: ${{ steps.plan.outputs.nodejs_dist_tag }} - run: | - npm publish --provenance --access public --tag "$DIST_TAG" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - - name: Build and test simplex-chat (PyPI) - if: steps.plan.outputs.python_version - working-directory: packages/simplex-chat-python - run: | - python -m pip install build -e ".[test]" - python -m pytest -q - python -m build - - - name: Publish simplex-chat (PyPI) - if: steps.plan.outputs.python_version - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: packages/simplex-chat-python/dist - repository-url: ${{ vars.PYPI_UPLOAD_URL || 'https://upload.pypi.org/legacy/' }} - password: ${{ secrets.PYPI_API_TOKEN }} - - - name: Report the published PyPI version - id: python - if: steps.plan.outputs.python_version - env: - VERSION: ${{ steps.plan.outputs.python_version }} - run: echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - # The only job here that writes to simplex-chat-libs; publish runs npm and pip installs without its token. - record: - needs: publish - if: ${{ !cancelled() && (needs.publish.outputs.types || needs.publish.outputs.nodejs || needs.publish.outputs.python) }} - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - - name: Record published versions - env: - GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} - LIBS_REPOSITORY: ${{ github.repository }}-libs - TYPES: ${{ needs.publish.outputs.types }} - NODEJS: ${{ needs.publish.outputs.nodejs }} - PYTHON: ${{ needs.publish.outputs.python }} - run: | - python3 scripts/ci/packages_release.py record --commit "$GITHUB_SHA" \ - --types "$TYPES" --nodejs "$NODEJS" --python "$PYTHON" - - release: - needs: [addon, build] - # A merged release PR changes only versions and pins. - if: >- - !cancelled() && needs.addon.outputs.release_branch && needs.build.result != 'failure' - && !startsWith(github.event.head_commit.message, 'packages: release') - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - # Outside the checkout: the release PR commits every change in it. - - name: Download add-on artifacts - if: needs.addon.outputs.missing - uses: actions/download-artifact@v8 - with: - pattern: simplex-nodejs-addon-* - merge-multiple: true - path: ${{ runner.temp }}/addons - - - name: Create add-on release - if: needs.addon.outputs.missing - uses: softprops/action-gh-release@v3 - with: - repository: ${{ github.repository }}-libs - tag_name: ${{ needs.addon.outputs.tag }} - files: ${{ runner.temp }}/addons/*.node - token: ${{ secrets.NODEJS_REPO_TOKEN }} - - - name: Update pins and versions - id: prepare - env: - GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }} - LIBS_REPOSITORY: ${{ github.repository }}-libs - NODEJS_VERSION: ${{ inputs.nodejs_version }} - TYPES_VERSION: ${{ inputs.types_version }} - PYTHON_VERSION: ${{ inputs.python_version }} - run: | - python3 scripts/ci/packages_release.py prepare \ - --nodejs-version "$NODEJS_VERSION" \ - --types-version "$TYPES_VERSION" \ - --python-version "$PYTHON_VERSION" \ - --summary "$RUNNER_TEMP/summary.md" - echo "title=$(head -n 1 "$RUNNER_TEMP/summary.md")" >> "$GITHUB_OUTPUT" - - - name: Create or update release PR - uses: peter-evans/create-pull-request@v8 - with: - token: ${{ secrets.RELEASE_PR_TOKEN }} - branch: packages-release/${{ github.ref_name }} - base: ${{ github.ref_name }} - title: ${{ steps.prepare.outputs.title }} - commit-message: ${{ steps.prepare.outputs.title }} - body-path: ${{ runner.temp }}/summary.md - author: SimpleX Chat - committer: SimpleX Chat - delete-branch: true + run: scripts/ci/package-tests.sh test diff --git a/docs/contributing/RELEASES.md b/docs/contributing/RELEASES.md new file mode 100644 index 0000000000..ea6123e4c8 --- /dev/null +++ b/docs/contributing/RELEASES.md @@ -0,0 +1,153 @@ +# Library and package releases + +## Releases + +| Release | Location | Version | Published on | +|---|---|---|---| +| libsimplex | simplex-chat-libs release `core-` | `version` of `simplex-chat.cabal` | push to master or stable with a new core version | +| Node.js add-on | simplex-chat-libs release `nodejs-addon-<7 characters of the commit>` | | push to master or stable with unreleased add-on sources | +| `@simplex-chat/types` | npm | `version` of `packages/simplex-chat-client/types/typescript/package.json` | library release with an unpublished types version | +| `simplex-chat` | npm | `version` of `packages/simplex-chat-nodejs/package.json` | push to master or stable with an unpublished version | +| `simplex-chat` | PyPI | `__version__` of `packages/simplex-chat-python/src/simplex_chat/_version.py` | push to master or stable with an unpublished version | + +Add-on sources: +- `packages/simplex-chat-nodejs/binding.gyp` +- `packages/simplex-chat-nodejs/cpp/` +- `node-addon-api` version in `devDependencies` of `packages/simplex-chat-nodejs/package.json` + +Release states in simplex-chat-libs: + +``` +draft created by maybe-release; assets uploaded by the build jobs +published every asset present; published by the releases job +``` + +npm tag: `beta` for a version with a prerelease part, `latest` otherwise. + +Releases and publications are independent of the Haskell test results. + +`releases.toml` of simplex-chat-libs, written by `build.yml`: + +``` +core.. commit of the library release; branch: master or stable +npm.. commit of the publication +pypi.. commit of the publication +``` + +## Library release + +1. On master or stable, after `git pull`: + + ``` + python3 scripts/ci/packages_release.py core-release + ``` + + Checked: version higher than the highest version recorded for the branch. + + Written: + - `version` of `simplex-chat.cabal` + - with types changed since the previous library release of the branch: types `version` (bumped), `simplexChatCore`, line `SimpleX Chat core:` of the types README + + `--types-version` sets the types version, for the first release of an app version, e.g. `--types-version 7.1.0-beta.7.1`. + +2. Commit, push to the same branch. + +3. Run of `build.yml`: + - `maybe-release`: version claimed for the commit in `releases.toml`, draft `core-` created + - build jobs: 6 library zips uploaded + - `releases`: release published + - `publish`: `@simplex-chat/types` published, when its version is unpublished + - `record`: types version recorded + +Failed claim, with every build job skipped: +- version lower than the highest version recorded for the branch +- types changed, types version published +- types changed, `simplexChatCore` different from the core version + +After a failed claim, a fix commit is pushed; the version is claimed by the next push. + +Incomplete draft: completed by a re-run of the failed jobs in the run of the claim. The version is skipped by later pushes, as it is recorded for another commit. + +## Add-on release + +Run of `build.yml` on a push to master or stable with unreleased add-on sources: +- `maybe-release`: draft `nodejs-addon-<7 characters of the commit>` created, notes `sources: ` +- build jobs, 5 platforms: add-on built, tested with the latest library release of the branch, uploaded +- `releases`: release published + +Incomplete draft: completed by a re-run or by the next push with the same add-on sources. + +## Package release + +1. On master or stable, after `git pull`: + + ``` + python3 scripts/ci/packages_release.py prepare [nodejs] [python] + ``` + + Both packages when none is listed. + + Checked: + - published library release recorded for the branch + - add-on release published with every asset (Node.js) + - types version published on npm (Node.js) + - Python types unchanged since the commit of the pinned library release (Python) + - pinned releases published with the pinned SHA-256 + - every pinned asset downloaded with a matching SHA-256 + + Written: + - pins in `packages/simplex-chat-nodejs/src/release.ts` and `_version.py`: latest library release of the branch, add-on release of the current sources (Node.js), SHA-256 of every asset + - `@simplex-chat/types` dependency: `^` (Node.js) + - package versions: a published version bumped when the pins or the package files changed + + `--nodejs-version` and `--python-version` set the versions, for the first release of an app version, e.g. `--nodejs-version 7.1.0-beta.7.1 --python-version 7.1.0b7`. + +2. Commit, push to the same branch. + +3. Run of `build.yml`: + - build jobs, 5 platforms: packages tested, Node.js with the pinned add-on + - `publish`: package tests of 5 platforms and pins checked, unpublished versions published with trusted publishing + - `record`: published versions recorded + +Versions unpublished after a failure: published by the next push to the branch. + +## Local publication + +1. Step 1 of the package release. + +2. Package tests in a Python virtual environment, on the local platform (`linux-x86_64`, `linux-aarch64`, `macos-aarch64`, `macos-x86_64` or `windows-x86_64`): + + ``` + scripts/ci/package-tests.sh publish nodejs python + ``` + +3. In `packages/simplex-chat-nodejs`, with the npm tag of the version: + + ``` + npm publish --access public --tag beta + ``` + +4. In `packages/simplex-chat-python`: + + ``` + python -m pip install build twine + python -m build + twine upload dist/simplex_chat-* + ``` + +5. Commit, push to the same branch. Locally published versions are skipped by `build.yml` and are not recorded in `releases.toml`. + +## Settings + +- simplex-chat-libs: release immutability enabled +- secret `NODEJS_REPO_TOKEN`: contents of simplex-chat-libs, read and write +- trusted publisher of `simplex-chat` and `@simplex-chat/types` on npm and of `simplex-chat` on PyPI: workflow `build.yml` +- secrets `NPM_TOKEN` and `PYPI_API_TOKEN`: used instead of trusted publishing when set +- variables `NPM_SCOPE`, `PYPI_PROJECT`, `PYPI_URL` and `PYPI_UPLOAD_URL`: publication under other names, for test repositories + +## Pull requests + +- `packages.yml`, for package changes: + - core unchanged: add-on built and packages tested on 5 platforms + - core changed: add-on built on 5 platforms when its sources changed +- `build.yml`, for core changes: package tests on Linux x86_64 with the library built from the pull request diff --git a/packages/simplex-chat-client/types/typescript/README.md b/packages/simplex-chat-client/types/typescript/README.md index e30cf0d0c3..8d5f88657d 100644 --- a/packages/simplex-chat-client/types/typescript/README.md +++ b/packages/simplex-chat-client/types/typescript/README.md @@ -4,6 +4,8 @@ This TypeScript library provides auto-generated types for bots API: commands and It is used in [simplex-chat](https://www.npmjs.com/package/simplex-chat) Node.js library. +SimpleX Chat core: 7.1.0.12 + [API reference](https://github.com/simplex-chat/simplex-chat/tree/stable/bots). ## License diff --git a/packages/simplex-chat-client/types/typescript/package.json b/packages/simplex-chat-client/types/typescript/package.json index 7bf43e0840..983292abd1 100644 --- a/packages/simplex-chat-client/types/typescript/package.json +++ b/packages/simplex-chat-client/types/typescript/package.json @@ -1,6 +1,7 @@ { "name": "@simplex-chat/types", - "version": "0.12.1", + "version": "7.1.0-beta.6.1", + "simplexChatCore": "7.1.0.12", "description": "TypeScript types for SimpleX Chat bot libraries", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/simplex-chat-nodejs/package.json b/packages/simplex-chat-nodejs/package.json index 67eefdee4a..eaf9de68fb 100644 --- a/packages/simplex-chat-nodejs/package.json +++ b/packages/simplex-chat-nodejs/package.json @@ -1,6 +1,6 @@ { "name": "simplex-chat", - "version": "7.1.0-beta.6.3", + "version": "7.1.0-beta.6.4", "main": "dist/index.js", "types": "dist/index.d.ts", "gypfile": false, @@ -24,7 +24,7 @@ "docs": "typedoc" }, "dependencies": { - "@simplex-chat/types": "^0.12.1", + "@simplex-chat/types": "^7.1.0-beta.6.1", "extract-zip": "^2.0.1", "fast-deep-equal": "^3.1.3" }, diff --git a/packages/simplex-chat-nodejs/src/release.ts b/packages/simplex-chat-nodejs/src/release.ts index b56435e210..ad9dfa4c78 100644 --- a/packages/simplex-chat-nodejs/src/release.ts +++ b/packages/simplex-chat-nodejs/src/release.ts @@ -1,16 +1,16 @@ // Written by scripts/ci/packages_release.py from the simplex-chat-libs releases. -export const LIBS_RELEASE = "core-7.1.0.11" +export const LIBS_RELEASE = "core-7.1.0.12" export const ADDON_RELEASE: string | undefined = "nodejs-addon-308c0960c707" export const SHA256: {readonly [asset: string]: string} = { - "simplex-chat-libs-linux-aarch64.zip": "d66aa1b30b9fa20225dad8ef75a9c605961e1c0adde2ba0283e2fcac4e15d260", - "simplex-chat-libs-linux-x86_64-postgres.zip": "925b3166f969b12082ff8ee659e5f2a9933fe40120deb80c9aa3002a5ba6c803", - "simplex-chat-libs-linux-x86_64.zip": "dabad0824e8813ea4a084fbf37e1ba4cf6dcea9cd8ccadc521fec200f246208a", - "simplex-chat-libs-macos-aarch64.zip": "aa74608b149fff7508a47eaac4284dcc6d189c6ed8b7c18428df9158bc176a61", - "simplex-chat-libs-macos-x86_64.zip": "a0e21457e1077be7f71b46c1c27452945b2cee2b6c50efb0004412ce5b7042ec", - "simplex-chat-libs-windows-x86_64.zip": "c01dad574daa101a1cbfd31ddbf2b3d0d986a4a42f97d789d2f878b2050b2b4b", + "simplex-chat-libs-linux-aarch64.zip": "924b8e14eaba5e309fa584c4bfe598421cf64983d63790c1c45247f592f60055", + "simplex-chat-libs-linux-x86_64-postgres.zip": "323f908b45dde751d6082f8885d07d81fba8a5ab8b2df7d3b8ab3a87e56299e5", + "simplex-chat-libs-linux-x86_64.zip": "995409e963db408bcf896361b669944484f845b6744fefe479aebb0ca2e7c5cd", + "simplex-chat-libs-macos-aarch64.zip": "64464aa48b996e5a96b4aa651bc57bc69e412d3c3804b589c67bd9dc52faddbf", + "simplex-chat-libs-macos-x86_64.zip": "94664f96ff7a2cd5a419096e7e843928faa31cb3addb8869fa1e4b5a0d0b3629", + "simplex-chat-libs-windows-x86_64.zip": "2e10844e5c119921139c373ee2971341aceef7dc4198429d3be6b1570a009d15", "simplex-chat-nodejs-linux-aarch64.node": "3636a82c56c48229e0326fa03d6ceb5c16dd3afbcc1e86337783dc53a7328d67", "simplex-chat-nodejs-linux-x86_64.node": "55f3479b03fdb57591179ed730da744f1aa362e7529df55221b60028415746e9", "simplex-chat-nodejs-macos-aarch64.node": "a3472872fb0816f435f52ee5471088e6545ebffb95e89fbf8e1ab82275f2d56e", diff --git a/packages/simplex-chat-python/README.md b/packages/simplex-chat-python/README.md index b86886d591..aa892d32ab 100644 --- a/packages/simplex-chat-python/README.md +++ b/packages/simplex-chat-python/README.md @@ -56,15 +56,6 @@ ruff check && pyright && pytest tests/ Wire types under `src/simplex_chat/types/_*.py` are generated. Regenerate with `cabal test simplex-chat-test --test-options='--match Python'`. -## Release - -Manual for now. Bump `_version.py:__version__`, build a wheel, upload to PyPI: - -```bash -uv build --wheel -uv publish --token "$PYPI_TOKEN" -``` - ## License [AGPL-3.0](./LICENSE) diff --git a/packages/simplex-chat-python/src/simplex_chat/_version.py b/packages/simplex-chat-python/src/simplex_chat/_version.py index 0786a81795..e5075096d4 100644 --- a/packages/simplex-chat-python/src/simplex_chat/_version.py +++ b/packages/simplex-chat-python/src/simplex_chat/_version.py @@ -1,16 +1,16 @@ """Package version and the simplex-chat-libs release it loads. -Written by scripts/ci/packages_release.py: the release PR bumps the version +Written by scripts/ci/packages_release.py prepare: it bumps the version and pins new library releases with the SHA-256 of their assets. """ -__version__ = "7.1.0b6" # PEP 440 — read by hatchling for wheel metadata -LIBS_RELEASE = "core-7.1.0.11" # simplex-chat-libs release tag +__version__ = "7.1.0b6.post1" # PEP 440 — read by hatchling for wheel metadata +LIBS_RELEASE = "core-7.1.0.12" # simplex-chat-libs release tag LIBS_SHA256 = { - "simplex-chat-libs-linux-aarch64.zip": "d66aa1b30b9fa20225dad8ef75a9c605961e1c0adde2ba0283e2fcac4e15d260", - "simplex-chat-libs-linux-x86_64-postgres.zip": "925b3166f969b12082ff8ee659e5f2a9933fe40120deb80c9aa3002a5ba6c803", - "simplex-chat-libs-linux-x86_64.zip": "dabad0824e8813ea4a084fbf37e1ba4cf6dcea9cd8ccadc521fec200f246208a", - "simplex-chat-libs-macos-aarch64.zip": "aa74608b149fff7508a47eaac4284dcc6d189c6ed8b7c18428df9158bc176a61", - "simplex-chat-libs-macos-x86_64.zip": "a0e21457e1077be7f71b46c1c27452945b2cee2b6c50efb0004412ce5b7042ec", - "simplex-chat-libs-windows-x86_64.zip": "c01dad574daa101a1cbfd31ddbf2b3d0d986a4a42f97d789d2f878b2050b2b4b", + "simplex-chat-libs-linux-aarch64.zip": "924b8e14eaba5e309fa584c4bfe598421cf64983d63790c1c45247f592f60055", + "simplex-chat-libs-linux-x86_64-postgres.zip": "323f908b45dde751d6082f8885d07d81fba8a5ab8b2df7d3b8ab3a87e56299e5", + "simplex-chat-libs-linux-x86_64.zip": "995409e963db408bcf896361b669944484f845b6744fefe479aebb0ca2e7c5cd", + "simplex-chat-libs-macos-aarch64.zip": "64464aa48b996e5a96b4aa651bc57bc69e412d3c3804b589c67bd9dc52faddbf", + "simplex-chat-libs-macos-x86_64.zip": "94664f96ff7a2cd5a419096e7e843928faa31cb3addb8869fa1e4b5a0d0b3629", + "simplex-chat-libs-windows-x86_64.zip": "2e10844e5c119921139c373ee2971341aceef7dc4198429d3be6b1570a009d15", } diff --git a/plans/2026-10-18-package-releases.md b/plans/2026-10-18-package-releases.md new file mode 100644 index 0000000000..286ab2cf26 --- /dev/null +++ b/plans/2026-10-18-package-releases.md @@ -0,0 +1,364 @@ +# Package releases + +Repository: simplex-chat/simplex-chat. +Prerequisite: PR #7750 merged; master runs are not cancelled. +Scope: releases in simplex-chat-libs, npm and PyPI. +App releases (`v*` tags in simplex-chat): unchanged. + +## Releases + +| Release | Repository | Name | Released when | Assets | +|---|---|---|---|---| +| libsimplex | simplex-chat-libs | `core-` | push to master or stable claims the version for its branch | `LIBS_ASSETS` | +| `@simplex-chat/types` | npm | version of its `package.json` | library release push, version absent from npm | | +| add-on | simplex-chat-libs | `nodejs-addon-<7 characters of the pushed commit sha>` | push to master or stable, published release of the add-on hash absent | `ADDON_ASSETS` | +| `simplex-chat` (npm) | npm | version of its `package.json` | push to master or stable, version absent from npm | | +| `simplex-chat` (PyPI) | PyPI | `__version__` of `_version.py` | push to master or stable, version absent from PyPI | | + +Release states in simplex-chat-libs: + +``` +absent +draft created before the build jobs; assets uploaded by the build jobs +published every asset present; immutable +``` + +Add-on hash: output of the current `addon_tag` function without its prefix. +Add-on release notes: `sources: `. + +Types source: `packages/simplex-chat-client/types/typescript/src`. +Python types source: `packages/simplex-chat-python/src/simplex_chat/types`. + +## Library claim + +Claim: entry `core..` = simplex-chat commit in `releases.toml` of simplex-chat-libs. +Branches: master, stable. +Written through the contents API with the blob sha; a concurrent writer re-reads after 409. + +Rows in order of evaluation: + +| `core..` | `claim-core` result | +|---|---| +| version lower than the highest version recorded for the branch | exit 1 | +| version recorded on any branch for this commit | `resume` | +| version recorded on any branch for another commit | `skip` | +| absent, types check failed | exit 1 | +| absent, types check passed | `new`: entry written | + +Previous core release: commit of the highest version recorded for the branch. + +Types check: + +``` +types changed types source differs from the previous core release +passed types unchanged + or types version unrecorded, absent from npm, simplexChatCore = +``` + +## Types versions + +``` +scheme ., as simplex-chat (npm) +first 7.1.0-beta.6.1, from core-release --types-version +next bump_npm_version of the types version at the previous core release +npm tag dist_tag: beta for a prerelease version, latest otherwise +``` + +## Manual settings + +1. simplex-chat-libs: immutable releases enabled. +2. npm trusted publisher of `simplex-chat`: workflow `build.yml`. +3. npm trusted publisher of `@simplex-chat/types`: workflow `build.yml`. +4. PyPI trusted publisher of `simplex-chat`: workflow `build.yml`. +5. Secret `RELEASE_PR_TOKEN`: deleted. +6. PR #7732: closed; branch `packages-release/master`: deleted. +7. `releases.toml` entry added: `core.master."7.1.0.11"` = `9711dcfc9c39ba125c1fb1674a3e039b1b9c0e81`. +8. Notes of release `nodejs-addon-308c0960c707`: `sources: 308c0960c707`. + +## Files + +- `scripts/ci/packages_release.py` +- `scripts/ci/test_packages_release.py` +- `scripts/ci/package-tests.sh`: new. +- `.github/workflows/build.yml` +- `.github/workflows/packages.yml` +- `packages/simplex-chat-client/types/typescript/package.json`: field `simplexChatCore`, `7.1.0.11`. +- `packages/simplex-chat-client/types/typescript/README.md`: line `SimpleX Chat core: 7.1.0.11`. + +## packages_release.py + +Asset set by tag prefix: + +``` +core- LIBS_ASSETS +nodejs-addon- ADDON_ASSETS +``` + +Commands: + +``` +core-release [--branch ] [--types-version ] + run locally + branch: the checked out branch by default + version not higher than the highest version recorded for the branch: exit 1 + version: of simplex-chat.cabal written + types changed or --types-version: types version written, as in Types versions + types version written: simplexChatCore and the README line written +claim-core + run in maybe-release + Library claim; prints claim=new, claim=resume or claim=skip + new: also prints commit= +release-status + prints status=published, status=draft or status=absent + drafts: from the authenticated release list +addon-release + releases nodejs-addon-* with notes "sources: " + published one: prints tag= and status=published + draft: prints tag= and status=draft + neither: prints status=absent +latest-libs [--branch ] + prints tag= + order: numeric components of the version +publish-release + draft with every asset of its set: published + missing assets: listed, exit 1 +check-pins + pins of release.ts and _version.py compared with the digests of published releases + mismatch, missing asset or unpublished release: listed, exit 1 +download + asset saved to / + SHA-256 compared with the API digest; mismatch: exit 1 +download-pinned + tag and SHA-256 from release.ts + asset saved to /; mismatch: exit 1 +prepare [nodejs] [python] [--branch ] + run locally + branch: the checked out branch by default + nodejs: simplex-chat (npm); python: simplex-chat (PyPI) + packages: the listed ones; all when none is listed + other package: pins and version unchanged + library pin: latest-libs of the branch + warning: core- newer than the pinned release + nodejs add-on pin: addon-release; exit 1 unless published with every asset + nodejs types dependency: ^; exit 1 unless published on npm + python: exit 1 when the Python types source differs from the commit recorded for the pinned core version + every pinned asset: download, SHA-256 compared with the API digest + changed package: directory without tests differs from the commit that set its version + versions: bumped as now; --nodejs-version, --python-version override + written: release.ts, _version.py, package.json of simplex-chat (npm) + summary: printed; --summary and --types-version removed +build-plan + pull requests only + --event and --release-missing removed +``` + +Commands removed: `addon-tag`. +`record`: option `--branch` of `--core`, default master; replaced by `claim-core` in maybe-release. +Unchanged: `changes`, `publish-plan`, `test-names`. + +`build_plan(core_changed, addon_changed)`: + +``` +core changed, add-on changed platforms: all; test: false +core changed, add-on unchanged platforms: none; test: false +core unchanged platforms: all; test: true +``` + +## package-tests.sh + +Commands from the build steps of `packages.yml` (lines 147-166). + +``` +build + types, npm package and add-on built + add-on copied to packages/simplex-chat-nodejs/build/simplex-chat-nodejs-.node +test + jest, as in packages.yml; SIMPLEX_LIBS_DIR and SIMPLEX_ADDON_PATH from the environment + pytest +addon + build + simplex-chat-libs-.zip of $LIBS_RELEASE: packages_release.py download, unpack to $RUNNER_TEMP/libs + jest with SIMPLEX_LIBS_DIR=$RUNNER_TEMP/libs/libs + add-on uploaded to draft $ADDON_TAG of $LIBS_REPOSITORY: gh release upload --clobber, token $GH_TOKEN +publish [nodejs] [python] + nodejs: types and npm package built + download-pinned simplex-chat-nodejs-.node to $RUNNER_TEMP/addon + add-on copied to packages/simplex-chat-nodejs/build/Release/simplex.node, the path of the tests + jest + python: pytest + run in CI and locally +``` + +## build.yml + +### maybe-release + +Outputs: + +``` +libs_tag core-, when the claim is new or resume +addon_tag tag of the add-on draft +addon_draft true when the add-on release is a draft after this job +libs_latest output tag of latest-libs +publish_types libs_tag set, types version printed by publish-plan +publish_nodejs nodejs version printed by publish-plan +publish_python python version printed by publish-plan +publish publish_nodejs or publish_python +``` + +Steps after "Create release": + +| Step | Condition | Action | +|---|---|---| +| Library release | push to master or stable | `claim-core `; new → `gh release create core- --draft --target `; resume → existing draft, `libs_tag` empty when published; skip → notice annotation | +| Add-on release | push to master or stable | `addon-release`: absent → `gh release create nodejs-addon-<7 characters of github.sha> --draft --notes "sources: "`; draft → reused; output `addon_draft` true when absent or draft | +| Latest library | `addon_draft` | `latest-libs --branch ` | +| Versions to publish | push to master or stable | `publish-plan`; environment `NPM_SCOPE`, `PYPI_PROJECT`, `PYPI_URL` as in `packages.yml` | + +Steps removed: "Check version change", "Record libs release", "Create libs release". +"Clone project": `fetch-depth: 2`, unchanged; the types check fetches the commit of the recorded types version. +Token of the release steps: `NODEJS_REPO_TOKEN`. +Library release jobs: concurrency groups with `libs_tag`, unchanged. + +### Library uploads + +Steps "Upload library" (build-linux, build-macos, build-windows) and "Upload postgres library": + +``` +action gh release upload --clobber --repo -libs +token NODEJS_REPO_TOKEN +conditions unchanged +``` + +### Package steps in build jobs + +Platform jobs: + +``` +build-linux matrix.libs entry linux-x86_64 +build-linux-push matrix.libs entry linux-aarch64 +build-macos both entries macos- +build-windows windows-latest windows-x86_64 +``` + +Steps, after "Run tests" (build-linux, build-macos) and after "Upload library" (build-windows): + +| Step | Condition | +|---|---| +| Setup Node.js 22 | `!cancelled()`; core pull request condition, `addon_draft` or `publish` | +| Setup Python 3.12 | `!cancelled()`, Setup Node.js succeeded | +| Package tests (pull request) | build-linux only, as now; `package-tests.sh build linux-x86_64`, then `test` with SIMPLEX_LIBS_DIR | +| Add-on | `!cancelled()`, `addon_draft`, Setup Python succeeded; `package-tests.sh addon ` | +| Package tests | `!cancelled()`, `publish`, Setup Python succeeded; `package-tests.sh publish `, with `nodejs` when `publish_nodejs`, with `python` when `publish_python` | +| Package tests result | Package tests succeeded; artifact `package-tests-` | + +Linux steps: on the host. + +Environment of "Add-on": + +``` +LIBS_RELEASE libs_latest +ADDON_TAG addon_tag +LIBS_REPOSITORY -libs +GH_TOKEN NODEJS_REPO_TOKEN +``` + +### Final jobs + +``` +releases + needs maybe-release, build-linux, build-linux-push, build-linux-postgres, build-macos, build-windows + if !cancelled(), libs_tag or addon_draft + step 1 publish-release ; libs_tag + step 2 publish-release ; !cancelled(), addon_draft + token NODEJS_REPO_TOKEN +publish + needs maybe-release, build-linux, build-linux-push, build-macos, build-windows, releases + if !cancelled(), publish or publish_types + permissions contents: read; id-token: write + env as in packages.yml: NODE_AUTH_TOKEN, NPM_SCOPE, PYPI_PROJECT, PYPI_URL + step 1 when publish: artifacts package-tests-*, 5 required + step 2 when publish: check-pins + steps "publish" job of packages.yml, from "Versions to publish" + types step publish_types and needs.releases.result == 'success' +record + as in packages.yml; needs publish +``` + +Comments of the `publish` job of `packages.yml`: moved with it. +Comment of the `record` job: removed. +Trusted publishing comment (`packages.yml` lines 12-15): moved above `publish`. + +Job removed: `packages`, with its header comment. + +## packages.yml + +``` +triggers pull_request, paths unchanged + push, workflow_dispatch, workflow_call: removed +addon job steps "Add-on release tag", "Release branch", "Check the add-on release": removed + outputs tag, release_branch, missing: removed + Build plan: --event and --release-missing removed +build job Checkout, Setup Node.js, Setup Python + package-tests.sh build + package-tests.sh test, when planned + "Upload add-on artifact": removed +jobs removed publish, record, release +header lines 3-10 and 17-28: removed +``` + +## Release procedure + +Library and types: + +1. `python3 scripts/ci/packages_release.py core-release ` on master or stable, with `--types-version` for a new app version. +2. Commit of the written files; push to the same branch. +3. `build.yml`: claim, draft, uploads by the build jobs, `releases`, types publication. +4. Failed claim: fix commit; the next push claims the version. +5. Incomplete draft: re-run of the failed build jobs, then of `releases` and `publish`. + +Add-on: + +1. Push to master or stable changes the add-on sources. +2. Incomplete draft: completed by a re-run or by the next push. + +Packages, published by GitHub: + +1. `git pull` on master or stable. +2. `python3 scripts/ci/packages_release.py prepare`, with package names and version overrides when needed. +3. Commit of the written files; push to the same branch. +4. Versions absent from the registries: published by `build.yml`. + +Packages, published locally: + +1. Steps 1 and 2 above. +2. `scripts/ci/package-tests.sh publish [nodejs] [python]`. +3. `npm publish` of `simplex-chat`; `python -m build` and `twine upload` of the Python package. +4. Step 3 above; `build.yml` skips the published versions. + +Published npm package: `files` of `package.json` and the pins; add-on and libsimplex downloaded at run time. + +## Verification + +1. Unit tests: + - `claim-core`: every row of the Library claim table + - types check: changed and unchanged types, recorded and unrecorded types version + - `core-release` with and without types changes + - `release-status` + - `addon-release` notes lookup + - `prepare` with one package listed + - `prepare python` with changed Python types + - `latest-libs` order + - `publish-release` with missing assets + - `check-pins` mismatch + - `download` digest mismatch + - `build_plan` cases +2. `gh release upload` to a draft by tag: `shared.FetchRelease` of gh looks up the draft by tag (`fetchDraftRelease`). +3. First library release after the merge: assets uploaded to the draft, release published by `publish-release`. +4. First master push after merge: claim `skip`, as `core-7.1.0.11` is recorded for `9711dcfc`; `addon-release` finds `nodejs-addon-308c0960c707`; package steps skipped. +5. Local `prepare`: pins `core-7.1.0.11` and `nodejs-addon-308c0960c707`; every download matches. +6. `core-release 7.1.0.12 --types-version 7.1.0-beta.6.1`, commit, push: claim `new`, draft, 6 assets uploaded, release published, types 7.1.0-beta.6.1 published with tag beta. +7. Master push with an add-on source change: draft created, 5 assets uploaded, release published. +8. Master push with versions from `prepare`: 5 artifacts `package-tests-*`, pins checked, versions published, versions recorded. +9. Second push during the run of step 6: claim `skip`. +10. Stable push after master is merged into stable, with a stable core version: claim `new` for `core.stable`. diff --git a/scripts/ci/package-tests.sh b/scripts/ci/package-tests.sh new file mode 100755 index 0000000000..33d83c4986 --- /dev/null +++ b/scripts/ci/package-tests.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -euo pipefail + +cd "$(dirname "$0")/../.." +nodejs=packages/simplex-chat-nodejs +python_package=packages/simplex-chat-python +python=$(command -v python3 || command -v python) +temp=${RUNNER_TEMP:-$(mktemp -d)} + +usage() { + echo "usage: $0 build | test | addon | publish [nodejs] [python]" >&2 + exit 1 +} + +build_package() { + (cd packages/simplex-chat-client/types/typescript && npm install --no-audit --no-fund && npx tsc) + (cd "$nodejs" && npm install --ignore-scripts --no-save --no-audit --no-fund ../simplex-chat-client/types/typescript) + if [ -n "${1:-}" ]; then + (cd "$nodejs" && npx --yes node-gyp rebuild) + fi + (cd "$nodejs" && npx tsc && cp src/simplex.* dist) + if [ -n "${1:-}" ]; then + cp "$nodejs/build/Release/simplex.node" "$nodejs/build/simplex-chat-nodejs-$1.node" + fi +} + +test_nodejs() { + (cd "$nodejs" && npx jest --forceExit --testNamePattern '^(?!.*use preset servers)') +} + +test_python() { + (unset SIMPLEX_LIBS_DIR && cd "$python_package" && "$python" -m pip install -e ".[test]" && "$python" -m pytest -q) +} + +case "${1:-}" in + build) + [ $# -eq 2 ] || usage + build_package "$2" + ;; + test) + [ $# -eq 1 ] || usage + test_nodejs + test_python + ;; + addon) + [ $# -eq 2 ] || usage + build_package "$2" + zip=simplex-chat-libs-$2.zip + "$python" scripts/ci/packages_release.py download "$LIBS_RELEASE" "$zip" "$temp/libs" + "$python" -m zipfile -e "$temp/libs/$zip" "$temp/libs" + SIMPLEX_LIBS_DIR=$temp/libs/libs test_nodejs + gh release upload "$ADDON_TAG" "$nodejs/build/simplex-chat-nodejs-$2.node" --clobber --repo "$LIBS_REPOSITORY" + ;; + publish) + [ $# -ge 2 ] || usage + platform=$2 + shift 2 + for suite in "$@"; do + case "$suite" in + nodejs) + build_package + addon=simplex-chat-nodejs-$platform.node + "$python" scripts/ci/packages_release.py download-pinned "$addon" "$temp/addon" + mkdir -p "$nodejs/build/Release" + cp "$temp/addon/$addon" "$nodejs/build/Release/simplex.node" + test_nodejs + ;; + python) + test_python + ;; + *) + usage + ;; + esac + done + ;; + *) + usage + ;; +esac diff --git a/scripts/ci/packages_release.py b/scripts/ci/packages_release.py index b13b9cc00d..fdfd7c3b68 100644 --- a/scripts/ci/packages_release.py +++ b/scripts/ci/packages_release.py @@ -1,13 +1,21 @@ #!/usr/bin/env python3 """Release automation for simplex-chat and @simplex-chat/types on npm and simplex-chat on PyPI. -addon-tag print the add-on release tag for the current sources -changes print whether the changes since a commit touch the chat core and the add-on -build-plan print the platforms to build the add-on on and whether to run the package tests -prepare update pins and versions for a release PR -publish-plan print the package versions that are not published yet -record record releases and published versions in releases.toml of simplex-chat-libs -test-names rename the packages to NPM_SCOPE and PYPI_PROJECT (test repositories) +core-release set the core version and the types version for a library release +claim-core claim the library release of the core version for a commit +release-status print whether a release of simplex-chat-libs is published, a draft or absent +addon-release print the add-on release of the current add-on sources +latest-libs print the newest published library release +publish-release publish a draft release that has every asset +check-pins check the pins of the packages against the published releases +download download a release asset and check its digest +download-pinned download a pinned asset and check its pin +changes print whether the changes since a commit touch the chat core and the add-on +build-plan print the platforms to build the add-on on and whether to run the package tests +prepare update pins and versions of the packages +publish-plan print the package versions that are not published yet +record record releases and published versions in releases.toml of simplex-chat-libs +test-names rename the packages to NPM_SCOPE and PYPI_PROJECT (test repositories) """ from __future__ import annotations @@ -19,12 +27,14 @@ import hashlib import json import os import re +import shlex import subprocess import sys +import tempfile import tomllib import urllib.error import urllib.request -from collections.abc import Callable +from collections.abc import Callable, Collection from dataclasses import dataclass, field, replace from typing import Any from pathlib import Path @@ -37,7 +47,15 @@ RELEASE_FILE = f"{NODEJS_DIR}/src/release.ts" PYTHON_VERSION_FILE = f"{PYTHON_DIR}/src/simplex_chat/_version.py" ADDON_SOURCES = [f"{NODEJS_DIR}/binding.gyp", f"{NODEJS_DIR}/cpp"] CORE_SOURCES = ["src", "simplex-chat.cabal", "cabal.project"] -ADDON_TAG_HASH_LENGTH = 12 +CABAL_FILE = "simplex-chat.cabal" +CABAL_VERSION = re.compile(r"^(version:\s*)(\S+)", re.M) +TYPES_SOURCES = [f"{TYPES_DIR}/src"] +TYPES_README = f"{TYPES_DIR}/README.md" +TYPES_README_CORE = re.compile(r"^SimpleX Chat core: .*$", re.M) +PYTHON_TYPES_SOURCES = [f"{PYTHON_DIR}/src/simplex_chat/types"] +ADDON_HASH_LENGTH = 12 +CORE_TAG = re.compile(r"^core-(\d+(?:\.\d+)*)$") +ADDON_TAG_PREFIX = "nodejs-addon-" LIBS_ASSETS = [ "simplex-chat-libs-linux-aarch64.zip", "simplex-chat-libs-linux-x86_64-postgres.zip", @@ -58,18 +76,27 @@ ADDON_ASSETS = [f"simplex-chat-nodejs-{platform}.node" for platform in ADDON_PLA SHA256_HEX = re.compile(r"^[0-9a-f]{64}$") RELEASES_FILE = "releases.toml" RECORD_ATTEMPTS = 5 +DOWNLOAD_CHUNK = 1 << 20 TOML_BARE_KEY = re.compile(r"^[A-Za-z0-9_-]+$") # Pure functions -def addon_tag(files: dict[str, bytes], node_addon_api: str) -> str: +def addon_hash(files: dict[str, bytes], node_addon_api: str) -> str: digest = hashlib.sha256() for name in sorted(files): digest.update(name.encode() + b"\0" + files[name] + b"\0") digest.update(f"node-addon-api@{node_addon_api}".encode()) - return f"nodejs-addon-{digest.hexdigest()[:ADDON_TAG_HASH_LENGTH]}" + return digest.hexdigest()[:ADDON_HASH_LENGTH] + + +def addon_notes(sources_hash: str) -> str: + return f"sources: {sources_hash}" + + +def version_key(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) def bump_npm_version(version: str) -> str: @@ -141,7 +168,7 @@ def render_python_version(version: str, pins: Pins) -> str: return ( '"""Package version and the simplex-chat-libs release it loads.\n' "\n" - "Written by scripts/ci/packages_release.py: the release PR bumps the version\n" + "Written by scripts/ci/packages_release.py prepare: it bumps the version\n" "and pins new library releases with the SHA-256 of their assets.\n" '"""\n' "\n" @@ -152,16 +179,84 @@ def render_python_version(version: str, pins: Pins) -> str: ) +def asset_digests(release: dict) -> dict[str, str]: + return {a["name"]: (a.get("digest") or "").removeprefix("sha256:") for a in release["assets"]} + + def release_digests(release: dict | None, assets: list[str]) -> dict[str, str] | None: """Asset digests of a GitHub release, or None when the release is missing or incomplete.""" if release is None: return None - digests = {a["name"]: (a.get("digest") or "").removeprefix("sha256:") for a in release["assets"]} + digests = asset_digests(release) if not all(SHA256_HEX.match(digests.get(asset, "")) for asset in assets): return None return {asset: digests[asset] for asset in assets} +def release_assets(tag: str) -> list[str]: + if CORE_TAG.match(tag): + return LIBS_ASSETS + if tag.startswith(ADDON_TAG_PREFIX): + return ADDON_ASSETS + raise ValueError(f"no asset set for release {tag}") + + +def preferred_release(releases: list[dict]) -> dict | None: + """The published release among them, otherwise the first draft.""" + return next((r for r in releases if not r["draft"]), next(iter(releases), None)) + + +def release_status(release: dict | None) -> str: + if release is None: + return "absent" + return "draft" if release["draft"] else "published" + + +def tag_release(releases: list[dict], tag: str) -> dict | None: + return preferred_release([r for r in releases if r["tag_name"] == tag]) + + +def addon_release(releases: list[dict], sources_hash: str) -> dict | None: + notes = addon_notes(sources_hash) + return preferred_release( + [r for r in releases if r["tag_name"].startswith(ADDON_TAG_PREFIX) and r.get("body") == notes] + ) + + +def core_version(tag: str) -> str | None: + match = CORE_TAG.match(tag) + return match[1] if match else None + + +def latest_libs(releases: list[dict], versions: Collection[str]) -> str | None: + """The newest published release of the core versions that has every library asset.""" + tags = [ + r["tag_name"] + for r in releases + if not r["draft"] and core_version(r["tag_name"]) in versions and release_digests(r, LIBS_ASSETS) + ] + return max(tags, key=lambda tag: version_key(core_version(tag)), default=None) + + +def missing_assets(release: dict) -> list[str]: + present = {a["name"] for a in release["assets"]} + return [asset for asset in release_assets(release["tag_name"]) if asset not in present] + + +def pin_problems(pins: Pins, published: dict[str, dict | None]) -> list[str]: + """Differences between the pins and the published releases of the pinned tags.""" + problems = [] + for tag, release in published.items(): + if release is None: + problems.append(f"{tag} is not published") + continue + digests = asset_digests(release) + for asset in release_assets(tag): + if pins.sha256.get(asset) != digests.get(asset): + problems.append(f"{asset} of {tag}: pinned {pins.sha256.get(asset)}, published {digests.get(asset)}") + return problems + + @dataclass(frozen=True) class Package: version: str @@ -177,10 +272,11 @@ class State: addon_digests: dict[str, str] | None nodejs_pins: Pins python_pins: Pins - types: Package + types_dependency: str + nodejs_types_dependency: str nodejs: Package python: Package - types_version: str | None = None + packages: frozenset[str] = frozenset({"nodejs", "python"}) nodejs_version: str | None = None python_version: str | None = None @@ -189,23 +285,16 @@ class State: class Plan: nodejs_pins: Pins python_pins: Pins - types_version: str - types_bumped: bool + types_dependency: str | None nodejs_version: str nodejs_changed: bool python_version: str python_changed: bool notes: list[str] = field(default_factory=list) - @property - def types_dependency(self) -> str | None: - return f"^{self.types_version}" if self.types_bumped else None - @property def title(self) -> str: parts = [] - if self.types_bumped: - parts.append(f"types {self.types_version}") if self.nodejs_changed: parts.append(f"nodejs {self.nodejs_version}") if self.python_changed: @@ -214,7 +303,7 @@ class Plan: def plan(state: State) -> Plan: - """What a release PR changes, from the state of the branch and the releases.""" + """What a package release changes, from the state of the branch and the releases.""" notes: list[str] = [] def pin(pins: Pins, field_name: str, tag: str, digests: dict[str, str] | None, what: str) -> tuple[Pins, bool]: @@ -227,27 +316,30 @@ def plan(state: State) -> Plan: notes.append(f"{what}: {tag}") return replace(pins, **{field_name: tag, "sha256": {**pins.sha256, **digests}}), True - nodejs_pins, nodejs_libs = pin( - state.nodejs_pins, "libs_release", state.libs_tag, state.libs_digests, "Node.js libsimplex" - ) - nodejs_pins, addon = pin(nodejs_pins, "addon_release", state.addon_tag, state.addon_digests, "Node.js add-on") - python_pins, python_libs = pin( - state.python_pins, "libs_release", state.libs_tag, state.libs_digests, "Python libsimplex" - ) - def next_version(pkg: Package, changed: bool, override: str | None, bump) -> str: if override: return override return bump(pkg.version) if changed and pkg.published else pkg.version - types_version = next_version(state.types, state.types.changed, state.types_version, bump_npm_version) - types_bumped = types_version != state.types.version - nodejs_changed = nodejs_libs or addon or types_bumped or state.nodejs.changed - nodejs_version = next_version(state.nodejs, nodejs_changed, state.nodejs_version, bump_npm_version) - python_changed = python_libs or state.python.changed - python_version = next_version(state.python, python_changed, state.python_version, bump_python_version) + nodejs_pins, nodejs_version, nodejs_changed, types_dependency = state.nodejs_pins, state.nodejs.version, False, None + if "nodejs" in state.packages: + nodejs_pins, nodejs_libs = pin( + state.nodejs_pins, "libs_release", state.libs_tag, state.libs_digests, "Node.js libsimplex" + ) + nodejs_pins, addon = pin(nodejs_pins, "addon_release", state.addon_tag, state.addon_digests, "Node.js add-on") + if state.types_dependency != state.nodejs_types_dependency: + types_dependency = state.types_dependency + notes.append(f"@simplex-chat/types: {types_dependency}") + nodejs_changed = nodejs_libs or addon or types_dependency is not None or state.nodejs.changed + nodejs_version = next_version(state.nodejs, nodejs_changed, state.nodejs_version, bump_npm_version) + python_pins, python_version, python_changed = state.python_pins, state.python.version, False + if "python" in state.packages: + python_pins, python_libs = pin( + state.python_pins, "libs_release", state.libs_tag, state.libs_digests, "Python libsimplex" + ) + python_changed = python_libs or state.python.changed + python_version = next_version(state.python, python_changed, state.python_version, bump_python_version) for name, old, new in [ - ("@simplex-chat/types", state.types.version, types_version), ("simplex-chat (npm)", state.nodejs.version, nodejs_version), ("simplex-chat (PyPI)", state.python.version, python_version), ]: @@ -256,8 +348,7 @@ def plan(state: State) -> Plan: return Plan( nodejs_pins=nodejs_pins, python_pins=python_pins, - types_version=types_version, - types_bumped=types_bumped, + types_dependency=types_dependency, nodejs_version=nodejs_version, nodejs_changed=nodejs_changed or nodejs_version != state.nodejs.version, python_version=python_version, @@ -272,31 +363,49 @@ class BuildPlan: test: bool -def build_plan( - event: str, release_missing: bool, libs_release: str, core_changed: bool, addon_changed: bool -) -> BuildPlan: - """Each package test runs once, against the libsimplex of the change: a pull request that changes the core - is tested by build.yml with its own libsimplex, a new library release with that release.""" - if libs_release: - return BuildPlan(ADDON_PLATFORMS, test=True) - if event == "pull_request": - if core_changed: - return BuildPlan(ADDON_PLATFORMS if addon_changed else [], test=False) - return BuildPlan(ADDON_PLATFORMS, test=True) - return BuildPlan(ADDON_PLATFORMS if release_missing else [], test=True) +def build_plan(core_changed: bool, addon_changed: bool) -> BuildPlan: + """The package tests of a pull request that changes the core run in build.yml, with its own libsimplex.""" + if core_changed: + return BuildPlan(ADDON_PLATFORMS if addon_changed else [], test=False) + return BuildPlan(ADDON_PLATFORMS, test=True) def build_matrix(platforms: list[str]) -> str: return json.dumps([{"platform": platform, "os": ADDON_RUNNERS[platform]} for platform in platforms]) -def recorded_commit(releases: dict, key: str, version: str) -> str | None: - """The simplex-chat commit recorded for a version; key is "core", "npm/" or "pypi/".""" +def core_key(branch: str) -> str: + return f"core/{branch}" + + +def recorded_versions(releases: dict, key: str) -> dict[str, str]: + """The recorded versions and their simplex-chat commits; key is "core/", "npm/" or "pypi/".""" table = releases for part in key.split("/", 1): table = table.get(part, {}) - commit = table.get(version) - return commit if isinstance(commit, str) else None + return {version: commit for version, commit in table.items() if isinstance(commit, str)} + + +def recorded_commit(releases: dict, key: str, version: str) -> str | None: + return recorded_versions(releases, key).get(version) + + +def highest_recorded(releases: dict, key: str) -> str | None: + return max(recorded_versions(releases, key), key=version_key, default=None) + + +def recorded_core_commit(releases: dict, version: str) -> str | None: + """The commit recorded for the core version on any branch: a release tag is shared by the branches.""" + for branch in releases.get("core", {}): + if commit := recorded_commit(releases, core_key(branch), version): + return commit + return None + + +def previous_core_commit(releases: dict, branch: str) -> str | None: + """The commit of the highest core version recorded for the branch.""" + previous = highest_recorded(releases, core_key(branch)) + return recorded_commit(releases, core_key(branch), previous) if previous else None def add_releases(releases: dict, entries: list[tuple[str, str]], commit: str) -> dict: @@ -357,14 +466,18 @@ def write_json(file: str, value: dict) -> None: (ROOT / file).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n") -def current_addon_tag() -> str: +def current_addon_hash() -> str: files = { path.relative_to(ROOT).as_posix(): path.read_bytes() for source in ADDON_SOURCES for path in ([ROOT / source] if (ROOT / source).is_file() else sorted((ROOT / source).rglob("*"))) if path.is_file() } - return addon_tag(files, read_json(f"{NODEJS_DIR}/package.json")["devDependencies"]["node-addon-api"]) + return addon_hash(files, read_json(f"{NODEJS_DIR}/package.json")["devDependencies"]["node-addon-api"]) + + +def cabal_version() -> str: + return CABAL_VERSION.search(read_text(CABAL_FILE))[2] def changed_since(base: str, sources: list[str]) -> bool: @@ -378,13 +491,9 @@ def changes(base: str) -> None: def print_build_plan(args: argparse.Namespace) -> None: - pull_request = args.event == "pull_request" result = build_plan( - args.event, - release_missing=args.release_missing, - libs_release=args.libs_release, - core_changed=pull_request and changed_since(args.base, CORE_SOURCES), - addon_changed=pull_request and changed_since(args.base, ADDON_SOURCES), + core_changed=changed_since(args.base, CORE_SOURCES), + addon_changed=changed_since(args.base, ADDON_SOURCES), ) print(f"platforms={build_matrix(result.platforms)}") print(f"test={'true' if result.test else ''}") @@ -429,10 +538,13 @@ def http_json(url: str, headers: dict[str, str] | None = None, method: str = "GE # Workflows pass unset repository variables as empty strings. -def npm_published(name: str, version: str) -> bool: +def npm_package_url(name: str) -> str: registry = os.environ.get("NPM_REGISTRY") or "https://registry.npmjs.org" - package = registry_name(name, os.environ.get("NPM_SCOPE")).replace("/", "%2F") - return http_json(f"{registry}/{package}/{version}") is not None + return f"{registry}/{registry_name(name, os.environ.get('NPM_SCOPE')).replace('/', '%2F')}" + + +def npm_published(name: str, version: str) -> bool: + return http_json(f"{npm_package_url(name)}/{version}") is not None def python_project() -> str: @@ -445,19 +557,50 @@ def pypi_published(version: str) -> bool: return http_json(f"{index}/pypi/{project}/{version}/json") is not None +def libs_repository() -> str: + return os.environ.get("LIBS_REPOSITORY") or "simplex-chat/simplex-chat-libs" + + def libs_api(path: str, method: str = "GET", body: dict | None = None) -> Any: """GitHub API of the simplex-chat-libs repository.""" - repo = os.environ.get("LIBS_REPOSITORY") or "simplex-chat/simplex-chat-libs" headers = {"Accept": "application/vnd.github+json"} if token := os.environ.get("GH_TOKEN"): headers["Authorization"] = f"Bearer {token}" - return http_json(f"https://api.github.com/repos/{repo}/{path}", headers, method, body) + return http_json(f"https://api.github.com/repos/{libs_repository()}/{path}", headers, method, body) def github_release(tag: str) -> dict | None: + """The published release of the tag.""" return libs_api(f"releases/tags/{tag}") +def libs_releases() -> list[dict]: + """The releases of simplex-chat-libs; drafts are listed only with a token that can push to it.""" + releases: list[dict] = [] + page = 1 + while batch := libs_api(f"releases?per_page=100&page={page}"): + releases += batch + page += 1 + return releases + + +def download(tag: str, asset: str, directory: str, sha256: str) -> Path: + """Saves the asset of the release to the directory; exits when its SHA-256 differs.""" + url = f"https://github.com/{libs_repository()}/releases/download/{tag}/{asset}" + target = Path(directory) / asset + target.parent.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + request = urllib.request.Request(url, headers={"User-Agent": "simplex-chat-release"}) + with urllib.request.urlopen(request, timeout=600) as response, target.open("wb") as file: + while chunk := response.read(DOWNLOAD_CHUNK): + file.write(chunk) + digest.update(chunk) + if digest.hexdigest() != sha256: + target.unlink() + sys.exit(f"SHA-256 of {asset} of {tag} is {digest.hexdigest()}, expected {sha256}") + return target + + def read_releases() -> tuple[dict, str | None]: """The releases and the blob of the file, which a change must name.""" file = libs_api(f"contents/{RELEASES_FILE}") @@ -495,6 +638,13 @@ def commit_exists(commit: str) -> bool: return found.returncode == 0 +def fetch_commit(commit: str) -> str: + """The commit, fetched from origin when the checkout lacks it.""" + if not commit_exists(commit): + git("fetch", "-q", "--depth=1", "origin", commit) + return commit + + def npm_key(directory: str) -> str: return f"npm/{read_json(f'{directory}/package.json')['name']}" @@ -517,21 +667,113 @@ def npm_package(directory: str, releases: dict) -> Package: ) -def prepare(args: argparse.Namespace) -> None: - cabal_version = read_field("simplex-chat.cabal", r"^version:\s*(\S+)") - libs_tag = f"core-{cabal_version}" - addon = current_addon_tag() - python_version, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE)) +def types_changed(releases: dict, branch: str) -> bool: + """Whether the types sources differ from those of the previous core release on the branch.""" + previous = previous_core_commit(releases, branch) + return previous is None or changed_since(fetch_commit(previous), TYPES_SOURCES) + + +def types_problem(version: str, releases: dict, branch: str) -> str | None: + """Why the types can't be released with core , or None when they can.""" + if not types_changed(releases, branch): + return None + types = read_json(f"{TYPES_DIR}/package.json") + if recorded_commit(releases, npm_key(TYPES_DIR), types["version"]) or npm_published(types["name"], types["version"]): + return f"{types['name']} changed, and its version {types['version']} is published" + if types.get("simplexChatCore") != version: + return f"simplexChatCore of {types['name']} is {types.get('simplexChatCore')}, expected {version}" + return None + + +def claim_core(version: str, commit: str, branch: str) -> None: releases, _ = read_releases() + highest = highest_recorded(releases, core_key(branch)) + if highest and version_key(version) < version_key(highest): + sys.exit(f"core {version} is lower than core {highest} recorded for {branch}") + if recorded := recorded_core_commit(releases, version): + print(f"claim={'resume' if recorded == commit else 'skip'}") + return + if problem := types_problem(version, releases, branch): + sys.exit(problem) + libs_commit = record_releases([(core_key(branch), version)], commit) + releases, _ = read_releases() + if recorded_core_commit(releases, version) == commit: + print("claim=new") + print(f"commit={libs_commit}") + else: + print("claim=skip") + + +def current_branch() -> str: + return git("rev-parse", "--abbrev-ref", "HEAD") + + +def core_release(version: str, branch: str, types_version: str | None) -> None: + releases, _ = read_releases() + highest = highest_recorded(releases, core_key(branch)) + if highest and version_key(version) <= version_key(highest): + sys.exit(f"core {version} is not higher than core {highest} recorded for {branch}") + (ROOT / CABAL_FILE).write_text(CABAL_VERSION.sub(rf"\g<1>{version}", read_text(CABAL_FILE), count=1)) + print(f"{CABAL_FILE}: {version}") + if not types_version and not types_changed(releases, branch): + return + types = read_json(f"{TYPES_DIR}/package.json") + if not types_version: + previous = previous_core_commit(releases, branch) + released = json.loads(git("show", f"{previous}:{TYPES_DIR}/package.json")) if previous else types + types_version = bump_npm_version(released["version"]) + readme, found = TYPES_README_CORE.subn(f"SimpleX Chat core: {version}", read_text(TYPES_README), count=1) + if not found: + sys.exit(f"{TYPES_README} has no line 'SimpleX Chat core: '") + write_json(f"{TYPES_DIR}/package.json", {**types, "version": types_version, "simplexChatCore": version}) + (ROOT / TYPES_README).write_text(readme) + print(f"{types['name']}: {types_version}") + + +def pinned_tags(pins: Pins) -> list[str]: + return [tag for tag in (pins.libs_release, pins.addon_release) if tag] + + +def published_pin_problems(pins: Pins) -> list[str]: + return pin_problems(pins, {tag: github_release(tag) for tag in pinned_tags(pins)}) + + +def prepare(args: argparse.Namespace) -> None: + packages = frozenset(args.packages or ["nodejs", "python"]) + branch = args.branch or current_branch() + releases, _ = read_releases() + libs = libs_releases() + libs_tag = latest_libs(libs, recorded_versions(releases, core_key(branch))) + if libs_tag is None: + sys.exit(f"no published library release recorded for {branch}") + core = core_version(libs_tag) + if version_key(cabal_version()) > version_key(core): + print(f"warning: core-{cabal_version()} is not published, {libs_tag} is pinned", file=sys.stderr) + sources_hash = current_addon_hash() + addon = addon_release(libs, sources_hash) + addon_digests = release_digests(addon, ADDON_ASSETS) if release_status(addon) == "published" else None + if "nodejs" in packages and addon_digests is None: + sys.exit(f"the add-on release of sources {sources_hash} is not published with every asset") + types = read_json(f"{TYPES_DIR}/package.json") + types_name = types["name"] + if "nodejs" in packages and not npm_published(types_name, types["version"]): + sys.exit(f"{types_name} {types['version']} is not published") + if "python" in packages: + core_commit = recorded_commit(releases, core_key(branch), core) + if changed_since(core_commit, PYTHON_TYPES_SOURCES): + sys.exit(f"the Python types differ from those of {libs_tag}") + nodejs = read_json(f"{NODEJS_DIR}/package.json") + python_version, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE)) result = plan( State( libs_tag=libs_tag, - libs_digests=release_digests(github_release(libs_tag), LIBS_ASSETS), - addon_tag=addon, - addon_digests=release_digests(github_release(addon), ADDON_ASSETS), + libs_digests=release_digests(tag_release(libs, libs_tag), LIBS_ASSETS), + addon_tag=addon["tag_name"] if addon else "", + addon_digests=addon_digests, nodejs_pins=parse_release(read_text(RELEASE_FILE)), python_pins=python_pins, - types=npm_package(TYPES_DIR, releases), + types_dependency=f"^{types['version']}", + nodejs_types_dependency=nodejs["dependencies"][types_name], nodejs=npm_package(NODEJS_DIR, releases), python=package_state( pypi_key(), @@ -542,24 +784,35 @@ def prepare(args: argparse.Namespace) -> None: releases, lambda: pypi_published(python_version), ), - types_version=args.types_version, + packages=packages, nodejs_version=args.nodejs_version, python_version=args.python_version, ) ) - (ROOT / RELEASE_FILE).write_text(render_release(result.nodejs_pins)) - (ROOT / PYTHON_VERSION_FILE).write_text(render_python_version(result.python_version, result.python_pins)) - types = read_json(f"{TYPES_DIR}/package.json") - write_json(f"{TYPES_DIR}/package.json", {**types, "version": result.types_version}) - nodejs = read_json(f"{NODEJS_DIR}/package.json") - dependencies = nodejs["dependencies"] - if result.types_dependency: - dependencies = {**dependencies, "@simplex-chat/types": result.types_dependency} - write_json(f"{NODEJS_DIR}/package.json", {**nodejs, "version": result.nodejs_version, "dependencies": dependencies}) - summary = result.title + "\n\n" + "".join(f"- {note}\n" for note in result.notes) - if args.summary: - Path(args.summary).write_text(summary) - print(summary, end="") + selected_pins = [ + pins for name, pins in [("nodejs", result.nodejs_pins), ("python", result.python_pins)] if name in packages + ] + if problems := [problem for pins in selected_pins for problem in published_pin_problems(pins)]: + sys.exit("\n".join(problems)) + assets = { + (tag, asset, pins.sha256[asset]) + for pins in selected_pins + for tag in pinned_tags(pins) + for asset in release_assets(tag) + } + with tempfile.TemporaryDirectory() as directory: + for tag, asset, sha256 in sorted(assets): + download(tag, asset, directory, sha256) + print(f"checked {asset} of {tag}", file=sys.stderr) + if "nodejs" in packages: + (ROOT / RELEASE_FILE).write_text(render_release(result.nodejs_pins)) + dependencies = nodejs["dependencies"] + if result.types_dependency: + dependencies = {**dependencies, types_name: result.types_dependency} + write_json(f"{NODEJS_DIR}/package.json", {**nodejs, "version": result.nodejs_version, "dependencies": dependencies}) + if "python" in packages: + (ROOT / PYTHON_VERSION_FILE).write_text(render_python_version(result.python_version, result.python_pins)) + print(result.title + "\n\n" + "".join(f"- {note}\n" for note in result.notes), end="") def publish_plan() -> None: @@ -579,7 +832,7 @@ def publish_plan() -> None: def record(args: argparse.Namespace) -> None: versions = [ - ("core", args.core), + (core_key(args.branch), args.core), (npm_key(TYPES_DIR), args.types), (npm_key(NODEJS_DIR), args.nodejs), (pypi_key(), args.python), @@ -617,35 +870,118 @@ def test_names() -> None: ) -def github_bool(value: str) -> bool: - """GitHub expressions pass booleans to the shell as "true", "false" or "".""" - return value == "true" +def print_release_status(tag: str) -> None: + print(f"status={release_status(tag_release(libs_releases(), tag))}") + + +def print_addon_release() -> None: + sources_hash = current_addon_hash() + release = addon_release(libs_releases(), sources_hash) + print(f"notes={shlex.quote(addon_notes(sources_hash))}") + print(f"status={release_status(release)}") + print(f"tag={release['tag_name'] if release else ''}") + + +def publish_release(tag: str) -> None: + release = tag_release(libs_releases(), tag) + if release is None: + sys.exit(f"{tag}: no release") + if not release["draft"]: + print(f"{tag} is published") + return + if missing := missing_assets(release): + sys.exit(f"{tag}: missing {', '.join(missing)}") + libs_api(f"releases/{release['id']}", "PATCH", {"draft": False}) + print(f"{tag} published") + + +def check_pins() -> None: + _, python_pins = parse_python_version(read_text(PYTHON_VERSION_FILE)) + problems = published_pin_problems(parse_release(read_text(RELEASE_FILE))) + published_pin_problems(python_pins) + if problems: + sys.exit("\n".join(problems)) + print("the pins match the published releases") + + +def download_asset(tag: str, asset: str, directory: str) -> None: + digests = release_digests(github_release(tag), [asset]) + if digests is None: + sys.exit(f"{asset} of {tag} is not published with a digest") + download(tag, asset, directory, digests[asset]) + + +def download_pinned(asset: str, directory: str) -> None: + pins = parse_release(read_text(RELEASE_FILE)) + tag = next((tag for tag in pinned_tags(pins) if asset in release_assets(tag)), None) + if tag is None or asset not in pins.sha256: + sys.exit(f"{asset} is not pinned in {RELEASE_FILE}") + download(tag, asset, directory, pins.sha256[asset]) def main(argv: list[str]) -> None: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) commands = parser.add_subparsers(dest="command", required=True) - commands.add_parser("addon-tag") + branch_help = "the branch of the core versions, the checked out branch by default" + core_release_parser = commands.add_parser("core-release") + core_release_parser.add_argument("version", help="the new core version") + core_release_parser.add_argument("--branch", help=branch_help) + core_release_parser.add_argument("--types-version", help="this types version instead of a bump when needed") + claim_parser = commands.add_parser("claim-core") + claim_parser.add_argument("version", help="the core version of simplex-chat.cabal") + claim_parser.add_argument("commit", help="the simplex-chat commit to release it from") + claim_parser.add_argument("branch", help="the branch of the commit") + status_parser = commands.add_parser("release-status") + status_parser.add_argument("tag") + commands.add_parser("addon-release") + latest_libs_parser = commands.add_parser("latest-libs") + latest_libs_parser.add_argument("--branch", help=branch_help) + publish_release_parser = commands.add_parser("publish-release") + publish_release_parser.add_argument("tag") + commands.add_parser("check-pins") + download_parser = commands.add_parser("download") + download_parser.add_argument("tag") + download_parser.add_argument("asset") + download_parser.add_argument("directory") + download_pinned_parser = commands.add_parser("download-pinned") + download_pinned_parser.add_argument("asset") + download_pinned_parser.add_argument("directory") changes_parser = commands.add_parser("changes") changes_parser.add_argument("base", help="the commit to compare with") build_parser = commands.add_parser("build-plan") - build_parser.add_argument("--event", required=True, help="github.event_name") - build_parser.add_argument("--base", default="", help="the base commit of a pull request") - build_parser.add_argument("--release-missing", type=github_bool, default=False) - build_parser.add_argument("--libs-release", default="") + build_parser.add_argument("--base", required=True, help="the base commit of the pull request") prepare_parser = commands.add_parser("prepare") - for package in ["nodejs", "types", "python"]: + prepare_parser.add_argument("packages", nargs="*", choices=["nodejs", "python"], help="all when none is listed") + prepare_parser.add_argument("--branch", help=branch_help) + for package in ["nodejs", "python"]: prepare_parser.add_argument(f"--{package}-version", help="this version instead of a bump when needed") - prepare_parser.add_argument("--summary", help="file for the PR title and body") commands.add_parser("publish-plan") record_parser = commands.add_parser("record") record_parser.add_argument("--commit", required=True, help="the simplex-chat commit released or published from") + record_parser.add_argument("--branch", default="master", help="the branch of the core version") for name in ["core", "types", "nodejs", "python"]: record_parser.add_argument(f"--{name}", default="", help=f"{name} version, recorded when not empty") commands.add_parser("test-names") args = parser.parse_args(argv) - if args.command == "addon-tag": - print(current_addon_tag()) + if args.command == "core-release": + core_release(args.version, args.branch or current_branch(), args.types_version) + elif args.command == "claim-core": + claim_core(args.version, args.commit, args.branch) + elif args.command == "release-status": + print_release_status(args.tag) + elif args.command == "addon-release": + print_addon_release() + elif args.command == "latest-libs": + releases, _ = read_releases() + versions = recorded_versions(releases, core_key(args.branch or current_branch())) + print(f"tag={latest_libs(libs_releases(), versions) or ''}") + elif args.command == "publish-release": + publish_release(args.tag) + elif args.command == "check-pins": + check_pins() + elif args.command == "download": + download_asset(args.tag, args.asset, args.directory) + elif args.command == "download-pinned": + download_pinned(args.asset, args.directory) elif args.command == "changes": changes(args.base) elif args.command == "build-plan": diff --git a/scripts/ci/test_packages_release.py b/scripts/ci/test_packages_release.py index 1838525c74..994e678690 100644 --- a/scripts/ci/test_packages_release.py +++ b/scripts/ci/test_packages_release.py @@ -1,7 +1,10 @@ +import argparse import base64 import contextlib import email.message +import hashlib import io +import json import subprocess import tomllib import urllib.error @@ -14,26 +17,41 @@ from unittest import mock import packages_release from packages_release import ( ADDON_ASSETS, + CABAL_FILE, NODEJS_DIR, + PYTHON_DIR, PYTHON_VERSION_FILE, + RELEASE_FILE, TYPES_DIR, + TYPES_README, ADDON_PLATFORMS, BuildPlan, LIBS_ASSETS, Package, Pins, State, - addon_tag, + addon_hash, + addon_notes, + addon_release, build_matrix, build_plan, bump_npm_version, bump_python_version, changed_since, add_releases, + claim_core, + core_release, + download, + highest_recorded, + latest_libs, + missing_assets, package_state, + pin_problems, + prepare, publish_plan, record_releases, recorded_commit, + release_status, render_releases, dist_tag, npm_published, @@ -45,6 +63,8 @@ from packages_release import ( release_digests, render_python_version, render_release, + tag_release, + types_problem, ) @@ -60,6 +80,10 @@ def github_release(assets: list[str], n: int) -> dict: return {"assets": [{"name": name, "digest": f"sha256:{sha(n)}"} for name in assets]} +def listed_release(tag: str, assets: list[str], n: int, draft: bool = False, body: str | None = None) -> dict: + return {"id": n, "tag_name": tag, "draft": draft, "body": body, **github_release(assets, n)} + + def state(**overrides) -> State: current = State( libs_tag="core-1.0.0.1", @@ -68,7 +92,8 @@ def state(**overrides) -> State: addon_digests=digests(ADDON_ASSETS, 2), nodejs_pins=Pins("core-1.0.0.1", {**digests(LIBS_ASSETS, 1), **digests(ADDON_ASSETS, 2)}, "nodejs-addon-aaa"), python_pins=Pins("core-1.0.0.1", digests(LIBS_ASSETS, 1)), - types=Package("0.12.1", changed=False, published=True), + types_dependency="^0.12.1", + nodejs_types_dependency="^0.12.1", nodejs=Package("7.1.0-beta.6.1", changed=False, published=True), python=Package("7.1.0b6", changed=False, published=True), ) @@ -101,15 +126,15 @@ class VersionTest(unittest.TestCase): self.assertEqual(registry_name("simplex-chat", None), "simplex-chat") -class AddonTagTest(unittest.TestCase): +class AddonHashTest(unittest.TestCase): def test_depends_on_sources_and_node_addon_api(self): files = {"a.cc": b"1", "b.h": b"2"} - tag = addon_tag(files, "^8.5.0") - self.assertRegex(tag, r"^nodejs-addon-[0-9a-f]{12}$") - self.assertEqual(addon_tag(dict(reversed(files.items())), "^8.5.0"), tag) - self.assertNotEqual(addon_tag({**files, "b.h": b"3"}, "^8.5.0"), tag) - self.assertNotEqual(addon_tag({"a.cc": b"1", "c.h": b"2"}, "^8.5.0"), tag) - self.assertNotEqual(addon_tag(files, "^8.6.0"), tag) + sources_hash = addon_hash(files, "^8.5.0") + self.assertRegex(sources_hash, r"^[0-9a-f]{12}$") + self.assertEqual(addon_hash(dict(reversed(files.items())), "^8.5.0"), sources_hash) + self.assertNotEqual(addon_hash({**files, "b.h": b"3"}, "^8.5.0"), sources_hash) + self.assertNotEqual(addon_hash({"a.cc": b"1", "c.h": b"2"}, "^8.5.0"), sources_hash) + self.assertNotEqual(addon_hash(files, "^8.6.0"), sources_hash) class PinFilesTest(unittest.TestCase): @@ -135,7 +160,7 @@ class ReleaseDigestsTest(unittest.TestCase): class PlanTest(unittest.TestCase): def test_nothing_changes_when_current(self): result = plan(state()) - self.assertEqual((result.nodejs_changed, result.python_changed, result.types_bumped), (False, False, False)) + self.assertEqual((result.nodejs_changed, result.python_changed, result.types_dependency), (False, False, None)) self.assertEqual(result.notes, []) def test_new_libs_release_updates_both_libraries(self): @@ -170,17 +195,17 @@ class PlanTest(unittest.TestCase): plan(state(python=Package("7.1.0b6", True, True))).title, "packages: release python 7.1.0b6.post1" ) - def test_types_bump_updates_the_nodejs_dependency(self): - result = plan(state(types=Package("0.12.1", True, True))) + def test_new_types_version_updates_the_nodejs_dependency(self): + result = plan(state(types_dependency="^0.12.2")) self.assertEqual(result.types_dependency, "^0.12.2") - self.assertEqual(result.title, "packages: release types 0.12.2, nodejs 7.1.0-beta.6.2") + self.assertEqual(result.title, "packages: release nodejs 7.1.0-beta.6.2") def test_all_changes_combine_into_one_release(self): result = plan( state( **NEW_LIBS, **NEW_ADDON, - types=Package("0.12.1", True, True), + types_dependency="^0.12.2", nodejs=Package("7.1.0-beta.6.1", True, True), python=Package("7.1.0b6", True, True), ) @@ -188,7 +213,16 @@ class PlanTest(unittest.TestCase): self.assertEqual( (result.nodejs_pins.libs_release, result.nodejs_pins.addon_release), ("core-1.0.0.2", "nodejs-addon-bbb") ) - self.assertEqual(result.title, "packages: release types 0.12.2, nodejs 7.1.0-beta.6.2, python 7.1.0b6.post1") + self.assertEqual(result.title, "packages: release nodejs 7.1.0-beta.6.2, python 7.1.0b6.post1") + + def test_unlisted_package_keeps_its_pins_and_version(self): + result = plan(state(**NEW_LIBS, types_dependency="^0.12.2", packages=frozenset({"python"}))) + self.assertEqual(result.nodejs_pins, state().nodejs_pins) + self.assertEqual( + (result.nodejs_version, result.nodejs_changed, result.types_dependency), ("7.1.0-beta.6.1", False, None) + ) + self.assertEqual(result.python_pins.libs_release, "core-1.0.0.2") + self.assertEqual(result.title, "packages: release python 7.1.0b6.post1") def test_unpublished_versions_are_not_bumped_again(self): result = plan( @@ -199,29 +233,18 @@ class PlanTest(unittest.TestCase): self.assertEqual(result.title, "packages: release nodejs 7.1.0-beta.6.2, python 7.1.0b6.post1") def test_requested_versions_win(self): - result = plan(state(nodejs_version="7.2.0", types_version="0.13.0", python_version="7.2.0")) - self.assertEqual(result.types_dependency, "^0.13.0") - self.assertEqual(result.title, "packages: release types 0.13.0, nodejs 7.2.0, python 7.2.0") + result = plan(state(nodejs_version="7.2.0", python_version="7.2.0")) + self.assertEqual(result.title, "packages: release nodejs 7.2.0, python 7.2.0") class BuildPlanTest(unittest.TestCase): - def plan(self, event="push", release_missing=False, libs_release="", core=False, addon=False): - return build_plan(event, release_missing, libs_release, core, addon) - - def test_push_builds_a_missing_addon_release_and_tests_it(self): - self.assertEqual(self.plan(release_missing=True), BuildPlan(ADDON_PLATFORMS, test=True)) - self.assertEqual(self.plan(), BuildPlan([], test=True)) - def test_pull_request_tests_with_the_pinned_libs(self): - self.assertEqual(self.plan("pull_request"), BuildPlan(ADDON_PLATFORMS, test=True)) - self.assertEqual(self.plan("pull_request", addon=True), BuildPlan(ADDON_PLATFORMS, test=True)) + self.assertEqual(build_plan(core_changed=False, addon_changed=False), BuildPlan(ADDON_PLATFORMS, test=True)) + self.assertEqual(build_plan(core_changed=False, addon_changed=True), BuildPlan(ADDON_PLATFORMS, test=True)) def test_core_change_leaves_tests_to_build_yml(self): - self.assertEqual(self.plan("pull_request", core=True), BuildPlan([], test=False)) - self.assertEqual(self.plan("pull_request", core=True, addon=True), BuildPlan(ADDON_PLATFORMS, test=False)) - - def test_new_library_release_is_tested_on_every_platform(self): - self.assertEqual(self.plan(libs_release="core-1.0.0.2"), BuildPlan(ADDON_PLATFORMS, test=True)) + self.assertEqual(build_plan(core_changed=True, addon_changed=False), BuildPlan([], test=False)) + self.assertEqual(build_plan(core_changed=True, addon_changed=True), BuildPlan(ADDON_PLATFORMS, test=False)) def test_matrix_names_the_runner_of_each_platform(self): self.assertEqual( @@ -297,6 +320,12 @@ class ReleasesFileTest(unittest.TestCase): self.assertIsNone(recorded_commit(self.RELEASES, "npm/simplex-chat", "7.1.0-beta.6.4")) self.assertIsNone(recorded_commit({}, "pypi/simplex-chat", "7.1.0b6")) + def test_highest_recorded_version_is_compared_numerically(self): + releases = {"core": {"7.1.0.9": "a", "7.1.0.11": "b"}, "npm": {"@simplex-chat/types": {"0.9.0": "c"}}} + self.assertEqual(highest_recorded(releases, "core"), "7.1.0.11") + self.assertEqual(highest_recorded(releases, "npm/@simplex-chat/types"), "0.9.0") + self.assertIsNone(highest_recorded(releases, "npm/simplex-chat")) + def test_adds_entries_and_keeps_recorded_commits(self): updated = add_releases(self.RELEASES, [("npm/simplex-chat", "7.1.0-beta.6.4"), ("core", "7.1.0.11")], "x") self.assertEqual(updated["npm"]["simplex-chat"], {"7.1.0-beta.6.3": "n1", "7.1.0-beta.6.4": "x"}) @@ -355,13 +384,10 @@ class RecordReleasesTest(unittest.TestCase): self.assertEqual(record_releases([("core", "1")], "other"), "libs0") -class PackageStateTest(unittest.TestCase): +class GitRepoTest(unittest.TestCase): def setUp(self): self.repo = Path(tempfile.mkdtemp()) self.git("init", "-q") - self.write("pkg/version.txt", 'version = "1"\n') - self.write("pkg/src.txt", "a") - self.commit("set version") def git(self, *args): subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=self.repo, check=True) @@ -374,6 +400,19 @@ class PackageStateTest(unittest.TestCase): self.git("add", "-A") self.git("commit", "-q", "-m", message) + def head(self): + return subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=self.repo, capture_output=True, text=True + ).stdout.strip() + + +class PackageStateTest(GitRepoTest): + def setUp(self): + super().setUp() + self.write("pkg/version.txt", 'version = "1"\n') + self.write("pkg/src.txt", "a") + self.commit("set version") + def state(self, releases=None, registry_published=False): with mock.patch.object(packages_release, "ROOT", self.repo): return package_state( @@ -383,11 +422,6 @@ class PackageStateTest(unittest.TestCase): def changed(self): return self.state().changed - def head(self): - return subprocess.run( - ["git", "rev-parse", "HEAD"], cwd=self.repo, capture_output=True, text=True - ).stdout.strip() - def test_recorded_version_is_published_and_compared_with_its_commit(self): self.write("pkg/src.txt", "b") self.commit("change published with version 1") @@ -467,5 +501,303 @@ class PackageStateTest(unittest.TestCase): self.assertFalse(self.changed()) +class ReleasesTest(unittest.TestCase): + def test_latest_libs_is_the_highest_complete_published_version(self): + releases = [ + listed_release("core-7.1.0.9", LIBS_ASSETS, 1), + listed_release("core-7.1.0.11", LIBS_ASSETS, 2), + listed_release("core-7.1.0.12", LIBS_ASSETS[1:], 3), + listed_release("core-7.1.0.13", LIBS_ASSETS, 4, draft=True), + listed_release("core-test-7.1.0.14", LIBS_ASSETS, 5), + listed_release("nodejs-addon-1111111", ADDON_ASSETS, 6), + ] + versions = {"7.1.0.9", "7.1.0.11", "7.1.0.12", "7.1.0.13", "test-7.1.0.14"} + self.assertEqual(latest_libs(releases, versions), "core-7.1.0.11") + self.assertEqual(latest_libs(releases, {"7.1.0.9"}), "core-7.1.0.9") + self.assertIsNone(latest_libs(releases, {"7.0.0.12"})) + + def test_published_release_is_preferred_to_a_draft(self): + draft = listed_release("core-7.1.0.12", LIBS_ASSETS, 1, draft=True) + published = listed_release("core-7.1.0.12", LIBS_ASSETS, 2) + self.assertEqual(tag_release([draft, published], "core-7.1.0.12"), published) + self.assertEqual(tag_release([draft], "core-7.1.0.12"), draft) + statuses = [ + release_status(tag_release(releases, "core-7.1.0.12")) for releases in ([draft, published], [draft], []) + ] + self.assertEqual(statuses, ["published", "draft", "absent"]) + + def test_addon_release_is_found_by_its_notes(self): + other = listed_release("nodejs-addon-1111111", ADDON_ASSETS, 1, body=addon_notes("bbb")) + draft = listed_release("nodejs-addon-2222222", ADDON_ASSETS, 2, draft=True, body=addon_notes("aaa")) + published = listed_release("nodejs-addon-3333333", ADDON_ASSETS, 3, body=addon_notes("aaa")) + self.assertEqual(addon_release([other, draft, published], "aaa"), published) + self.assertEqual(addon_release([other, draft], "aaa"), draft) + self.assertIsNone(addon_release([other], "aaa")) + + def test_missing_assets_of_a_draft(self): + self.assertEqual(missing_assets(listed_release("core-7.1.0.12", LIBS_ASSETS[1:], 1)), LIBS_ASSETS[:1]) + self.assertEqual(missing_assets(listed_release("nodejs-addon-1111111", ADDON_ASSETS, 1)), []) + + def test_pin_problems_list_mismatches_and_unpublished_releases(self): + pins = Pins("core-7.1.0.11", {**digests(LIBS_ASSETS, 1), **digests(ADDON_ASSETS, 2)}, "nodejs-addon-1111111") + published = { + "core-7.1.0.11": github_release(LIBS_ASSETS, 1), + "nodejs-addon-1111111": github_release(ADDON_ASSETS, 2), + } + self.assertEqual(pin_problems(pins, published), []) + self.assertEqual( + len(pin_problems(pins, {**published, "core-7.1.0.11": github_release(LIBS_ASSETS, 3)})), len(LIBS_ASSETS) + ) + self.assertEqual( + pin_problems(pins, {**published, "nodejs-addon-1111111": None}), ["nodejs-addon-1111111 is not published"] + ) + + +class PublishReleaseTest(unittest.TestCase): + def publish(self, releases): + api = mock.Mock() + with ( + mock.patch.object(packages_release, "libs_releases", return_value=releases), + mock.patch.object(packages_release, "libs_api", api), + contextlib.redirect_stdout(io.StringIO()), + ): + packages_release.publish_release("core-7.1.0.12") + return api + + def test_complete_draft_is_published(self): + api = self.publish([listed_release("core-7.1.0.12", LIBS_ASSETS, 7, draft=True)]) + api.assert_called_once_with("releases/7", "PATCH", {"draft": False}) + + def test_draft_with_missing_assets_stays_a_draft(self): + with self.assertRaises(SystemExit): + self.publish([listed_release("core-7.1.0.12", LIBS_ASSETS[1:], 7, draft=True)]) + + def test_published_release_is_left_unchanged(self): + self.publish([listed_release("core-7.1.0.12", LIBS_ASSETS, 7)]).assert_not_called() + + +class ClaimCoreTest(unittest.TestCase): + def claim(self, recorded, after_record=None, problem=None, version="7.1.0.12", branch="master"): + reads = iter([(recorded, "blob"), (after_record or recorded, "blob")]) + self.record = mock.Mock(return_value="libs1") + output = io.StringIO() + with ( + mock.patch.object(packages_release, "read_releases", lambda: next(reads)), + mock.patch.object(packages_release, "record_releases", self.record), + mock.patch.object(packages_release, "types_problem", return_value=problem), + contextlib.redirect_stdout(output), + ): + claim_core(version, "sx1", branch) + return output.getvalue().splitlines() + + def test_lower_version_fails(self): + with self.assertRaises(SystemExit): + self.claim({"core": {"master": {"7.1.0.13": "sx0"}}}) + + def test_version_recorded_for_this_commit_resumes(self): + self.assertEqual(self.claim({"core": {"master": {"7.1.0.12": "sx1"}}}), ["claim=resume"]) + + def test_version_recorded_for_another_commit_is_skipped(self): + self.assertEqual(self.claim({"core": {"master": {"7.1.0.11": "sx0", "7.1.0.12": "sx0"}}}), ["claim=skip"]) + + def test_version_recorded_on_another_branch_is_skipped(self): + recorded = {"core": {"master": {"7.1.0.11": "sx0"}, "stable": {"7.1.0.12": "sx2"}}} + self.assertEqual(self.claim(recorded), ["claim=skip"]) + + def test_failed_types_check_fails_before_recording(self): + with self.assertRaises(SystemExit): + self.claim({"core": {"master": {"7.1.0.11": "sx0"}}}, problem="types changed") + self.record.assert_not_called() + + def test_unrecorded_version_is_claimed(self): + claimed = {"core": {"master": {"7.1.0.11": "sx0", "7.1.0.12": "sx1"}}} + self.assertEqual(self.claim({"core": {"master": {"7.1.0.11": "sx0"}}}, claimed), ["claim=new", "commit=libs1"]) + self.record.assert_called_once_with([("core/master", "7.1.0.12")], "sx1") + + def test_stable_versions_are_compared_with_stable_only(self): + recorded = {"core": {"master": {"7.1.0.11": "sx0"}, "stable": {"7.0.0.12": "sx3"}}} + claimed = {"core": {**recorded["core"], "stable": {"7.0.0.12": "sx3", "7.0.0.13": "sx1"}}} + self.assertEqual( + self.claim(recorded, claimed, version="7.0.0.13", branch="stable"), ["claim=new", "commit=libs1"] + ) + self.record.assert_called_once_with([("core/stable", "7.0.0.13")], "sx1") + with self.assertRaises(SystemExit): + self.claim(recorded, version="7.0.0.11", branch="stable") + + def test_concurrent_claim_of_another_commit_is_skipped(self): + claimed = {"core": {"master": {"7.1.0.11": "sx0", "7.1.0.12": "sx2"}}} + self.assertEqual(self.claim({"core": {"master": {"7.1.0.11": "sx0"}}}, claimed), ["claim=skip"]) + + +class TypesProblemTest(unittest.TestCase): + def problem(self, types, changed=True, published=False, releases=None): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + root = Path(directory.name) + (root / TYPES_DIR).mkdir(parents=True) + (root / TYPES_DIR / "package.json").write_text(json.dumps({"name": "@simplex-chat/types", **types})) + with ( + mock.patch.object(packages_release, "ROOT", root), + mock.patch.object(packages_release, "types_changed", return_value=changed), + mock.patch.object(packages_release, "npm_published", return_value=published), + ): + return types_problem("7.1.0.12", releases or {}, "master") + + def test_unchanged_types_pass(self): + self.assertIsNone(self.problem({"version": "0.12.1", "simplexChatCore": "7.1.0.11"}, changed=False)) + + def test_changed_types_with_a_new_version_for_the_core_pass(self): + self.assertIsNone(self.problem({"version": "0.12.2", "simplexChatCore": "7.1.0.12"})) + + def test_changed_types_with_a_published_version_fail(self): + self.assertIsNotNone(self.problem({"version": "0.12.1", "simplexChatCore": "7.1.0.12"}, published=True)) + recorded = {"npm": {"@simplex-chat/types": {"0.12.1": "c"}}} + self.assertIsNotNone(self.problem({"version": "0.12.1", "simplexChatCore": "7.1.0.12"}, releases=recorded)) + + def test_changed_types_for_another_core_fail(self): + self.assertIsNotNone(self.problem({"version": "0.12.2", "simplexChatCore": "7.1.0.11"})) + + +class CoreReleaseTest(GitRepoTest): + def setUp(self): + super().setUp() + self.write(CABAL_FILE, "name: simplex-chat\nversion: 7.1.0.11\n") + types = {"name": "@simplex-chat/types", "version": "0.12.1", "simplexChatCore": "7.1.0.11"} + self.write(f"{TYPES_DIR}/package.json", json.dumps(types, indent=2) + "\n") + self.write(TYPES_README, "# Types\n\nSimpleX Chat core: 7.1.0.11\n") + self.write(f"{TYPES_DIR}/src/types.ts", "a") + self.commit("types 0.12.1") + self.releases = {"core": {"master": {"7.1.0.11": self.head()}}} + + def release(self, version, types_version=None): + with ( + mock.patch.object(packages_release, "ROOT", self.repo), + mock.patch.object(packages_release, "read_releases", return_value=(self.releases, "blob")), + contextlib.redirect_stdout(io.StringIO()), + ): + core_release(version, "master", types_version) + + def read(self, file): + return (self.repo / file).read_text() + + def types(self): + return json.loads(self.read(f"{TYPES_DIR}/package.json")) + + def test_changed_types_get_a_new_version_for_the_core(self): + self.write(f"{TYPES_DIR}/src/types.ts", "b") + self.commit("types change") + self.release("7.1.0.12") + self.assertIn("version: 7.1.0.12\n", self.read(CABAL_FILE)) + self.assertEqual((self.types()["version"], self.types()["simplexChatCore"]), ("0.12.2", "7.1.0.12")) + self.assertIn("SimpleX Chat core: 7.1.0.12\n", self.read(TYPES_README)) + + def test_unchanged_types_keep_their_version(self): + self.release("7.1.0.12") + self.assertIn("version: 7.1.0.12\n", self.read(CABAL_FILE)) + self.assertEqual((self.types()["version"], self.types()["simplexChatCore"]), ("0.12.1", "7.1.0.11")) + + def test_requested_types_version_wins(self): + self.release("7.1.0.12", "7.1.0-beta.6.1") + self.assertEqual((self.types()["version"], self.types()["simplexChatCore"]), ("7.1.0-beta.6.1", "7.1.0.12")) + + def test_types_version_is_bumped_from_the_previous_core_release(self): + self.write(f"{TYPES_DIR}/package.json", json.dumps({**self.types(), "version": "0.12.5"})) + self.write(f"{TYPES_DIR}/src/types.ts", "b") + self.commit("types change and a manual version") + self.release("7.1.0.12") + self.assertEqual(self.types()["version"], "0.12.2") + + def test_version_must_be_higher_than_the_recorded_one(self): + with self.assertRaises(SystemExit): + self.release("7.1.0.11") + self.assertIn("version: 7.1.0.11\n", self.read(CABAL_FILE)) + + +class PrepareTest(GitRepoTest): + def setUp(self): + super().setUp() + nodejs = { + "name": "simplex-chat", + "version": "7.1.0-beta.6.1", + "dependencies": {"@simplex-chat/types": "^0.12.1"}, + "devDependencies": {"node-addon-api": "^8.5.0"}, + } + self.write(CABAL_FILE, "version: 1.0.0.2\n") + self.write(f"{NODEJS_DIR}/binding.gyp", "{}") + self.write(f"{NODEJS_DIR}/cpp/simplex.cc", "add-on") + self.write(f"{NODEJS_DIR}/package.json", json.dumps(nodejs)) + self.write(RELEASE_FILE, render_release(state().nodejs_pins)) + self.write(f"{TYPES_DIR}/package.json", json.dumps({"name": "@simplex-chat/types", "version": "0.12.1"})) + self.write(PYTHON_VERSION_FILE, render_python_version("7.1.0b6", state().python_pins)) + self.write(f"{PYTHON_DIR}/src/simplex_chat/types/_types.py", "a") + self.write(f"{PYTHON_DIR}/pyproject.toml", 'name = "simplex-chat"\n') + self.commit("packages") + with mock.patch.object(packages_release, "ROOT", self.repo): + sources_hash = packages_release.current_addon_hash() + self.libs = [ + listed_release("core-1.0.0.2", LIBS_ASSETS, 3), + listed_release("nodejs-addon-1111111", ADDON_ASSETS, 4, body=addon_notes(sources_hash)), + ] + self.releases = { + "core": {"master": {"1.0.0.2": self.head()}}, + "npm": {"simplex-chat": {"7.1.0-beta.6.1": self.head()}}, + "pypi": {"simplex-chat": {"7.1.0b6": self.head()}}, + } + + def prepare(self, *packages): + published = {release["tag_name"]: release for release in self.libs} + download = mock.Mock() + with ( + mock.patch.object(packages_release, "ROOT", self.repo), + mock.patch.object(packages_release, "read_releases", return_value=(self.releases, "blob")), + mock.patch.object(packages_release, "libs_releases", return_value=self.libs), + mock.patch.object(packages_release, "github_release", published.get), + mock.patch.object(packages_release, "npm_published", return_value=True), + mock.patch.object(packages_release, "pypi_published", return_value=True), + mock.patch.object(packages_release, "download", download), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + prepare( + argparse.Namespace(packages=list(packages), branch="master", nodejs_version=None, python_version=None) + ) + return download + + def test_listed_package_is_released_alone(self): + download = self.prepare("nodejs") + pins = parse_release((self.repo / RELEASE_FILE).read_text()) + self.assertEqual((pins.libs_release, pins.addon_release), ("core-1.0.0.2", "nodejs-addon-1111111")) + self.assertEqual(json.loads((self.repo / NODEJS_DIR / "package.json").read_text())["version"], "7.1.0-beta.6.2") + python = parse_python_version((self.repo / PYTHON_VERSION_FILE).read_text()) + self.assertEqual(python, ("7.1.0b6", state().python_pins)) + self.assertEqual(download.call_count, len(LIBS_ASSETS) + len(ADDON_ASSETS)) + + def test_python_types_newer_than_the_pinned_core_stop_the_release(self): + self.write(f"{PYTHON_DIR}/src/simplex_chat/types/_types.py", "b") + self.commit("python types change") + with self.assertRaises(SystemExit): + self.prepare("python") + self.assertEqual(parse_python_version((self.repo / PYTHON_VERSION_FILE).read_text())[0], "7.1.0b6") + + +class DownloadTest(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.directory = directory.name + + def download(self, sha256): + with mock.patch.object(packages_release.urllib.request, "urlopen", return_value=io.BytesIO(b"data")): + return download("core-1.0.0.1", "a.zip", self.directory, sha256) + + def test_matching_digest_keeps_the_file(self): + self.assertEqual(self.download(hashlib.sha256(b"data").hexdigest()).read_bytes(), b"data") + + def test_mismatching_digest_removes_the_file(self): + with self.assertRaises(SystemExit): + self.download(sha(1)) + self.assertFalse((Path(self.directory) / "a.zip").exists()) + + if __name__ == "__main__": unittest.main() diff --git a/simplex-chat.cabal b/simplex-chat.cabal index 16ab1fcb59..747e32db2e 100644 --- a/simplex-chat.cabal +++ b/simplex-chat.cabal @@ -5,7 +5,7 @@ cabal-version: 1.12 -- see: https://github.com/sol/hpack name: simplex-chat -version: 7.1.0.11 +version: 7.1.0.12 category: Web, System, Services, Cryptography homepage: https://github.com/simplex-chat/simplex-chat#readme author: simplex.chat