mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-10-20 08:02:24 +00:00
nodejs: lazy per-backend libsimplex download (#7543)
* ci: drop dlltool from nodejs libs release * nodejs: download libsimplex into the user cache * nodejs: load libsimplex at runtime per backend * support bot: select db backend with --db * python: match backend switch error text * nodejs: download prebuilt add-on at runtime * ci: build nodejs add-on for libs release * calculator bot: load libs and add-on in image * support bot: document add-on override * nodejs: keep worker test output free of colors * nodejs: verify sha-256 of downloaded assets * python: pin core libs release with sha-256 * ci: add packages release script * ci: release nodejs add-on and open release PR * ci: publish packages after release PR * nodejs: keep npm from building the add-on * calculator bot: build the add-on in the image * ci: merge package release workflows * ci: open release PR when publishing fails * ci: build the add-on with node-addon-api only * ci: skip publishing in a called packages workflow * ci: update package release actions to Node 24 * nodejs: fix receive test timeout * ci: test packages on pull requests * nodejs: reuse the database in the close stress test * ci: keep failed package tests from hanging * ci: skip preset server tests * ci: plan package builds from the changes * ci: test packages with a given libsimplex * ci: test packages with libsimplex of core changes
This commit is contained in:
sh
@@ -80,13 +80,23 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
libs_tag: ${{ steps.libs_commit.outputs.tag }}
|
||||
core_changed: ${{ steps.core.outputs.core }}
|
||||
steps:
|
||||
- name: Clone project
|
||||
if: github.event_name == 'push'
|
||||
if: github.event_name == 'push' || github.event_name == 'pull_request'
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Check chat core change
|
||||
id: core
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
BASE: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
git fetch -q --depth=1 origin "$BASE"
|
||||
python3 scripts/ci/packages_release.py changes "$BASE" | grep '^core=' >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build changelog
|
||||
id: build_changelog
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
@@ -300,8 +310,9 @@ jobs:
|
||||
done
|
||||
strip /out/simplex-chat
|
||||
|
||||
# Also for pull requests that change the chat core, for the package tests (packages-pr).
|
||||
- name: Build library
|
||||
if: needs.maybe-release.outputs.libs_tag && matrix.libs
|
||||
if: matrix.libs && (needs.maybe-release.outputs.libs_tag || (needs.maybe-release.outputs.core_changed && matrix.arch == 'x86_64'))
|
||||
shell: docker exec -t builder sh -eu {0}
|
||||
run: |
|
||||
SKIP_VLC=1 scripts/desktop/build-lib-linux.sh
|
||||
@@ -310,6 +321,14 @@ jobs:
|
||||
find apps/multiplatform/common/src/commonMain/cpp/desktop/libs/linux-${{ matrix.arch }} -maxdepth 1 -type f -exec cp {} release-libs/$name/libs/ \;
|
||||
(cd release-libs/$name && zip -r ../$name.zip libs)
|
||||
|
||||
- name: Upload library artifact
|
||||
if: matrix.libs && needs.maybe-release.outputs.core_changed && matrix.arch == 'x86_64'
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: simplex-chat-libs-linux-${{ matrix.arch }}
|
||||
path: release-libs/simplex-chat-libs-linux-${{ matrix.arch }}.zip
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload library
|
||||
if: needs.maybe-release.outputs.libs_tag && matrix.libs
|
||||
uses: softprops/action-gh-release@v2
|
||||
@@ -837,3 +856,34 @@ jobs:
|
||||
bin_hash: ${{ steps.windows_desktop_build.outputs.package_hash }}
|
||||
github_ref: ${{ github.ref }}
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# =========================
|
||||
# Packages release
|
||||
# =========================
|
||||
|
||||
# A new library release is tested with the Node.js and Python packages and updates them in a release PR.
|
||||
|
||||
packages:
|
||||
needs: [maybe-release, build-linux, build-linux-postgres, build-macos, build-windows]
|
||||
if: ${{ !cancelled() && needs.maybe-release.outputs.libs_tag }}
|
||||
# the called workflow can't be granted more than its caller, and its publish job needs these
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
uses: ./.github/workflows/packages.yml
|
||||
with:
|
||||
libs_release: ${{ needs.maybe-release.outputs.libs_tag }}
|
||||
secrets: inherit
|
||||
|
||||
# A pull request that changes the chat core is tested with the Node.js and Python packages.
|
||||
|
||||
packages-pr:
|
||||
needs: [maybe-release, build-linux]
|
||||
# build-linux uploads the library before its chat tests run
|
||||
if: ${{ !cancelled() && needs.maybe-release.outputs.core_changed }}
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
uses: ./.github/workflows/packages.yml
|
||||
with:
|
||||
libs_artifact: true
|
||||
|
||||
@@ -0,0 +1,354 @@
|
||||
name: packages
|
||||
|
||||
# Releases simplex-chat (npm, PyPI) and @simplex-chat/types (npm) from master and stable:
|
||||
# 1. A missing Node.js add-on release (nodejs-addon-<hash of its sources>) is built and uploaded to simplex-chat-libs.
|
||||
# 2. Versions that are not published yet are published, and tagged <package>-v<version> on their commit.
|
||||
# npm prerelease versions get dist-tag beta, others latest.
|
||||
# 3. The release PR packages-release/<branch> pins new library and add-on releases with their SHA-256
|
||||
# and bumps the versions of changed packages. Merging it publishes them (step 2).
|
||||
# Package tests run on every platform with the pinned libsimplex. build.yml runs them with the libsimplex it builds:
|
||||
# for pull requests that change the chat core (linux-x86_64), and for a new library release before its release PR.
|
||||
|
||||
# Publishing uses trusted publishing (OIDC) for this workflow on npm and PyPI,
|
||||
# or the NPM_TOKEN and PYPI_API_TOKEN secrets when they are set.
|
||||
# Test repositories: variables NPM_SCOPE, PYPI_PROJECT and PYPI_URL / PYPI_UPLOAD_URL (TestPyPI)
|
||||
# publish under other names.
|
||||
|
||||
# Secrets:
|
||||
# -------
|
||||
# NODEJS_REPO_TOKEN
|
||||
# Only select repositories: simplex-chat-libs
|
||||
# Permissions:
|
||||
# * Contents (Read and Write)
|
||||
# RELEASE_PR_TOKEN
|
||||
# Only select repositories: simplex-chat
|
||||
# Permissions:
|
||||
# * Contents (Read and Write)
|
||||
# * Pull requests (Read and Write)
|
||||
# A PR opened with GITHUB_TOKEN does not run the checks of other workflows.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
nodejs_version:
|
||||
description: "simplex-chat npm version, bumped when needed by default"
|
||||
required: false
|
||||
types_version:
|
||||
description: "@simplex-chat/types version, bumped when needed by default"
|
||||
required: false
|
||||
python_version:
|
||||
description: "simplex-chat PyPI version, bumped when needed by default"
|
||||
required: false
|
||||
# build.yml: a pull request that changes the chat core, or a new library release
|
||||
workflow_call:
|
||||
inputs:
|
||||
libs_artifact:
|
||||
description: "test with simplex-chat-libs-linux-x86_64 built by the calling run"
|
||||
type: boolean
|
||||
default: false
|
||||
libs_release:
|
||||
description: "test with this simplex-chat-libs release"
|
||||
type: string
|
||||
default: ""
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- stable
|
||||
paths:
|
||||
- "packages/simplex-chat-nodejs/**"
|
||||
- "packages/simplex-chat-client/types/typescript/**"
|
||||
- "packages/simplex-chat-python/**"
|
||||
- ".github/workflows/packages.yml"
|
||||
pull_request:
|
||||
paths:
|
||||
- "packages/simplex-chat-nodejs/**"
|
||||
- "packages/simplex-chat-client/types/typescript/**"
|
||||
- "packages/simplex-chat-python/**"
|
||||
- ".github/workflows/packages.yml"
|
||||
|
||||
concurrency:
|
||||
# build.yml and this workflow both run for a pull request that changes the core and the packages
|
||||
group: ${{ github.workflow }}-packages-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
NPM_SCOPE: ${{ vars.NPM_SCOPE }}
|
||||
PYPI_PROJECT: ${{ vars.PYPI_PROJECT }}
|
||||
PYPI_URL: ${{ vars.PYPI_URL }}
|
||||
|
||||
jobs:
|
||||
addon:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
tag: ${{ steps.tag.outputs.tag }}
|
||||
release_branch: ${{ steps.branch.outputs.release_branch }}
|
||||
missing: ${{ steps.release.outputs.missing }}
|
||||
platforms: ${{ steps.plan.outputs.platforms }}
|
||||
test: ${{ steps.plan.outputs.test }}
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Add-on release tag
|
||||
id: tag
|
||||
run: echo "tag=$(python3 scripts/ci/packages_release.py addon-tag)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Release branch
|
||||
id: branch
|
||||
if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/stable')
|
||||
run: echo "release_branch=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Check the add-on release
|
||||
id: release
|
||||
if: steps.branch.outputs.release_branch
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }}
|
||||
LIBS_REPOSITORY: ${{ github.repository }}-libs
|
||||
TAG: ${{ steps.tag.outputs.tag }}
|
||||
run: |
|
||||
if ! gh release view "$TAG" --repo "$LIBS_REPOSITORY" > /dev/null 2>&1; then
|
||||
echo "missing=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build plan
|
||||
id: plan
|
||||
env:
|
||||
EVENT: ${{ github.event_name }}
|
||||
BASE: ${{ github.event.pull_request.base.sha }}
|
||||
RELEASE_MISSING: ${{ steps.release.outputs.missing }}
|
||||
LIBS_ARTIFACT: ${{ inputs.libs_artifact }}
|
||||
LIBS_RELEASE: ${{ inputs.libs_release }}
|
||||
run: |
|
||||
if [ -n "$BASE" ]; then git fetch -q --depth=1 origin "$BASE"; fi
|
||||
python3 scripts/ci/packages_release.py build-plan --event "$EVENT" --base "$BASE" \
|
||||
--release-missing "$RELEASE_MISSING" --libs-artifact "$LIBS_ARTIFACT" --libs-release "$LIBS_RELEASE" >> "$GITHUB_OUTPUT"
|
||||
|
||||
build:
|
||||
name: "build ${{ matrix.platform }}"
|
||||
needs: addon
|
||||
if: needs.addon.outputs.platforms != '[]'
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include: ${{ fromJSON(needs.addon.outputs.platforms) }}
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
working-directory: packages/simplex-chat-nodejs
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
# The local types: the dependency can be bumped to a version that is not published yet.
|
||||
- name: Build simplex-chat (npm) and add-on
|
||||
run: |
|
||||
(cd ../simplex-chat-client/types/typescript && npm install --no-audit --no-fund && npx tsc)
|
||||
npm install --ignore-scripts --no-save --no-audit --no-fund ../simplex-chat-client/types/typescript
|
||||
npx --yes node-gyp rebuild
|
||||
npx tsc && cp src/simplex.* dist
|
||||
cp build/Release/simplex.node build/simplex-chat-nodejs-${{ matrix.platform }}.node
|
||||
|
||||
- name: Download libsimplex of this run
|
||||
if: needs.addon.outputs.test && inputs.libs_artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: simplex-chat-libs-${{ matrix.platform }}
|
||||
path: ${{ runner.temp }}/libsimplex
|
||||
|
||||
- name: Download libsimplex release
|
||||
if: needs.addon.outputs.test && inputs.libs_release
|
||||
env:
|
||||
URL: https://github.com/${{ github.repository }}-libs/releases/download/${{ inputs.libs_release }}/simplex-chat-libs-${{ matrix.platform }}.zip
|
||||
run: curl -fsSL --create-dirs -o "$RUNNER_TEMP/libsimplex/simplex-chat-libs-${{ matrix.platform }}.zip" "$URL"
|
||||
|
||||
- name: Unpack libsimplex
|
||||
if: needs.addon.outputs.test && (inputs.libs_artifact || inputs.libs_release)
|
||||
run: python -m zipfile -e "$RUNNER_TEMP/libsimplex/simplex-chat-libs-${{ matrix.platform }}.zip" "$RUNNER_TEMP/libsimplex"
|
||||
|
||||
- name: Test simplex-chat (npm)
|
||||
if: needs.addon.outputs.test
|
||||
env:
|
||||
SIMPLEX_LIBS_DIR: ${{ (inputs.libs_artifact || inputs.libs_release) && format('{0}/libsimplex/libs', runner.temp) || '' }}
|
||||
# A failed test can leave chat controllers running, and jest then waits for them.
|
||||
# Tests on the public preset servers fail in CI when a server is slow to connect.
|
||||
run: npx jest --forceExit --testNamePattern '^(?!.*use preset servers)'
|
||||
|
||||
- name: Test simplex-chat (PyPI)
|
||||
if: needs.addon.outputs.test
|
||||
working-directory: packages/simplex-chat-python
|
||||
run: |
|
||||
python -m pip install -e ".[test]"
|
||||
python -m pytest -q
|
||||
|
||||
- name: Upload add-on artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: simplex-nodejs-addon-${{ matrix.platform }}
|
||||
path: packages/simplex-chat-nodejs/build/simplex-chat-nodejs-${{ matrix.platform }}.node
|
||||
if-no-files-found: error
|
||||
|
||||
publish:
|
||||
needs: addon
|
||||
# Trusted publishing accepts this workflow only when it is not called by another one (build.yml).
|
||||
if: needs.addon.outputs.release_branch && contains(github.workflow_ref, '/.github/workflows/packages.yml@')
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Versions to publish
|
||||
id: plan
|
||||
run: python3 scripts/ci/packages_release.py publish-plan >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup Node.js
|
||||
if: steps.plan.outputs.types_version || steps.plan.outputs.nodejs_version
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: 24
|
||||
registry-url: https://registry.npmjs.org
|
||||
|
||||
- name: Setup Python
|
||||
if: steps.plan.outputs.python_version
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
# Before the test names: the build installs @simplex-chat/types from this repository.
|
||||
- name: Build and test simplex-chat (npm)
|
||||
if: steps.plan.outputs.nodejs_version
|
||||
working-directory: packages/simplex-chat-nodejs
|
||||
run: |
|
||||
(cd ../simplex-chat-client/types/typescript && npm install --no-audit --no-fund && npx tsc)
|
||||
npm install --ignore-scripts --no-save --no-audit --no-fund ../simplex-chat-client/types/typescript
|
||||
npx tsc && cp src/simplex.* dist
|
||||
npx jest tests/libs.test.ts tests/api.unit.test.ts
|
||||
|
||||
- name: Use test package names
|
||||
if: vars.NPM_SCOPE || vars.PYPI_PROJECT
|
||||
run: python3 scripts/ci/packages_release.py test-names
|
||||
|
||||
- name: Publish @simplex-chat/types
|
||||
if: steps.plan.outputs.types_version
|
||||
working-directory: packages/simplex-chat-client/types/typescript
|
||||
env:
|
||||
VERSION: ${{ steps.plan.outputs.types_version }}
|
||||
DIST_TAG: ${{ steps.plan.outputs.types_dist_tag }}
|
||||
run: |
|
||||
npm install --no-audit --no-fund
|
||||
npm publish --provenance --access public --tag "$DIST_TAG"
|
||||
git tag "types-v$VERSION" && git push origin "types-v$VERSION"
|
||||
|
||||
- name: Publish simplex-chat (npm)
|
||||
if: steps.plan.outputs.nodejs_version
|
||||
working-directory: packages/simplex-chat-nodejs
|
||||
env:
|
||||
VERSION: ${{ steps.plan.outputs.nodejs_version }}
|
||||
DIST_TAG: ${{ steps.plan.outputs.nodejs_dist_tag }}
|
||||
run: |
|
||||
npm publish --provenance --access public --tag "$DIST_TAG"
|
||||
git tag "nodejs-v$VERSION" && git push origin "nodejs-v$VERSION"
|
||||
|
||||
- name: Build and test simplex-chat (PyPI)
|
||||
if: steps.plan.outputs.python_version
|
||||
working-directory: packages/simplex-chat-python
|
||||
run: |
|
||||
python -m pip install build -e ".[test]"
|
||||
python -m pytest -q
|
||||
python -m build
|
||||
|
||||
- name: Publish simplex-chat (PyPI)
|
||||
if: steps.plan.outputs.python_version
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
packages-dir: packages/simplex-chat-python/dist
|
||||
repository-url: ${{ vars.PYPI_UPLOAD_URL || 'https://upload.pypi.org/legacy/' }}
|
||||
password: ${{ secrets.PYPI_API_TOKEN }}
|
||||
|
||||
- name: Tag the published PyPI version
|
||||
if: steps.plan.outputs.python_version
|
||||
env:
|
||||
VERSION: ${{ steps.plan.outputs.python_version }}
|
||||
run: git tag "python-v$VERSION" && git push origin "python-v$VERSION"
|
||||
|
||||
release:
|
||||
# After publish, for its version tags. A failed publish leaves its versions unbumped.
|
||||
needs: [addon, build, publish]
|
||||
# A merged release PR changes only versions and pins.
|
||||
if: >-
|
||||
!cancelled() && needs.addon.outputs.release_branch && needs.build.result != 'failure'
|
||||
&& !startsWith(github.event.head_commit.message, 'packages: release')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Outside the checkout: the release PR commits every change in it.
|
||||
- name: Download add-on artifacts
|
||||
if: needs.addon.outputs.missing
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: simplex-nodejs-addon-*
|
||||
merge-multiple: true
|
||||
path: ${{ runner.temp }}/addons
|
||||
|
||||
- name: Create add-on release
|
||||
if: needs.addon.outputs.missing
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
repository: ${{ github.repository }}-libs
|
||||
tag_name: ${{ needs.addon.outputs.tag }}
|
||||
files: ${{ runner.temp }}/addons/*.node
|
||||
token: ${{ secrets.NODEJS_REPO_TOKEN }}
|
||||
|
||||
- name: Update pins and versions
|
||||
id: prepare
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.NODEJS_REPO_TOKEN }}
|
||||
LIBS_REPOSITORY: ${{ github.repository }}-libs
|
||||
NODEJS_VERSION: ${{ inputs.nodejs_version }}
|
||||
TYPES_VERSION: ${{ inputs.types_version }}
|
||||
PYTHON_VERSION: ${{ inputs.python_version }}
|
||||
run: |
|
||||
python3 scripts/ci/packages_release.py prepare \
|
||||
--nodejs-version "$NODEJS_VERSION" \
|
||||
--types-version "$TYPES_VERSION" \
|
||||
--python-version "$PYTHON_VERSION" \
|
||||
--summary "$RUNNER_TEMP/summary.md"
|
||||
echo "title=$(head -n 1 "$RUNNER_TEMP/summary.md")" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create or update release PR
|
||||
uses: peter-evans/create-pull-request@v8
|
||||
with:
|
||||
token: ${{ secrets.RELEASE_PR_TOKEN }}
|
||||
branch: packages-release/${{ github.ref_name }}
|
||||
base: ${{ github.ref_name }}
|
||||
title: ${{ steps.prepare.outputs.title }}
|
||||
commit-message: ${{ steps.prepare.outputs.title }}
|
||||
body-path: ${{ runner.temp }}/summary.md
|
||||
author: SimpleX Chat <build@simplex.chat>
|
||||
committer: SimpleX Chat <build@simplex.chat>
|
||||
delete-branch: true
|
||||
Reference in New Issue
Block a user