From 40dfa65589a8ddeef7b086f8feadaa6212763fba Mon Sep 17 00:00:00 2001 From: Ivan Date: Sun, 3 May 2026 16:06:56 -0500 Subject: [PATCH] feat(ci): add pypi publish workflow and add macos debug script --- .github/workflows/build-release.yml | 47 ++-- .github/workflows/build.yml | 31 +-- .github/workflows/pypi.yml | 222 ++++++++++++++++++ .../ci/github-ensure-macos-x86-64-homebrew.sh | 20 ++ scripts/ci/github-slsa-hashes-dist.sh | 30 +++ 5 files changed, 304 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/pypi.yml create mode 100755 scripts/ci/github-ensure-macos-x86-64-homebrew.sh create mode 100644 scripts/ci/github-slsa-hashes-dist.sh diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 740a8f0..dffa82d 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -353,6 +353,10 @@ jobs: if: matrix.label == 'macos' run: /usr/sbin/softwareupdate --install-rosetta --agree-to-license || true + - name: Ensure x86_64 Homebrew (/usr/local) for universal slice + if: matrix.label == 'macos' + run: bash scripts/ci/github-ensure-macos-x86-64-homebrew.sh + - name: Set up Python x64 for cx_Freeze universal slice id: python_x64 if: matrix.label == 'macos' @@ -370,32 +374,24 @@ jobs: if: matrix.label == 'macos' run: | set -euo pipefail - # The Rosetta (x86_64) Homebrew lives at /usr/local/bin/brew on - # Apple Silicon GitHub runners. libyaml must be available as - # x86_64 so PyYAML's C extension (_yaml.cpython-3xx-darwin.so) - # compiles for the darwin-x64 cx_Freeze slice. Without this, - # pip falls back to pure Python pyyaml, which is absent from - # the x64 tree while the arm64 tree has the binary, causing - # unify-backend-plain-files.sh to fail. - if [[ -x /usr/local/bin/brew ]]; then - arch -x86_64 /usr/local/bin/brew install libyaml || true - else - echo "x86_64 Homebrew not found at /usr/local/bin/brew; PyYAML C extension may fall back to pure Python for x64 slice." - fi + # x86_64 libyaml for the darwin-x64 cx_Freeze slice (see + # github-ensure-macos-x86-64-homebrew.sh; /usr/local/bin/brew is + # guaranteed before this step on Apple Silicon runners). + arch -x86_64 /usr/local/bin/brew install libyaml || true - name: Install x86_64 codec2 for pycodec2 (universal slice) if: matrix.label == 'macos' run: | set -euo pipefail - if [[ -x /usr/local/bin/brew ]]; then - arch -x86_64 /usr/local/bin/brew install codec2 - else - echo "x86_64 Homebrew not found at /usr/local/bin/brew; cannot build pycodec2 x64 slice." >&2 - exit 1 - fi + arch -x86_64 /usr/local/bin/brew install codec2 _codec2="$(arch -x86_64 /usr/local/bin/brew --prefix codec2)" test -f "${_codec2}/include/codec2/codec2.h" - test -f "${_codec2}/lib/libcodec2.dylib" + shopt -s nullglob + _libs=("${_codec2}/lib"/libcodec2*.dylib) + if [[ ${#_libs[@]} -eq 0 ]]; then + echo "no libcodec2*.dylib under ${_codec2}/lib" >&2 + exit 1 + fi - name: Install project deps into x64 Python (mac universal cx_Freeze) if: matrix.label == 'macos' @@ -407,15 +403,10 @@ jobs: CFLAGS: "-arch x86_64" run: | set -euo pipefail - if [[ -x /usr/local/bin/brew ]]; then - _codec2="$(arch -x86_64 /usr/local/bin/brew --prefix codec2)" - export LDFLAGS="-L${_codec2}/lib -arch x86_64" - export CPPFLAGS="-I${_codec2}/include" - export PKG_CONFIG_PATH="${_codec2}/lib/pkgconfig:/usr/local/lib/pkgconfig:/usr/local/share/pkgconfig" - else - export LDFLAGS="-arch x86_64" - export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:/usr/local/share/pkgconfig" - fi + _codec2="$(arch -x86_64 /usr/local/bin/brew --prefix codec2)" + export LDFLAGS="-L${_codec2}/lib -arch x86_64" + export CPPFLAGS="-I${_codec2}/include" + export PKG_CONFIG_PATH="${_codec2}/lib/pkgconfig:/usr/local/lib/pkgconfig:/usr/local/share/pkgconfig" arch -x86_64 "$PY_X64" -m pip install -U pip setuptools wheel arch -x86_64 "$PY_X64" -m pip install "cx-freeze>=7.0.0" arch -x86_64 "$PY_X64" -m pip install -e . diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bb12955..036ea05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -102,6 +102,10 @@ jobs: if: matrix.label == 'macos' run: /usr/sbin/softwareupdate --install-rosetta --agree-to-license || true + - name: Ensure x86_64 Homebrew (/usr/local) for universal slice + if: matrix.label == 'macos' + run: bash scripts/ci/github-ensure-macos-x86-64-homebrew.sh + - name: Set up Python x64 for cx_Freeze universal slice id: python_x64 if: matrix.label == 'macos' @@ -119,11 +123,7 @@ jobs: if: matrix.label == 'macos' run: | set -euo pipefail - if [[ -x /usr/local/bin/brew ]]; then - arch -x86_64 /usr/local/bin/brew install codec2 - else - echo "x86_64 Homebrew not found at /usr/local/bin/brew; pycodec2 x64 slice build may fail." >&2 - fi + arch -x86_64 /usr/local/bin/brew install codec2 - name: Install project deps into x64 Python (mac universal cx_Freeze) if: matrix.label == 'macos' @@ -135,19 +135,14 @@ jobs: CFLAGS: "-arch x86_64" run: | set -euo pipefail - if [[ -x /usr/local/bin/brew ]]; then - _codec2="$(arch -x86_64 /usr/local/bin/brew --prefix codec2)" - export LDFLAGS="-L${_codec2}/lib -arch x86_64" - export CPPFLAGS="-I${_codec2}/include" - export PKG_CONFIG_PATH="${_codec2}/lib/pkgconfig:/usr/local/lib/pkgconfig:/usr/local/share/pkgconfig" - else - export LDFLAGS="-arch x86_64" - export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:/usr/local/share/pkgconfig" - fi - "$PY_X64" -m pip install -U pip setuptools wheel - "$PY_X64" -m pip install "cx-freeze>=7.0.0" - "$PY_X64" -m pip install -e . - "$PY_X64" scripts/patch_lxst_pyogg_ogg_ctypes.py + _codec2="$(arch -x86_64 /usr/local/bin/brew --prefix codec2)" + export LDFLAGS="-L${_codec2}/lib -arch x86_64" + export CPPFLAGS="-I${_codec2}/include" + export PKG_CONFIG_PATH="${_codec2}/lib/pkgconfig:/usr/local/lib/pkgconfig:/usr/local/share/pkgconfig" + arch -x86_64 "$PY_X64" -m pip install -U pip setuptools wheel + arch -x86_64 "$PY_X64" -m pip install "cx-freeze>=7.0.0" + arch -x86_64 "$PY_X64" -m pip install -e . + arch -x86_64 "$PY_X64" scripts/patch_lxst_pyogg_ogg_ctypes.py - name: Download frontend artifact uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 0000000..a473afe --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,222 @@ +# Publish ``reticulum-meshchatx`` sdist and wheel to PyPI (Trusted Publishing). +# Builds the Vite frontend and offline bundles via the reusable frontend workflow, +# places them under ``meshchatx/public/``, then runs ``python -m build``. +# +# Tag runs also emit SLSA generic provenance (``generator_generic_slsa3``) for the +# exact ``dist/`` digests and optional Cosign ``*.cosign.bundle`` files next to each +# distribution (same scripts as ``build-release.yml``; skips if ``COSIGN_PRIVATE_KEY`` unset). +# PyPI upload uses a staging directory so bundles are not sent to the index. +# +# PyPI Trusted Publisher must reference this file as ``pypi.yml`` and use the +# ``pypi`` GitHub Environment (with required reviewers if you enabled protection). +# +# Pinned first-party actions (bump tag and SHA together when upgrading): +# actions/checkout@v6.0.1 8e8c483db84b4bee98b60c0593521ed34d9990e8 +# actions/setup-python@v6.2.0 a309ff8b426b58ec0e2a45f0f869d46889d02405 +# actions/upload-artifact@v5.0.0 330a01c490aca151604b8cf639adc76d48f6c5d4 +# actions/download-artifact@v5.0.0 634f93cb2916e3fdff6788551b99b062d0335ce0 +# +# SLSA generator (must stay @vX.Y.Z semver per upstream): +# slsa-framework/slsa-github-generator/generator_generic_slsa3.yml@v2.1.0 +# +# Third-party pin (resolve before bumping ``release/v1``): +# curl -sS "https://api.github.com/repos/pypa/gh-action-pypi-publish/commits/release/v1" | jq -r '.sha' +# pypa/gh-action-pypi-publish@release/v1 -> cef221092ed1bacb1cc03d23a2d87d1d172e277b + +name: Publish to PyPI + +on: + push: + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: read + actions: read + id-token: write + +concurrency: + group: pypi-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + NODE_OPTIONS: --max-old-space-size=8192 + PYTHON_VERSION: "3.14" + COSIGN_VERSION: "3.0.6" + +jobs: + frontend: + name: Build frontend artifact + uses: ./.github/workflows/frontend-build.yml + permissions: + contents: read + with: + artifact_name: meshchatx-frontend-pypi-${{ github.run_id }}-${{ github.run_attempt }} + retention_days: 2 + + build: + name: Build Python distributions + needs: frontend + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + hashes: ${{ steps.slsa-hashes.outputs.hashes }} + defaults: + run: + shell: bash + env: + FRONTEND_ARTIFACT_NAME: ${{ needs.frontend.outputs.artifact_name }} + steps: + - name: Checkout + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Download frontend artifact + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 + with: + name: ${{ env.FRONTEND_ARTIFACT_NAME }} + path: meshchatx/public + + - name: Verify frontend bundle in tree + run: | + set -euo pipefail + test -f meshchatx/public/index.html + test -d meshchatx/public/assets + test -d meshchatx/public/reticulum-docs-bundled/current + + - name: Install build + run: python -m pip install -U pip "build>=1.2.0" + + - name: Build sdist and wheel + run: python -m build + + - name: Verify wheel contains frontend assets + run: | + set -euo pipefail + whl=(dist/*.whl) + if [[ "${#whl[@]}" -ne 1 ]]; then + echo "Expected exactly one wheel in dist/" >&2 + ls -la dist >&2 + exit 1 + fi + export WHEEL_PATH="${whl[0]}" + python - <<'PY' + import os + import sys + import zipfile + + path = os.environ["WHEEL_PATH"] + with zipfile.ZipFile(path) as zf: + names = zf.namelist() + if "meshchatx/public/index.html" not in names: + print("missing: meshchatx/public/index.html", file=sys.stderr) + sys.exit(1) + if not any(n.startswith("meshchatx/public/assets/") for n in names): + print("missing: meshchatx/public/assets/", file=sys.stderr) + sys.exit(1) + print("wheel contains meshchatx/public frontend paths") + PY + + - name: SLSA subject hashes (PyPI dists) + id: slsa-hashes + if: github.ref_type == 'tag' + run: bash scripts/ci/github-slsa-hashes-dist.sh + + - name: Cosign attestations (sdist + wheels) + if: github.ref_type == 'tag' + env: + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_REF: ${{ github.ref }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_WORKFLOW_FILE: pypi.yml + COSIGN_VERSION: ${{ env.COSIGN_VERSION }} + run: | + set -eu + if [ -z "${COSIGN_PRIVATE_KEY:-}" ]; then + echo "Skipping cosign attestations (no COSIGN_PRIVATE_KEY)." + exit 0 + fi + sh scripts/ci/setup-cosign.sh "${COSIGN_VERSION}" + printf '%s\n' "$COSIGN_PRIVATE_KEY" > /tmp/cosign.key + chmod 600 /tmp/cosign.key + export COSIGN_KEY_PATH=/tmp/cosign.key + sh scripts/ci/attest-release-assets.sh ./dist + rm -f /tmp/cosign.key + + - name: Store distribution packages + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 + with: + name: python-package-distributions + path: dist/ + if-no-files-found: error + + slsa-provenance-pypi: + name: SLSA provenance (PyPI dists) + needs: build + if: github.ref_type == 'tag' + permissions: + id-token: write + contents: write + actions: read + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.1.0 + with: + base64-subjects: ${{ needs.build.outputs.hashes }} + upload-assets: false + provenance-name: meshchatx-pypi-${{ github.ref_name }}.intoto.jsonl + + publish-to-pypi: + name: Publish to PyPI + if: >- + github.ref_type == 'tag' && + needs.build.result == 'success' && + needs.slsa-provenance-pypi.result == 'success' + needs: + - build + - slsa-provenance-pypi + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: + name: pypi + url: https://pypi.org/project/meshchatx/ + permissions: + actions: read + contents: read + id-token: write + steps: + - name: Download distributions + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 + with: + name: python-package-distributions + path: dist/ + + - name: Stage PyPI uploads (exclude Cosign bundles) + run: | + set -euo pipefail + mkdir -p pypi-upload + shopt -s nullglob + files=(dist/*.whl dist/*.tar.gz) + if [[ "${#files[@]}" -eq 0 ]]; then + echo "No .whl or .tar.gz in dist/" >&2 + ls -la dist >&2 + exit 1 + fi + cp -v "${files[@]}" pypi-upload/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b + with: + packages-dir: pypi-upload/ diff --git a/scripts/ci/github-ensure-macos-x86-64-homebrew.sh b/scripts/ci/github-ensure-macos-x86-64-homebrew.sh new file mode 100755 index 0000000..c458d74 --- /dev/null +++ b/scripts/ci/github-ensure-macos-x86-64-homebrew.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Ensure an x86_64 (Rosetta) Homebrew exists at /usr/local/bin/brew. +# GitHub-hosted Apple Silicon runners ship /opt/homebrew only; cx_Freeze universal +# x64 slices need x86_64 libraries from /usr/local (see codec2, libyaml steps). +set -euo pipefail + +if [[ "$(uname -s)" != "Darwin" ]]; then + exit 0 +fi + +if [[ -x /usr/local/bin/brew ]]; then + exit 0 +fi + +echo "github-ensure-macos-x86-64-homebrew: installing Homebrew under Rosetta into /usr/local (one-time on this runner)" >&2 +export NONINTERACTIVE=1 +export HOMEBREW_NO_ANALYTICS=1 +export HOMEBREW_NO_AUTO_UPDATE=1 +arch -x86_64 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +test -x /usr/local/bin/brew diff --git a/scripts/ci/github-slsa-hashes-dist.sh b/scripts/ci/github-slsa-hashes-dist.sh new file mode 100644 index 0000000..b579519 --- /dev/null +++ b/scripts/ci/github-slsa-hashes-dist.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Emit base64-encoded sha256sum lines for files in ./dist (SLSA generic generator input). +# Excludes *.cosign.bundle. Writes "hashes=" to GITHUB_OUTPUT when set. +set -euo pipefail + +cd "$(dirname "$0")/../.." +if [ ! -d dist ]; then + echo "dist/ missing" >&2 + exit 1 +fi + +tmp="$(mktemp)" +trap 'rm -f "$tmp"' EXIT + +( + cd dist + find . -maxdepth 1 -type f ! -name '*.cosign.bundle' -printf '%P\0' | sort -z | xargs -0 sha256sum +) >"$tmp" + +if [ ! -s "$tmp" ]; then + echo "No files to hash under dist/" >&2 + exit 1 +fi + +b64="$(base64 -w0 <"$tmp")" +if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "hashes=${b64}" >>"$GITHUB_OUTPUT" +else + printf '%s\n' "$b64" +fi