mirror of
https://github.com/simplex-chat/simplexmq.git
synced 2026-10-16 10:19:57 +00:00
EvgenyandEvgeny @ SimpleX Chat
3db4029bb9
* ci: build cache * update build job * rename steps, cache all builds, patch timestamps for ghc 8.10.7 * refactor, save O0 caches in master and stable builds * update build to include toolchain fingerprint * do not save cache on exact match * remove caching docker from prs --------- Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>
470 lines
18 KiB
YAML
470 lines
18 KiB
YAML
name: build
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- master
|
|
- stable
|
|
tags:
|
|
- "v*"
|
|
pull_request:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/v') }}
|
|
|
|
jobs:
|
|
|
|
# =============================
|
|
# Create release
|
|
# =============================
|
|
|
|
# Create release, but only if it's triggered by tag push.
|
|
# On pull requests/commits push, this job will always complete.
|
|
|
|
maybe-release:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Clone project
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
uses: actions/checkout@v3
|
|
with:
|
|
submodules: recursive
|
|
|
|
- name: Build changelog
|
|
id: build_changelog
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
uses: simplex-chat/release-changelog-builder-action@v5
|
|
with:
|
|
configuration: .github/changelog_conf.json
|
|
failOnError: true
|
|
ignorePreReleases: true
|
|
commitMode: true
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Create release
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
uses: simplex-chat/action-gh-release@v2
|
|
with:
|
|
body: |
|
|
See full changelog [here](https://github.com/simplex-chat/simplexmq/blob/master/CHANGELOG.md).
|
|
|
|
Commits:
|
|
${{ steps.build_changelog.outputs.changelog }}
|
|
prerelease: true
|
|
files: |
|
|
LICENSE
|
|
fail_on_unmatched_files: true
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
# =============================
|
|
# Main build job
|
|
# =============================
|
|
|
|
build:
|
|
name: "ubuntu-${{ matrix.os }}-${{ matrix.arch }}, GHC: ${{ matrix.ghc }}"
|
|
needs: maybe-release
|
|
permissions:
|
|
contents: write
|
|
packages: write
|
|
env:
|
|
apps: "smp-server xftp-server ntf-server xftp"
|
|
opt: ${{ github.event_name == 'pull_request' && 'O0' || 'O2' }}
|
|
ghc_options: ${{ github.event_name == 'pull_request' && '--ghc-options=-O0' || '' }}
|
|
release: ${{ startsWith(github.ref, 'refs/tags/v') }}
|
|
postgres_dist: ${{ startsWith(github.ref, 'refs/tags/v') && 'dist-newstyle' || 'dist-postgres' }}
|
|
branch: ${{ github.base_ref || github.ref_name }}
|
|
cache_o0: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/stable') }}
|
|
image_cache: ghcr.io/simplex-chat/simplexmq-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}
|
|
source_hashes: dist-newstyle/sources.sha256
|
|
runs-on: ${{ matrix.runner }}
|
|
services:
|
|
postgres:
|
|
image: postgres:15
|
|
env:
|
|
POSTGRES_HOST_AUTH_METHOD: trust # Allows passwordless access
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
ports:
|
|
# Maps tcp port 5432 on service container to the host
|
|
- 5432:5432
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- os: 22.04
|
|
os_underscore: 22_04
|
|
arch: x86-64
|
|
runner: "ubuntu-22.04"
|
|
ghc: "8.10.7"
|
|
should_run: ${{ !(github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }}
|
|
- os: 22.04
|
|
os_underscore: 22_04
|
|
arch: x86-64
|
|
runner: "ubuntu-22.04"
|
|
ghc: "9.6.3"
|
|
should_run: true
|
|
- os: 24.04
|
|
os_underscore: 24_04
|
|
arch: x86-64
|
|
runner: "ubuntu-24.04"
|
|
ghc: "9.6.3"
|
|
should_run: true
|
|
- os: 22.04
|
|
os_underscore: 22_04
|
|
arch: aarch64
|
|
runner: "ubuntu-22.04-arm"
|
|
ghc: "9.6.3"
|
|
should_run: true
|
|
- os: 24.04
|
|
os_underscore: 24_04
|
|
arch: aarch64
|
|
runner: "ubuntu-24.04-arm"
|
|
ghc: "9.6.3"
|
|
should_run: true
|
|
steps:
|
|
- name: Clone project
|
|
if: matrix.should_run == true
|
|
uses: actions/checkout@v3
|
|
with:
|
|
submodules: recursive
|
|
|
|
- name: Set up Docker Buildx
|
|
if: matrix.should_run == true
|
|
uses: simplex-chat/docker-setup-buildx-action@v3
|
|
|
|
- name: Setup swap
|
|
if: matrix.ghc == '8.10.7' && matrix.should_run == true
|
|
uses: ./.github/actions/swap
|
|
with:
|
|
swap-size-gb: 20
|
|
|
|
- name: Install PostgreSQL 15 client tools
|
|
if: matrix.os == '22.04' && matrix.should_run == true
|
|
shell: bash
|
|
run: |
|
|
# Import the repository signing key
|
|
sudo install -d /usr/share/postgresql-common/pgdg
|
|
sudo curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
|
|
# Add the PostgreSQL APT repository
|
|
sudo sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
|
|
# Update repository and install postgresql tools
|
|
sudo apt update
|
|
sudo apt -y install postgresql-client-15
|
|
|
|
- name: Log in to GitHub Container Registry
|
|
if: matrix.should_run == true && env.release != 'true'
|
|
shell: bash
|
|
env:
|
|
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: echo "$GHCR_TOKEN" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
|
|
|
|
- name: Compute image cache week
|
|
id: week
|
|
if: matrix.should_run == true && env.release != 'true'
|
|
run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Build and cache Docker image
|
|
if: matrix.should_run == true
|
|
uses: simplex-chat/docker-build-push-action@v6
|
|
with:
|
|
context: .
|
|
load: true
|
|
file: Dockerfile.build
|
|
tags: build/${{ matrix.os }}:latest
|
|
build-args: |
|
|
TAG=${{ matrix.os }}
|
|
GHC=${{ matrix.ghc }}
|
|
cache-from: ${{ env.release != 'true' && format('type=registry,ref={0}-{1}', env.image_cache, steps.week.outputs.week) || '' }}
|
|
cache-to: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && format('type=registry,ref={0}-{1},mode=max,ignore-error=true', env.image_cache, steps.week.outputs.week) || '' }}
|
|
|
|
- name: Start container
|
|
if: matrix.should_run == true
|
|
shell: bash
|
|
run: |
|
|
mkdir -p ~/.cabal
|
|
docker run -t -d \
|
|
--device /dev/fuse \
|
|
--cap-add SYS_ADMIN \
|
|
--security-opt apparmor:unconfined \
|
|
--name builder \
|
|
-v ~/.cabal:/root/.cabal \
|
|
-v /home/runner/work/_temp:/home/runner/work/_temp \
|
|
-v ${{ github.workspace }}:/project \
|
|
build/${{ matrix.os }}:latest
|
|
|
|
- name: Compute cache key prefixes
|
|
id: keys
|
|
if: matrix.should_run == true && env.release != 'true'
|
|
run: |
|
|
toolchain=$(docker exec builder dpkg-query -W -f '${db:Status-Status} ${binary:Package} ${Version}\n' 'binutils*' 'cpp*' 'gcc*' '*-dev' | grep '^installed ' | sha256sum | cut -c1-16)
|
|
target=${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}-$toolchain
|
|
echo "deps=deps-$target" >> "$GITHUB_OUTPUT"
|
|
echo "dist=dist-$target" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Restore dependencies from cache
|
|
id: restore-deps
|
|
if: matrix.should_run == true && env.release != 'true'
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/.cabal/store
|
|
key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ github.sha }}
|
|
restore-keys: |
|
|
${{ steps.keys.outputs.deps }}-${{ env.branch }}-
|
|
${{ steps.keys.outputs.deps }}-
|
|
|
|
- name: Warn if dependencies not in cache
|
|
if: matrix.should_run == true && env.release != 'true' && steps.restore-deps.outputs.cache-matched-key == ''
|
|
run: echo "::warning title=Cache miss::${{ steps.keys.outputs.deps }}-${{ env.branch }}"
|
|
|
|
- name: Restore build directories from cache
|
|
id: restore-build
|
|
if: matrix.should_run == true && env.release != 'true'
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: |
|
|
dist-newstyle
|
|
dist-postgres
|
|
key: ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-${{ github.sha }}
|
|
restore-keys: |
|
|
${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-
|
|
${{ steps.keys.outputs.dist }}-${{ env.opt }}-
|
|
|
|
- name: Warn if build directories not in cache
|
|
if: matrix.should_run == true && env.release != 'true' && steps.restore-build.outputs.cache-matched-key == ''
|
|
run: echo "::warning title=Cache miss::${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}"
|
|
|
|
- name: Set old timestamps on unchanged sources (GHC 8.10.7)
|
|
if: matrix.ghc == '8.10.7' && steps.restore-build.outputs.cache-matched-key != '' && hashFiles(env.source_hashes) != ''
|
|
run: scripts/ci/source-timestamps.sh restore ${{ env.source_hashes }}
|
|
|
|
- name: Build smp-server, xftp-server (postgresql) and tests
|
|
if: matrix.should_run == true
|
|
shell: docker exec -t builder sh -eu {0}
|
|
run: |
|
|
chmod -fR 777 ~/.cabal ./dist-newstyle ./dist-postgres || :; git config --global --add safe.directory '*'
|
|
scripts/ci/prune-old-builds.sh dist-newstyle dist-postgres
|
|
cabal update
|
|
cabal build --builddir=${{ env.postgres_dist }} --jobs=$(nproc) --enable-tests -fserver_postgres ${{ env.ghc_options }}
|
|
mkdir -p /out
|
|
for i in smp-server xftp-server ntf-server simplexmq-test; do
|
|
bin=$(find /project/${{ env.postgres_dist }} -name "$i" -type f -executable)
|
|
chmod +x "$bin"
|
|
cp "$bin" /out/
|
|
done
|
|
strip /out/smp-server /out/xftp-server /out/ntf-server
|
|
|
|
- name: Copy simplexmq-test from container
|
|
if: matrix.should_run == true
|
|
shell: bash
|
|
run: |
|
|
docker cp builder:/out/simplexmq-test .
|
|
|
|
- name: Copy smp-server, xftp-server (postgresql) from container and prepare it
|
|
if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true
|
|
id: prepare-postgres
|
|
shell: bash
|
|
run: |
|
|
printf 'bins<<EOF\n' > bins.output
|
|
printf 'hashes<<EOF\n' > hashes.output
|
|
|
|
for i in smp-server xftp-server; do
|
|
name="${i}-postgres-ubuntu-${{ matrix.os_underscore }}-${{ matrix.arch }}"
|
|
docker cp builder:/out/$i $name
|
|
|
|
path="${{ github.workspace }}/$name"
|
|
hash="SHA2-256($name)= $(openssl sha256 $path | cut -d' ' -f 2)"
|
|
|
|
printf '%s\n' "$path" >> bins.output
|
|
printf '%s\n\n' "$hash" >> hashes.output
|
|
done
|
|
printf 'EOF\n' >> bins.output
|
|
printf 'EOF\n' >> hashes.output
|
|
|
|
cat bins.output >> "$GITHUB_OUTPUT"
|
|
cat hashes.output >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Build everything else (standard)
|
|
id: build-standard
|
|
if: matrix.should_run == true
|
|
shell: docker exec -t builder sh -eu {0}
|
|
run: |
|
|
cabal build --jobs=$(nproc) ${{ env.ghc_options }}
|
|
mkdir -p /out
|
|
for i in smp-server xftp-server xftp; do
|
|
bin=$(find /project/dist-newstyle -name "$i" -type f -executable)
|
|
strip "$bin"
|
|
chmod +x "$bin"
|
|
cp "$bin" /out/
|
|
done
|
|
|
|
- name: Copy binaries from container and prepare them
|
|
id: prepare-regular
|
|
if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true
|
|
shell: bash
|
|
run: |
|
|
docker cp builder:/out .
|
|
|
|
printf 'bins<<EOF\n' > bins.output
|
|
printf 'hashes<<EOF\n' > hashes.output
|
|
for i in ${{ env.apps }}; do
|
|
name="$i-ubuntu-${{ matrix.os_underscore }}-${{ matrix.arch }}"
|
|
|
|
mv ./out/$i ./$name
|
|
|
|
path="${{ github.workspace }}/$name"
|
|
hash="SHA2-256($name)= $(openssl sha256 $path | cut -d' ' -f 2)"
|
|
|
|
printf '%s\n' "$path" >> bins.output
|
|
printf '%s\n\n' "$hash" >> hashes.output
|
|
done
|
|
printf 'EOF\n' >> bins.output
|
|
printf 'EOF\n' >> hashes.output
|
|
|
|
cat bins.output >> "$GITHUB_OUTPUT"
|
|
cat hashes.output >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Upload binaries
|
|
if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true
|
|
uses: simplex-chat/action-gh-release@v2
|
|
with:
|
|
append_body: true
|
|
prerelease: true
|
|
fail_on_unmatched_files: true
|
|
body: |
|
|
${{ steps.prepare-regular.outputs.hashes }}
|
|
${{ steps.prepare-postgres.outputs.hashes }}
|
|
files: |
|
|
${{ steps.prepare-regular.outputs.bins }}
|
|
${{ steps.prepare-postgres.outputs.bins }}
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Record source hashes (GHC 8.10.7)
|
|
if: matrix.should_run == true && matrix.ghc == '8.10.7'
|
|
shell: docker exec -t builder sh -eu {0}
|
|
run: scripts/ci/source-timestamps.sh record ${{ env.source_hashes }}
|
|
|
|
- name: Compute dependency cache key
|
|
id: hash-deps
|
|
if: matrix.should_run == true && env.release != 'true'
|
|
shell: bash
|
|
run: echo "hash=$(ls ~/.cabal/store/ghc-*/ | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Save dependencies to cache
|
|
if: matrix.should_run == true && env.release != 'true' && steps.restore-deps.outputs.cache-matched-key != format('{0}-{1}-{2}', steps.keys.outputs.deps, env.branch, steps.hash-deps.outputs.hash)
|
|
uses: actions/cache/save@v4
|
|
with:
|
|
path: ~/.cabal/store
|
|
key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ steps.hash-deps.outputs.hash }}
|
|
|
|
- name: Save build directories to cache
|
|
if: matrix.should_run == true && env.release != 'true' && steps.restore-build.outputs.cache-hit != 'true'
|
|
uses: actions/cache/save@v4
|
|
with:
|
|
path: |
|
|
dist-newstyle
|
|
dist-postgres
|
|
key: ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-${{ github.sha }}
|
|
|
|
- name: Test
|
|
if: matrix.should_run == true && matrix.arch == 'x86-64'
|
|
timeout-minutes: 120
|
|
shell: bash
|
|
env:
|
|
PGHOST: localhost
|
|
run: |
|
|
i=1
|
|
attempts=1
|
|
${{ (github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }} && attempts=3
|
|
while [ "$i" -le "$attempts" ]; do
|
|
if ./simplexmq-test; then
|
|
break
|
|
else
|
|
echo "Attempt $i failed, retrying..."
|
|
i=$((i + 1))
|
|
sleep 1
|
|
fi
|
|
done
|
|
if [ "$i" -gt "$attempts" ]; then
|
|
echo "All "$attempts" attempts failed."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Remove -O2 build directories
|
|
id: remove-o2
|
|
if: env.cache_o0 == 'true' && !cancelled() && steps.build-standard.outcome == 'success'
|
|
shell: docker exec -t builder sh -eu {0}
|
|
run: rm -rf dist-newstyle dist-postgres
|
|
|
|
- name: Restore -O0 build directories from cache
|
|
id: restore-build-o0
|
|
if: ${{ !cancelled() && steps.remove-o2.outcome == 'success' }}
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: |
|
|
dist-newstyle
|
|
dist-postgres
|
|
key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }}
|
|
restore-keys: |
|
|
${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-
|
|
${{ steps.keys.outputs.dist }}-O0-
|
|
|
|
- name: Set old timestamps on unchanged sources for -O0 build (GHC 8.10.7)
|
|
if: ${{ !cancelled() && matrix.ghc == '8.10.7' && steps.restore-build-o0.outputs.cache-matched-key != '' && hashFiles(env.source_hashes) != '' }}
|
|
run: scripts/ci/source-timestamps.sh restore ${{ env.source_hashes }}
|
|
|
|
- name: Build at -O0 for pull request caches
|
|
id: build-o0
|
|
if: ${{ !cancelled() && steps.remove-o2.outcome == 'success' }}
|
|
shell: docker exec -t builder sh -eu {0}
|
|
run: |
|
|
scripts/ci/prune-old-builds.sh dist-newstyle dist-postgres
|
|
cabal build --builddir=dist-postgres --jobs=$(nproc) --enable-tests -fserver_postgres --ghc-options=-O0
|
|
cabal build --jobs=$(nproc) --ghc-options=-O0
|
|
|
|
- name: Record source hashes for -O0 build (GHC 8.10.7)
|
|
if: ${{ !cancelled() && matrix.ghc == '8.10.7' && steps.build-o0.outcome == 'success' }}
|
|
shell: docker exec -t builder sh -eu {0}
|
|
run: scripts/ci/source-timestamps.sh record ${{ env.source_hashes }}
|
|
|
|
- name: Save -O0 build directories to cache
|
|
if: ${{ !cancelled() && steps.build-o0.outcome == 'success' && steps.restore-build-o0.outputs.cache-hit != 'true' }}
|
|
uses: actions/cache/save@v4
|
|
with:
|
|
path: |
|
|
dist-newstyle
|
|
dist-postgres
|
|
key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }}
|
|
|
|
# =============================
|
|
# Resolver test job
|
|
# =============================
|
|
|
|
# The SNRC resolver is Python, so this job needs none of the Haskell toolchain
|
|
# above and runs independently of it.
|
|
|
|
resolver-test:
|
|
name: "resolver (python)"
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Clone project
|
|
uses: actions/checkout@v3
|
|
|
|
- name: Set up uv
|
|
# uv takes Python from .python-version, the version of the Dockerfile's runtime stage.
|
|
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
|
|
- name: Lint
|
|
working-directory: scripts/resolver/service
|
|
run: uv run --frozen ruff check .
|
|
|
|
- name: Test
|
|
working-directory: scripts/resolver/service
|
|
run: uv run --frozen pytest -v
|