diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6d445b8b1..a6a3283c5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,8 +66,19 @@ jobs: build: name: "ubuntu-${{ matrix.os }}-${{ matrix.arch }}, GHC: ${{ matrix.ghc }}" needs: maybe-release + permissions: + contents: write + packages: write env: apps: "smp-server xftp-server ntf-server xftp" + opt: ${{ github.event_name == 'pull_request' && 'O0' || 'O2' }} + ghc_options: ${{ github.event_name == 'pull_request' && '--ghc-options=-O0' || '' }} + release: ${{ startsWith(github.ref, 'refs/tags/v') }} + postgres_dist: ${{ startsWith(github.ref, 'refs/tags/v') && 'dist-newstyle' || 'dist-postgres' }} + branch: ${{ github.base_ref || github.ref_name }} + cache_o0: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/stable') }} + image_cache: ghcr.io/simplex-chat/simplexmq-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }} + source_hashes: dist-newstyle/sources.sha256 runs-on: ${{ matrix.runner }} services: postgres: @@ -146,6 +157,18 @@ jobs: sudo apt update sudo apt -y install postgresql-client-15 + - name: Log in to GitHub Container Registry + if: matrix.should_run == true && env.release != 'true' + shell: bash + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: echo "$GHCR_TOKEN" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + + - name: Compute image cache week + id: week + if: matrix.should_run == true && env.release != 'true' + run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" + - name: Build and cache Docker image if: matrix.should_run == true uses: simplex-chat/docker-build-push-action@v6 @@ -157,20 +180,14 @@ jobs: build-args: | TAG=${{ matrix.os }} GHC=${{ matrix.ghc }} - - - name: Cache dependencies - if: matrix.should_run == true - uses: actions/cache@v4 - with: - path: | - ~/.cabal/store - dist-newstyle - key: ubuntu-${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}-${{ hashFiles('cabal.project', 'simplexmq.cabal') }} + cache-from: ${{ env.release != 'true' && format('type=registry,ref={0}-{1}', env.image_cache, steps.week.outputs.week) || '' }} + cache-to: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && format('type=registry,ref={0}-{1},mode=max,ignore-error=true', env.image_cache, steps.week.outputs.week) || '' }} - name: Start container if: matrix.should_run == true shell: bash run: | + mkdir -p ~/.cabal docker run -t -d \ --device /dev/fuse \ --cap-add SYS_ADMIN \ @@ -181,21 +198,66 @@ jobs: -v ${{ github.workspace }}:/project \ build/${{ matrix.os }}:latest + - name: Compute cache key prefixes + id: keys + if: matrix.should_run == true && env.release != 'true' + run: | + toolchain=$(docker exec builder dpkg-query -W -f '${db:Status-Status} ${binary:Package} ${Version}\n' 'binutils*' 'cpp*' 'gcc*' '*-dev' | grep '^installed ' | sha256sum | cut -c1-16) + target=${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}-$toolchain + echo "deps=deps-$target" >> "$GITHUB_OUTPUT" + echo "dist=dist-$target" >> "$GITHUB_OUTPUT" + + - name: Restore dependencies from cache + id: restore-deps + if: matrix.should_run == true && env.release != 'true' + uses: actions/cache/restore@v4 + with: + path: ~/.cabal/store + key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ github.sha }} + restore-keys: | + ${{ steps.keys.outputs.deps }}-${{ env.branch }}- + ${{ steps.keys.outputs.deps }}- + + - name: Warn if dependencies not in cache + if: matrix.should_run == true && env.release != 'true' && steps.restore-deps.outputs.cache-matched-key == '' + run: echo "::warning title=Cache miss::${{ steps.keys.outputs.deps }}-${{ env.branch }}" + + - name: Restore build directories from cache + id: restore-build + if: matrix.should_run == true && env.release != 'true' + uses: actions/cache/restore@v4 + with: + path: | + dist-newstyle + dist-postgres + key: ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-${{ github.sha }} + restore-keys: | + ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}- + ${{ steps.keys.outputs.dist }}-${{ env.opt }}- + + - name: Warn if build directories not in cache + if: matrix.should_run == true && env.release != 'true' && steps.restore-build.outputs.cache-matched-key == '' + run: echo "::warning title=Cache miss::${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}" + + - name: Set old timestamps on unchanged sources (GHC 8.10.7) + if: matrix.ghc == '8.10.7' && steps.restore-build.outputs.cache-matched-key != '' && hashFiles(env.source_hashes) != '' + run: scripts/ci/source-timestamps.sh restore ${{ env.source_hashes }} + - name: Build smp-server, xftp-server (postgresql) and tests if: matrix.should_run == true shell: docker exec -t builder sh -eu {0} run: | - chmod -fR 777 ~/.cabal ./dist-newstyle || :; git config --global --add safe.directory '*' - cabal clean + chmod -fR 777 ~/.cabal ./dist-newstyle ./dist-postgres || :; git config --global --add safe.directory '*' + scripts/ci/prune-old-builds.sh dist-newstyle dist-postgres cabal update - cabal build --jobs=$(nproc) --enable-tests -fserver_postgres + cabal build --builddir=${{ env.postgres_dist }} --jobs=$(nproc) --enable-tests -fserver_postgres ${{ env.ghc_options }} mkdir -p /out - for i in smp-server xftp-server simplexmq-test; do - bin=$(find /project/dist-newstyle -name "$i" -type f -executable) + for i in smp-server xftp-server ntf-server simplexmq-test; do + bin=$(find /project/${{ env.postgres_dist }} -name "$i" -type f -executable) chmod +x "$bin" - mv "$bin" /out/ + cp "$bin" /out/ done - strip /out/smp-server /out/xftp-server + strip /out/smp-server /out/xftp-server /out/ntf-server - name: Copy simplexmq-test from container if: matrix.should_run == true @@ -228,16 +290,17 @@ jobs: cat hashes.output >> "$GITHUB_OUTPUT" - name: Build everything else (standard) + id: build-standard if: matrix.should_run == true shell: docker exec -t builder sh -eu {0} run: | - cabal build --jobs=$(nproc) + cabal build --jobs=$(nproc) ${{ env.ghc_options }} mkdir -p /out - for i in ${{ env.apps }}; do + for i in smp-server xftp-server xftp; do bin=$(find /project/dist-newstyle -name "$i" -type f -executable) strip "$bin" chmod +x "$bin" - mv "$bin" /out/ + cp "$bin" /out/ done - name: Copy binaries from container and prepare them @@ -282,6 +345,33 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Record source hashes (GHC 8.10.7) + if: matrix.should_run == true && matrix.ghc == '8.10.7' + shell: docker exec -t builder sh -eu {0} + run: scripts/ci/source-timestamps.sh record ${{ env.source_hashes }} + + - name: Compute dependency cache key + id: hash-deps + if: matrix.should_run == true && env.release != 'true' + shell: bash + run: echo "hash=$(ls ~/.cabal/store/ghc-*/ | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT" + + - name: Save dependencies to cache + if: matrix.should_run == true && env.release != 'true' && steps.restore-deps.outputs.cache-matched-key != format('{0}-{1}-{2}', steps.keys.outputs.deps, env.branch, steps.hash-deps.outputs.hash) + uses: actions/cache/save@v4 + with: + path: ~/.cabal/store + key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ steps.hash-deps.outputs.hash }} + + - name: Save build directories to cache + if: matrix.should_run == true && env.release != 'true' && steps.restore-build.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: | + dist-newstyle + dist-postgres + key: ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-${{ github.sha }} + - name: Test if: matrix.should_run == true && matrix.arch == 'x86-64' timeout-minutes: 120 @@ -306,6 +396,52 @@ jobs: exit 1 fi + - name: Remove -O2 build directories + id: remove-o2 + if: env.cache_o0 == 'true' && !cancelled() && steps.build-standard.outcome == 'success' + shell: docker exec -t builder sh -eu {0} + run: rm -rf dist-newstyle dist-postgres + + - name: Restore -O0 build directories from cache + id: restore-build-o0 + if: ${{ !cancelled() && steps.remove-o2.outcome == 'success' }} + uses: actions/cache/restore@v4 + with: + path: | + dist-newstyle + dist-postgres + key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }} + restore-keys: | + ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}- + ${{ steps.keys.outputs.dist }}-O0- + + - name: Set old timestamps on unchanged sources for -O0 build (GHC 8.10.7) + if: ${{ !cancelled() && matrix.ghc == '8.10.7' && steps.restore-build-o0.outputs.cache-matched-key != '' && hashFiles(env.source_hashes) != '' }} + run: scripts/ci/source-timestamps.sh restore ${{ env.source_hashes }} + + - name: Build at -O0 for pull request caches + id: build-o0 + if: ${{ !cancelled() && steps.remove-o2.outcome == 'success' }} + shell: docker exec -t builder sh -eu {0} + run: | + scripts/ci/prune-old-builds.sh dist-newstyle dist-postgres + cabal build --builddir=dist-postgres --jobs=$(nproc) --enable-tests -fserver_postgres --ghc-options=-O0 + cabal build --jobs=$(nproc) --ghc-options=-O0 + + - name: Record source hashes for -O0 build (GHC 8.10.7) + if: ${{ !cancelled() && matrix.ghc == '8.10.7' && steps.build-o0.outcome == 'success' }} + shell: docker exec -t builder sh -eu {0} + run: scripts/ci/source-timestamps.sh record ${{ env.source_hashes }} + + - name: Save -O0 build directories to cache + if: ${{ !cancelled() && steps.build-o0.outcome == 'success' && steps.restore-build-o0.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: | + dist-newstyle + dist-postgres + key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }} + # ============================= # Resolver test job # ============================= diff --git a/scripts/ci/prune-old-builds.sh b/scripts/ci/prune-old-builds.sh new file mode 100755 index 000000000..c122ed338 --- /dev/null +++ b/scripts/ci/prune-old-builds.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +version=$(awk '/^version:/ {print $2}' simplexmq.cabal) +for dir in "$@"; do + if [ -d "$dir/build" ]; then + find "$dir/build" -mindepth 3 -maxdepth 3 -type d -name 'simplexmq-*' ! -name "simplexmq-$version" -exec rm -rf {} + + fi +done diff --git a/scripts/ci/source-timestamps.sh b/scripts/ci/source-timestamps.sh new file mode 100755 index 000000000..e8bc8b1ce --- /dev/null +++ b/scripts/ci/source-timestamps.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -eu + +command=$1 +hashes=$2 + +sources() { + git ls-files -z '*.hs' '*.hsc' +} + +case "$command" in + record) + sources | xargs -0 sha256sum > "$hashes" + ;; + restore) + sources | xargs -0 touch + while read -r hash file; do + if [ -f "$file" ] && [ "$(sha256sum "$file" | cut -d' ' -f1)" = "$hash" ]; then + touch -d 2000-01-01 "$file" + fi + done < "$hashes" + ;; + *) + echo "usage: $0 record|restore " >&2 + exit 1 + ;; +esac diff --git a/tests/Util.hs b/tests/Util.hs index 9a4049d68..eaa670f06 100644 --- a/tests/Util.hs +++ b/tests/Util.hs @@ -56,7 +56,7 @@ instance Example a => Example (TestWrapper a) where ci <- envCI runTest `E.catches` [E.Handler (onTestFailure ci), E.Handler (onTestException ci)] where - tt = 120 + tt = 300 runTest = timeout (tt * 1000000) (evaluateExample action params hooks state) `finally` callCommand "sync" >>= \case Just r -> pure r