mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-10-21 03:34:19 +00:00
ci: cache build artifacts for faster builds (#7730)
* ci: cache build artifacts for faster builds * test build * compression * fix macos-latest * debug build * better caching * test cache * update build * test cache * ci: check toolchain restore * ci: remove Windows toolchain cache * ci: skip tests on macOS Intel --------- Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>
This commit is contained in:
Evgeny
co-authored by
Evgeny @ SimpleX Chat
Evgeny @ SimpleX Chat
parent
a932c8298a
commit
9a90d8e230
@@ -10,21 +10,19 @@ inputs:
|
||||
github_ref:
|
||||
required: true
|
||||
description: "Git reference"
|
||||
os:
|
||||
required: true
|
||||
description: "Target OS"
|
||||
cache_path:
|
||||
required: false
|
||||
default: "~/.cabal/store"
|
||||
description: "Cache path"
|
||||
cabal_ver:
|
||||
required: false
|
||||
default: 3.10.2.0
|
||||
description: "GHC version to install"
|
||||
outputs:
|
||||
cabal-store:
|
||||
description: "Cabal store path"
|
||||
value: ${{ steps.setup.outputs.cabal-store }}
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Setup Haskell
|
||||
id: setup
|
||||
uses: simplex-chat/setup-haskell-action@v2
|
||||
with:
|
||||
ghc-version: ${{ inputs.ghc_ver }}
|
||||
@@ -37,11 +35,3 @@ runs:
|
||||
distribution: 'corretto'
|
||||
java-version: ${{ inputs.java_ver }}
|
||||
cache: 'gradle'
|
||||
|
||||
- name: Restore cached build
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
${{ inputs.cache_path }}
|
||||
dist-newstyle
|
||||
key: ${{ inputs.os }}-ghc${{ inputs.ghc_ver }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}
|
||||
|
||||
+528
-45
@@ -27,6 +27,14 @@ on:
|
||||
- "scripts/desktop/**"
|
||||
- ".github/**"
|
||||
|
||||
env:
|
||||
opt: ${{ github.event_name == 'pull_request' && 'O0' || 'O2' }}
|
||||
ghc_options: ${{ github.event_name == 'pull_request' && '--ghc-options=-O0' || '' }}
|
||||
release: ${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
branch: ${{ github.base_ref || github.ref_name }}
|
||||
cache_o0: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/stable') }}
|
||||
ZSTD_CLEVEL: 9
|
||||
|
||||
# This workflow uses custom actions (prepare-build and prepare-release) defined in:
|
||||
#
|
||||
# .github/actions/
|
||||
@@ -162,6 +170,12 @@ jobs:
|
||||
name: "ubuntu-${{ matrix.os }}-${{ matrix.arch }} (CLI,Desktop), GHC: ${{ matrix.ghc }}"
|
||||
needs: [maybe-release, variables]
|
||||
runs-on: ${{ matrix.runner }}
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
env:
|
||||
image_cache: ghcr.io/simplex-chat/simplex-chat-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}
|
||||
source_hashes: dist-newstyle/sources.sha256
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-linux-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ghc }}-${{ needs.maybe-release.outputs.libs_tag }}
|
||||
cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/v') && !needs.maybe-release.outputs.libs_tag }}
|
||||
@@ -231,19 +245,22 @@ jobs:
|
||||
shell: bash
|
||||
run: ./scripts/ci/linux_util_free_space.sh
|
||||
|
||||
- name: Restore cached build
|
||||
if: matrix.should_run == true
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cabal/store
|
||||
dist-newstyle
|
||||
key: ubuntu-${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: matrix.should_run == true
|
||||
uses: simplex-chat/docker-setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
if: matrix.should_run == true && env.release != 'true'
|
||||
shell: bash
|
||||
env:
|
||||
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: echo "$GHCR_TOKEN" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
- name: Compute image cache week
|
||||
id: week
|
||||
if: matrix.should_run == true && env.release != 'true'
|
||||
run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build and cache Docker image
|
||||
if: matrix.should_run == true
|
||||
uses: simplex-chat/docker-build-push-action@v6
|
||||
@@ -258,6 +275,8 @@ jobs:
|
||||
GHC=${{ matrix.ghc }}
|
||||
USER_UID=${{ steps.ids.outputs.uid }}
|
||||
USER_GID=${{ steps.ids.outputs.gid }}
|
||||
cache-from: ${{ env.release != 'true' && format('type=registry,ref={0}-{1}', env.image_cache, steps.week.outputs.week) || '' }}
|
||||
cache-to: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && format('type=registry,ref={0}-{1},mode=max,ignore-error=true', env.image_cache, steps.week.outputs.week) || '' }}
|
||||
|
||||
# Docker needs these flags for AppImage build:
|
||||
# --device /dev/fuse
|
||||
@@ -267,16 +286,94 @@ jobs:
|
||||
if: matrix.should_run == true
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p ~/.cabal
|
||||
docker run -t -d \
|
||||
--device /dev/fuse \
|
||||
--cap-add SYS_ADMIN \
|
||||
--security-opt apparmor:unconfined \
|
||||
--name builder \
|
||||
-v ~/.cabal:/root/.cabal \
|
||||
-e CABAL_DIR=/home/builder/.cabal \
|
||||
-v ~/.cabal:/home/builder/.cabal \
|
||||
-v /home/runner/work/_temp:/home/runner/work/_temp \
|
||||
-v ${{ github.workspace }}:/project \
|
||||
build/${{ matrix.os }}:latest
|
||||
|
||||
- name: Compute cache key prefixes
|
||||
id: keys
|
||||
if: matrix.should_run == true && env.release != 'true'
|
||||
run: |
|
||||
toolchain=$(docker exec builder dpkg-query -W -f '${db:Status-Status} ${binary:Package} ${Version}\n' 'binutils*' 'cpp*' 'gcc*' '*-dev' | grep '^installed ' | sha256sum | cut -c1-16)
|
||||
target=ubuntu-${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}-$toolchain
|
||||
echo "deps=deps-$target" >> "$GITHUB_OUTPUT"
|
||||
echo "dist=dist-$target" >> "$GITHUB_OUTPUT"
|
||||
echo "dist_lib=dist-lib-$target" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Restore dependencies from cache
|
||||
id: restore-deps
|
||||
if: matrix.should_run == true && env.release != 'true'
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: ~/.cabal/store
|
||||
key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ steps.keys.outputs.deps }}-${{ env.branch }}-
|
||||
${{ steps.keys.outputs.deps }}-
|
||||
|
||||
- name: Warn if dependencies not in cache
|
||||
if: matrix.should_run == true && env.release != 'true' && steps.restore-deps.outputs.cache-matched-key == ''
|
||||
run: echo "::warning title=Cache miss::${{ steps.keys.outputs.deps }}-${{ env.branch }}"
|
||||
|
||||
- name: Restore source repositories from cache
|
||||
id: restore-src
|
||||
if: matrix.should_run == true && env.release != 'true'
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: dist-newstyle/src
|
||||
key: src-${{ runner.os }}-${{ hashFiles('cabal.project') }}
|
||||
restore-keys: src-${{ runner.os }}-
|
||||
|
||||
- name: Warn if source repositories not in cache
|
||||
if: matrix.should_run == true && env.release != 'true' && steps.restore-src.outputs.cache-matched-key == ''
|
||||
run: echo "::warning title=Cache miss::src-${{ runner.os }}"
|
||||
|
||||
- name: Restore build directory from cache
|
||||
id: restore-build
|
||||
if: matrix.should_run == true && env.release != 'true'
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: |
|
||||
dist-newstyle/*
|
||||
!dist-newstyle/src
|
||||
key: ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-
|
||||
${{ steps.keys.outputs.dist }}-${{ env.opt }}-
|
||||
|
||||
- name: Warn if build directory not in cache
|
||||
if: matrix.should_run == true && env.release != 'true' && steps.restore-build.outputs.cache-matched-key == ''
|
||||
run: echo "::warning title=Cache miss::${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}"
|
||||
|
||||
- name: Restore library build directory from cache
|
||||
id: restore-build-lib
|
||||
if: env.release != 'true' && matrix.libs && needs.maybe-release.outputs.core_changed && matrix.arch == 'x86_64'
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: |
|
||||
dist-lib/*
|
||||
!dist-lib/src
|
||||
key: ${{ steps.keys.outputs.dist_lib }}-${{ env.branch }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ steps.keys.outputs.dist_lib }}-${{ env.branch }}-
|
||||
${{ steps.keys.outputs.dist_lib }}-
|
||||
|
||||
- name: Warn if library build directory not in cache
|
||||
if: steps.restore-build-lib.outcome == 'success' && steps.restore-build-lib.outputs.cache-matched-key == ''
|
||||
run: echo "::warning title=Cache miss::${{ steps.keys.outputs.dist_lib }}-${{ env.branch }}"
|
||||
|
||||
- name: Set old timestamps on unchanged sources (GHC 8.10.7)
|
||||
if: matrix.ghc == '8.10.7' && steps.restore-build.outputs.cache-matched-key != '' && hashFiles(env.source_hashes) != ''
|
||||
run: scripts/ci/source-timestamps.sh restore ${{ env.source_hashes }}
|
||||
|
||||
- name: Prepare cabal.project.local
|
||||
if: matrix.should_run == true
|
||||
shell: bash
|
||||
@@ -285,33 +382,81 @@ jobs:
|
||||
echo "package direct-sqlcipher" >> cabal.project.local
|
||||
echo " flags: +openssl" >> cabal.project.local
|
||||
|
||||
# chmod/git commands are used to workaround permission issues when cache is restored
|
||||
- name: Build CLI
|
||||
id: build-cli
|
||||
if: matrix.should_run == true
|
||||
shell: docker exec -t builder sh -eu {0}
|
||||
run: |
|
||||
cabal clean
|
||||
scripts/ci/prune-old-builds.sh dist-newstyle
|
||||
cabal update
|
||||
cabal build -j --enable-tests
|
||||
cabal build -j --enable-tests ${{ env.ghc_options }}
|
||||
mkdir -p /out
|
||||
for i in simplex-chat simplex-chat-test; do
|
||||
bin=$(find /project/dist-newstyle -name "$i" -type f -executable)
|
||||
chmod +x "$bin"
|
||||
mv "$bin" /out/
|
||||
cp "$bin" /out/
|
||||
done
|
||||
strip /out/simplex-chat
|
||||
|
||||
# Also for pull requests that change the chat core, for the package tests (packages-pr).
|
||||
- name: Build library
|
||||
id: build-lib
|
||||
if: matrix.libs && (needs.maybe-release.outputs.libs_tag || (needs.maybe-release.outputs.core_changed && matrix.arch == 'x86_64'))
|
||||
shell: docker exec -t builder sh -eu {0}
|
||||
run: |
|
||||
SKIP_VLC=1 scripts/desktop/build-lib-linux.sh
|
||||
scripts/ci/prune-old-builds.sh dist-lib
|
||||
mkdir -p dist-lib
|
||||
rm -rf dist-lib/src
|
||||
ln -s ../dist-newstyle/src dist-lib/src
|
||||
DIST_DIR=dist-lib SKIP_VLC=1 scripts/desktop/build-lib-linux.sh
|
||||
name=simplex-chat-libs-linux-${{ matrix.arch }}
|
||||
mkdir -p release-libs/$name/libs
|
||||
find apps/multiplatform/common/src/commonMain/cpp/desktop/libs/linux-${{ matrix.arch }} -maxdepth 1 -type f -exec cp {} release-libs/$name/libs/ \;
|
||||
(cd release-libs/$name && zip -r ../$name.zip libs)
|
||||
|
||||
- name: Record source hashes (GHC 8.10.7)
|
||||
if: matrix.should_run == true && matrix.ghc == '8.10.7'
|
||||
shell: docker exec -t builder sh -eu {0}
|
||||
run: scripts/ci/source-timestamps.sh record ${{ env.source_hashes }}
|
||||
|
||||
- name: Compute dependency cache key
|
||||
id: hash-deps
|
||||
if: matrix.should_run == true && env.release != 'true'
|
||||
shell: bash
|
||||
run: echo "hash=$(ls ~/.cabal/store/ghc-*/ | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Save dependencies to cache
|
||||
if: matrix.should_run == true && env.release != 'true' && steps.restore-deps.outputs.cache-matched-key != format('{0}-{1}-{2}', steps.keys.outputs.deps, env.branch, steps.hash-deps.outputs.hash)
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: ~/.cabal/store
|
||||
key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ steps.hash-deps.outputs.hash }}
|
||||
|
||||
- name: Save source repositories to cache
|
||||
if: matrix.should_run == true && env.release != 'true' && matrix.libs && matrix.arch == 'x86_64' && steps.restore-src.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: dist-newstyle/src
|
||||
key: src-${{ runner.os }}-${{ hashFiles('cabal.project') }}
|
||||
|
||||
- name: Save build directory to cache
|
||||
if: matrix.should_run == true && env.release != 'true' && steps.restore-build.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: |
|
||||
dist-newstyle/*
|
||||
!dist-newstyle/src
|
||||
key: ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-${{ github.sha }}
|
||||
|
||||
- name: Save library build directory to cache
|
||||
if: steps.build-lib.outcome == 'success' && env.release != 'true' && steps.restore-build-lib.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: |
|
||||
dist-lib/*
|
||||
!dist-lib/src
|
||||
key: ${{ steps.keys.outputs.dist_lib }}-${{ env.branch }}-${{ github.sha }}
|
||||
|
||||
- name: Upload library artifact
|
||||
if: matrix.libs && needs.maybe-release.outputs.core_changed && matrix.arch == 'x86_64'
|
||||
uses: actions/upload-artifact@v7
|
||||
@@ -437,13 +582,6 @@ jobs:
|
||||
github_ref: ${{ github.ref }}
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Fix permissions for cache
|
||||
if: matrix.should_run == true
|
||||
shell: bash
|
||||
run: |
|
||||
sudo chmod -R 777 dist-newstyle ~/.cabal
|
||||
sudo chown -R $(id -u):$(id -g) dist-newstyle ~/.cabal
|
||||
|
||||
- name: Mount RAM disk for tests
|
||||
if: matrix.should_run == true && matrix.arch == 'x86_64'
|
||||
shell: bash
|
||||
@@ -474,6 +612,51 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Remove -O2 build directory
|
||||
id: remove-o2
|
||||
if: env.cache_o0 == 'true' && !cancelled() && steps.build-cli.outcome == 'success'
|
||||
shell: docker exec -t builder sh -eu {0}
|
||||
run: find dist-newstyle -mindepth 1 -maxdepth 1 ! -name src -exec rm -rf {} +
|
||||
|
||||
- name: Restore -O0 build directory from cache
|
||||
id: restore-build-o0
|
||||
if: ${{ !cancelled() && steps.remove-o2.outcome == 'success' }}
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: |
|
||||
dist-newstyle/*
|
||||
!dist-newstyle/src
|
||||
key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-
|
||||
${{ steps.keys.outputs.dist }}-O0-
|
||||
|
||||
- name: Set old timestamps on unchanged sources for -O0 build (GHC 8.10.7)
|
||||
if: ${{ !cancelled() && matrix.ghc == '8.10.7' && steps.restore-build-o0.outputs.cache-matched-key != '' && hashFiles(env.source_hashes) != '' }}
|
||||
run: scripts/ci/source-timestamps.sh restore ${{ env.source_hashes }}
|
||||
|
||||
- name: Build at -O0 for pull request caches
|
||||
id: build-o0
|
||||
if: ${{ !cancelled() && steps.remove-o2.outcome == 'success' }}
|
||||
shell: docker exec -t builder sh -eu {0}
|
||||
run: |
|
||||
scripts/ci/prune-old-builds.sh dist-newstyle
|
||||
cabal build -j --enable-tests --ghc-options=-O0
|
||||
|
||||
- name: Record source hashes for -O0 build (GHC 8.10.7)
|
||||
if: ${{ !cancelled() && matrix.ghc == '8.10.7' && steps.build-o0.outcome == 'success' }}
|
||||
shell: docker exec -t builder sh -eu {0}
|
||||
run: scripts/ci/source-timestamps.sh record ${{ env.source_hashes }}
|
||||
|
||||
- name: Save -O0 build directory to cache
|
||||
if: ${{ !cancelled() && steps.build-o0.outcome == 'success' && steps.restore-build-o0.outputs.cache-hit != 'true' }}
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: |
|
||||
dist-newstyle/*
|
||||
!dist-newstyle/src
|
||||
key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }}
|
||||
|
||||
# =================================
|
||||
# Linux PostgreSQL Library Build
|
||||
# =================================
|
||||
@@ -497,14 +680,6 @@ jobs:
|
||||
shell: bash
|
||||
run: ./scripts/ci/linux_util_free_space.sh
|
||||
|
||||
- name: Restore cached build
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cabal/store
|
||||
dist-newstyle
|
||||
key: ubuntu-22.04-x86_64-postgres-ghc${{ needs.variables.outputs.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: simplex-chat/docker-setup-buildx-action@v3
|
||||
|
||||
@@ -525,13 +700,35 @@ jobs:
|
||||
- name: Start container
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p ~/.cabal
|
||||
docker run -t -d \
|
||||
--name builder \
|
||||
-v ~/.cabal:/root/.cabal \
|
||||
-e CABAL_DIR=/home/builder/.cabal \
|
||||
-v ~/.cabal:/home/builder/.cabal \
|
||||
-v /home/runner/work/_temp:/home/runner/work/_temp \
|
||||
-v ${{ github.workspace }}:/project \
|
||||
build/22.04:latest
|
||||
|
||||
- name: Compute cache key prefix
|
||||
id: keys
|
||||
run: |
|
||||
toolchain=$(docker exec builder dpkg-query -W -f '${db:Status-Status} ${binary:Package} ${Version}\n' 'binutils*' 'cpp*' 'gcc*' '*-dev' | grep '^installed ' | sha256sum | cut -c1-16)
|
||||
echo "deps=deps-ubuntu-22.04-x86_64-postgres-ghc${{ needs.variables.outputs.GHC_VER }}-$toolchain" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Restore dependencies from cache
|
||||
id: restore-deps
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: ~/.cabal/store
|
||||
key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ steps.keys.outputs.deps }}-${{ env.branch }}-
|
||||
${{ steps.keys.outputs.deps }}-
|
||||
|
||||
- name: Warn if dependencies not in cache
|
||||
if: steps.restore-deps.outputs.cache-matched-key == ''
|
||||
run: echo "::warning title=Cache miss::${{ steps.keys.outputs.deps }}-${{ env.branch }}"
|
||||
|
||||
- name: Prepare cabal.project.local
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -542,10 +739,21 @@ jobs:
|
||||
- name: Build postgres library
|
||||
shell: docker exec -t builder sh -eu {0}
|
||||
run: |
|
||||
cabal clean
|
||||
cabal update
|
||||
scripts/desktop/build-lib-linux.sh postgres
|
||||
|
||||
- name: Compute dependency cache key
|
||||
id: hash-deps
|
||||
shell: bash
|
||||
run: echo "hash=$(ls ~/.cabal/store/ghc-*/ | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Save dependencies to cache
|
||||
if: steps.restore-deps.outputs.cache-matched-key != format('{0}-{1}-{2}', steps.keys.outputs.deps, env.branch, steps.hash-deps.outputs.hash)
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: ~/.cabal/store
|
||||
key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ steps.hash-deps.outputs.hash }}
|
||||
|
||||
- name: Copy libs from container
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -565,12 +773,6 @@ jobs:
|
||||
files: postgres-libs/*.zip
|
||||
token: ${{ secrets.NODEJS_REPO_TOKEN }}
|
||||
|
||||
- name: Fix permissions for cache
|
||||
shell: bash
|
||||
run: |
|
||||
sudo chmod -R 777 dist-newstyle ~/.cabal
|
||||
sudo chown -R $(id -u):$(id -g) dist-newstyle ~/.cabal
|
||||
|
||||
# =========================
|
||||
# MacOS Build
|
||||
# =========================
|
||||
@@ -604,14 +806,49 @@ jobs:
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Point Command Line Tools SDK to Xcode SDK
|
||||
shell: bash
|
||||
run: sudo ln -sfn "$(xcrun --sdk macosx --show-sdk-path)" /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
|
||||
|
||||
- name: Compute toolchain cache key
|
||||
id: toolchain-key
|
||||
if: env.release != 'true'
|
||||
shell: bash
|
||||
run: echo "key=ghcup-${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}-cabal3.10.2.0-$({ sw_vers -productVersion; clang --version; } | shasum -a 256 | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Restore toolchain from cache
|
||||
id: restore-toolchain
|
||||
if: env.release != 'true'
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: |
|
||||
~/.ghcup/ghc/${{ matrix.ghc }}
|
||||
~/.ghcup/bin/*-${{ matrix.ghc }}
|
||||
~/.ghcup/bin/cabal-3.10.2.0
|
||||
key: ${{ steps.toolchain-key.outputs.key }}
|
||||
|
||||
- name: Warn if toolchain not in cache
|
||||
if: env.release != 'true' && steps.restore-toolchain.outputs.cache-hit != 'true'
|
||||
run: echo "::warning title=Cache miss::${{ steps.toolchain-key.outputs.key }}"
|
||||
|
||||
- name: Prepare build
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-build
|
||||
with:
|
||||
java_ver: ${{ needs.variables.outputs.JAVA_VER }}
|
||||
ghc_ver: ${{ matrix.ghc }}
|
||||
os: ${{ matrix.os }}
|
||||
github_ref: ${{ github.ref }}
|
||||
|
||||
- name: Save toolchain to cache
|
||||
if: env.release != 'true' && steps.restore-toolchain.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: |
|
||||
~/.ghcup/ghc/${{ matrix.ghc }}
|
||||
~/.ghcup/bin/*-${{ matrix.ghc }}
|
||||
~/.ghcup/bin/cabal-3.10.2.0
|
||||
key: ${{ steps.toolchain-key.outputs.key }}
|
||||
|
||||
- name: Install OpenSSL
|
||||
run: brew install openssl@3.0
|
||||
|
||||
@@ -628,15 +865,100 @@ jobs:
|
||||
echo " extra-lib-dirs: ${{ matrix.openssl_dir }}/openssl@3.0/lib" >> cabal.project.local
|
||||
echo " flags: +openssl" >> cabal.project.local
|
||||
|
||||
- name: Compute cache key prefixes
|
||||
id: keys
|
||||
if: env.release != 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
toolchain=$({ sw_vers -productVersion; clang --version; } | shasum -a 256 | cut -c1-16)
|
||||
target=${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}-$toolchain
|
||||
echo "deps=deps-$target" >> "$GITHUB_OUTPUT"
|
||||
echo "dist=dist-$target" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Restore dependencies from cache
|
||||
id: restore-deps
|
||||
if: env.release != 'true'
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: ${{ steps.prepare.outputs.cabal-store }}
|
||||
key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ steps.keys.outputs.deps }}-${{ env.branch }}-
|
||||
${{ steps.keys.outputs.deps }}-
|
||||
|
||||
- name: Warn if dependencies not in cache
|
||||
if: env.release != 'true' && steps.restore-deps.outputs.cache-matched-key == ''
|
||||
run: echo "::warning title=Cache miss::${{ steps.keys.outputs.deps }}-${{ env.branch }}"
|
||||
|
||||
- name: Restore source repositories from cache
|
||||
id: restore-src
|
||||
if: env.release != 'true'
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: dist-newstyle/src
|
||||
key: src-${{ runner.os }}-${{ hashFiles('cabal.project') }}
|
||||
restore-keys: src-${{ runner.os }}-
|
||||
|
||||
- name: Warn if source repositories not in cache
|
||||
if: env.release != 'true' && steps.restore-src.outputs.cache-matched-key == ''
|
||||
run: echo "::warning title=Cache miss::src-${{ runner.os }}"
|
||||
|
||||
- name: Restore build directory from cache
|
||||
id: restore-build
|
||||
if: env.release != 'true' && !needs.maybe-release.outputs.libs_tag
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: |
|
||||
dist-newstyle/*
|
||||
!dist-newstyle/src
|
||||
key: ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-
|
||||
${{ steps.keys.outputs.dist }}-${{ env.opt }}-
|
||||
|
||||
- name: Warn if build directory not in cache
|
||||
if: steps.restore-build.outcome == 'success' && steps.restore-build.outputs.cache-matched-key == ''
|
||||
run: echo "::warning title=Cache miss::${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}"
|
||||
|
||||
- name: Build CLI
|
||||
id: mac_cli_build
|
||||
shell: bash
|
||||
run: |
|
||||
cabal build -j --enable-tests
|
||||
path=$(cabal list-bin simplex-chat)
|
||||
scripts/ci/prune-old-builds.sh dist-newstyle
|
||||
cabal build -j --enable-tests ${{ env.ghc_options }}
|
||||
path=$(cabal list-bin simplex-chat | tail -n 1)
|
||||
echo "bin_path=$path" >> $GITHUB_OUTPUT
|
||||
echo "bin_hash=$(echo SHA2-256\(${{ matrix.cli_asset_name }}\)= $(openssl sha256 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Compute dependency cache key
|
||||
id: hash-deps
|
||||
if: env.release != 'true'
|
||||
shell: bash
|
||||
run: echo "hash=$(ls '${{ steps.prepare.outputs.cabal-store }}'/ghc-*/ | shasum -a 256 | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Save dependencies to cache
|
||||
if: env.release != 'true' && steps.restore-deps.outputs.cache-matched-key != format('{0}-{1}-{2}', steps.keys.outputs.deps, env.branch, steps.hash-deps.outputs.hash)
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: ${{ steps.prepare.outputs.cabal-store }}
|
||||
key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ steps.hash-deps.outputs.hash }}
|
||||
|
||||
- name: Save source repositories to cache
|
||||
if: env.release != 'true' && matrix.arch == 'x86_64' && steps.restore-src.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: dist-newstyle/src
|
||||
key: src-${{ runner.os }}-${{ hashFiles('cabal.project') }}
|
||||
|
||||
- name: Save build directory to cache
|
||||
if: env.release != 'true' && steps.restore-build.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: |
|
||||
dist-newstyle/*
|
||||
!dist-newstyle/src
|
||||
key: ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-${{ github.sha }}
|
||||
|
||||
- name: Upload CLI
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
uses: ./.github/actions/prepare-release
|
||||
@@ -673,6 +995,7 @@ jobs:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Run tests
|
||||
if: matrix.arch == 'aarch64'
|
||||
timeout-minutes: 120
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -680,7 +1003,7 @@ jobs:
|
||||
attempts=1
|
||||
${{ (github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }} && attempts=3
|
||||
while [ "$i" -le "$attempts" ]; do
|
||||
if cabal test --test-show-details=direct; then
|
||||
if cabal test --test-show-details=direct ${{ env.ghc_options }}; then
|
||||
break
|
||||
else
|
||||
echo "Attempt $i failed, retrying..."
|
||||
@@ -713,6 +1036,42 @@ jobs:
|
||||
files: release-libs/*.zip
|
||||
token: ${{ secrets.NODEJS_REPO_TOKEN }}
|
||||
|
||||
- name: Remove -O2 build directory
|
||||
id: remove-o2
|
||||
if: env.cache_o0 == 'true' && !cancelled() && steps.mac_cli_build.outcome == 'success'
|
||||
shell: bash
|
||||
run: find dist-newstyle -mindepth 1 -maxdepth 1 ! -name src -exec rm -rf {} +
|
||||
|
||||
- name: Restore -O0 build directory from cache
|
||||
id: restore-build-o0
|
||||
if: ${{ !cancelled() && steps.remove-o2.outcome == 'success' }}
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: |
|
||||
dist-newstyle/*
|
||||
!dist-newstyle/src
|
||||
key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-
|
||||
${{ steps.keys.outputs.dist }}-O0-
|
||||
|
||||
- name: Build at -O0 for pull request caches
|
||||
id: build-o0
|
||||
if: ${{ !cancelled() && steps.remove-o2.outcome == 'success' }}
|
||||
shell: bash
|
||||
run: |
|
||||
scripts/ci/prune-old-builds.sh dist-newstyle
|
||||
cabal build -j --enable-tests --ghc-options=-O0
|
||||
|
||||
- name: Save -O0 build directory to cache
|
||||
if: ${{ !cancelled() && steps.build-o0.outcome == 'success' && steps.restore-build-o0.outputs.cache-hit != 'true' }}
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: |
|
||||
dist-newstyle/*
|
||||
!dist-newstyle/src
|
||||
key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }}
|
||||
|
||||
# =========================
|
||||
# Windows Build
|
||||
# =========================
|
||||
@@ -739,12 +1098,11 @@ jobs:
|
||||
submodules: recursive
|
||||
|
||||
- name: Prepare build
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-build
|
||||
with:
|
||||
java_ver: ${{ needs.variables.outputs.JAVA_VER }}
|
||||
ghc_ver: ${{ matrix.ghc }}
|
||||
os: ${{ matrix.os }}
|
||||
cache_path: "C:/cabal"
|
||||
github_ref: ${{ github.ref }}
|
||||
|
||||
- name: Configure pagefile (Windows)
|
||||
@@ -768,6 +1126,61 @@ jobs:
|
||||
toolchain:p
|
||||
cmake:p
|
||||
|
||||
- name: Compute cache key prefixes
|
||||
id: keys
|
||||
if: env.release != 'true'
|
||||
shell: msys2 {0}
|
||||
run: |
|
||||
toolchain=$(gcc --version | sha256sum | cut -c1-16)
|
||||
target=windows-x86_64-ghc${{ matrix.ghc }}-$toolchain
|
||||
echo "deps=deps-$target" >> "$GITHUB_OUTPUT"
|
||||
echo "dist=dist-$target" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Restore dependencies from cache
|
||||
id: restore-deps
|
||||
if: env.release != 'true'
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: ${{ steps.prepare.outputs.cabal-store }}
|
||||
key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ steps.keys.outputs.deps }}-${{ env.branch }}-
|
||||
${{ steps.keys.outputs.deps }}-
|
||||
|
||||
- name: Warn if dependencies not in cache
|
||||
if: env.release != 'true' && steps.restore-deps.outputs.cache-matched-key == ''
|
||||
run: echo "::warning title=Cache miss::${{ steps.keys.outputs.deps }}-${{ env.branch }}"
|
||||
|
||||
- name: Restore source repositories from cache
|
||||
id: restore-src
|
||||
if: env.release != 'true'
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: dist-newstyle/src
|
||||
key: src-${{ runner.os }}-${{ hashFiles('cabal.project') }}
|
||||
restore-keys: src-${{ runner.os }}-
|
||||
|
||||
- name: Warn if source repositories not in cache
|
||||
if: env.release != 'true' && steps.restore-src.outputs.cache-matched-key == ''
|
||||
run: echo "::warning title=Cache miss::src-${{ runner.os }}"
|
||||
|
||||
- name: Restore build directory from cache
|
||||
id: restore-build
|
||||
if: env.release != 'true' && !needs.maybe-release.outputs.libs_tag
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: |
|
||||
dist-newstyle/*
|
||||
!dist-newstyle/src
|
||||
key: ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-
|
||||
${{ steps.keys.outputs.dist }}-${{ env.opt }}-
|
||||
|
||||
- name: Warn if build directory not in cache
|
||||
if: steps.restore-build.outcome == 'success' && steps.restore-build.outputs.cache-matched-key == ''
|
||||
run: echo "::warning title=Cache miss::${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}"
|
||||
|
||||
# rm -rf dist-newstyle/src/{direct-sq,simplexmq}* is here because of the bug in cabal's dependency which prevents second build from finishing
|
||||
# (simplexmq is removed because cabal cannot delete its read-only git submodule pack files - blst, libbbs - on Windows)
|
||||
- name: Build CLI
|
||||
@@ -784,14 +1197,44 @@ jobs:
|
||||
echo " extra-include-dirs: $openssl_windows_style_path\include" >> cabal.project.local
|
||||
echo " extra-lib-dirs: $openssl_windows_style_path" >> cabal.project.local
|
||||
|
||||
scripts/ci/prune-old-builds.sh dist-newstyle
|
||||
rm -rf dist-newstyle/src/direct-sq* dist-newstyle/src/simplexmq*
|
||||
sed -i "s/, unix /--, unix /" simplex-chat.cabal
|
||||
cabal build -j --enable-tests
|
||||
cabal build -j --enable-tests ${{ env.ghc_options }}
|
||||
rm -rf dist-newstyle/src/direct-sq* dist-newstyle/src/simplexmq*
|
||||
path=$(cabal list-bin simplex-chat | tail -n 1)
|
||||
echo "bin_path=$path" >> $GITHUB_OUTPUT
|
||||
echo "bin_hash=$(echo SHA2-256\(${{ matrix.cli_asset_name }}\)= $(openssl sha256 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Compute dependency cache key
|
||||
id: hash-deps
|
||||
if: env.release != 'true'
|
||||
shell: msys2 {0}
|
||||
run: echo "hash=$(ls "$(cygpath -u '${{ steps.prepare.outputs.cabal-store }}')"/ghc-*/ | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Save dependencies to cache
|
||||
if: env.release != 'true' && steps.restore-deps.outputs.cache-matched-key != format('{0}-{1}-{2}', steps.keys.outputs.deps, env.branch, steps.hash-deps.outputs.hash)
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: ${{ steps.prepare.outputs.cabal-store }}
|
||||
key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ steps.hash-deps.outputs.hash }}
|
||||
|
||||
- name: Save source repositories to cache
|
||||
if: env.release != 'true' && steps.restore-src.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: dist-newstyle/src
|
||||
key: src-${{ runner.os }}-${{ hashFiles('cabal.project') }}
|
||||
|
||||
- name: Save build directory to cache
|
||||
if: env.release != 'true' && steps.restore-build.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: |
|
||||
dist-newstyle/*
|
||||
!dist-newstyle/src
|
||||
key: ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-${{ github.sha }}
|
||||
|
||||
- name: Build library
|
||||
if: needs.maybe-release.outputs.libs_tag
|
||||
shell: msys2 {0}
|
||||
@@ -848,6 +1291,46 @@ jobs:
|
||||
github_ref: ${{ github.ref }}
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Remove -O2 build directory
|
||||
id: remove-o2
|
||||
if: env.cache_o0 == 'true' && !needs.maybe-release.outputs.libs_tag && !cancelled() && steps.windows_cli_build.outcome == 'success'
|
||||
shell: msys2 {0}
|
||||
run: find dist-newstyle -mindepth 1 -maxdepth 1 ! -name src -exec rm -rf {} +
|
||||
|
||||
- name: Restore -O0 build directory from cache
|
||||
id: restore-build-o0
|
||||
if: ${{ !cancelled() && steps.remove-o2.outcome == 'success' }}
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: |
|
||||
dist-newstyle/*
|
||||
!dist-newstyle/src
|
||||
key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-
|
||||
${{ steps.keys.outputs.dist }}-O0-
|
||||
|
||||
- name: Build at -O0 for pull request caches
|
||||
id: build-o0
|
||||
if: ${{ !cancelled() && steps.remove-o2.outcome == 'success' }}
|
||||
shell: msys2 {0}
|
||||
run: |
|
||||
export PATH=$PATH:/c/ghcup/bin:$(echo /c/tools/ghc-*/bin || echo)
|
||||
scripts/desktop/prepare-openssl-windows.sh
|
||||
scripts/ci/prune-old-builds.sh dist-newstyle
|
||||
rm -rf dist-newstyle/src/direct-sq* dist-newstyle/src/simplexmq*
|
||||
cabal build -j --enable-tests --ghc-options=-O0
|
||||
rm -rf dist-newstyle/src/direct-sq* dist-newstyle/src/simplexmq*
|
||||
|
||||
- name: Save -O0 build directory to cache
|
||||
if: ${{ !cancelled() && steps.build-o0.outcome == 'success' && steps.restore-build-o0.outputs.cache-hit != 'true' }}
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: |
|
||||
dist-newstyle/*
|
||||
!dist-newstyle/src
|
||||
key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }}
|
||||
|
||||
# =========================
|
||||
# Packages release
|
||||
# =========================
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
# CI build cache
|
||||
|
||||
Repository: simplex-chat/simplex-chat.
|
||||
Branch: `ep/ci-build-cache`.
|
||||
Model: simplexmq at `e91711c7`:
|
||||
- `.github/workflows/build.yml`
|
||||
- `scripts/ci/source-timestamps.sh`
|
||||
- `scripts/ci/prune-old-builds.sh`
|
||||
|
||||
Repository cache limit: 80 GB.
|
||||
|
||||
## Files
|
||||
|
||||
- `.github/workflows/build.yml`
|
||||
- `.github/actions/prepare-build/action.yml`
|
||||
- `scripts/ci/source-timestamps.sh`: copied from simplexmq.
|
||||
- `scripts/ci/prune-old-builds.sh`: new.
|
||||
- `scripts/desktop/build-lib-linux.sh`: build directory from `DIST_DIR`, default `dist-newstyle`.
|
||||
|
||||
## Job environment
|
||||
|
||||
Jobs: build-linux, build-linux-postgres, build-macos, build-windows.
|
||||
|
||||
opt
|
||||
- pull request: `O0`
|
||||
- other events: `O2`
|
||||
|
||||
ghc_options
|
||||
- pull request: `--ghc-options=-O0`
|
||||
- other events: empty
|
||||
|
||||
release
|
||||
- `v*` tag: `true`
|
||||
|
||||
branch
|
||||
- pull request: base branch
|
||||
- other events: ref name
|
||||
|
||||
cache_o0
|
||||
- push to master or stable: `true`
|
||||
|
||||
ZSTD_CLEVEL
|
||||
- `9`: zstd level of cache archives
|
||||
|
||||
## Cache entries
|
||||
|
||||
deps
|
||||
- path: cabal store
|
||||
- build-linux, build-linux-postgres: `~/.cabal/store`
|
||||
- build-macos, build-windows: output `cabal-store` of prepare-build
|
||||
- key: `deps-<target>-<branch>-<store hash>`
|
||||
- restore key 1: `deps-<target>-<branch>-`
|
||||
- restore key 2: `deps-<target>-`
|
||||
- saved when the store hash differs from the matched key
|
||||
|
||||
toolchain
|
||||
- job: build-macos
|
||||
- path:
|
||||
- `~/.ghcup/ghc/<ghc>`
|
||||
- `~/.ghcup/bin/*-<ghc>`
|
||||
- `~/.ghcup/bin/cabal-3.10.2.0`
|
||||
- key: `ghcup-<os>-<arch>-ghc<ghc>-cabal3.10.2.0-<toolchain>`
|
||||
- restored before "Prepare build"
|
||||
- saved after "Prepare build" when the key is not matched
|
||||
|
||||
src
|
||||
- jobs: build-linux, build-macos, build-windows
|
||||
- path: `dist-newstyle/src`
|
||||
- key: `src-<runner OS>-<hash of cabal.project>`
|
||||
- restore key: `src-<runner OS>-`
|
||||
- saved when the matched key differs from the key, by:
|
||||
- build-linux: ubuntu-22.04 x86_64, GHC 9.6.3
|
||||
- build-macos: x86_64
|
||||
- build-windows
|
||||
|
||||
dist
|
||||
- jobs: build-linux, build-macos, build-windows
|
||||
- path:
|
||||
- `dist-newstyle/*`
|
||||
- `!dist-newstyle/src`
|
||||
- key: `dist-<target>-<opt>-<branch>-<sha>`
|
||||
- restore key 1: `dist-<target>-<opt>-<branch>-`
|
||||
- restore key 2: `dist-<target>-<opt>-`
|
||||
- restored except on libs release pushes in build-macos and build-windows
|
||||
- saved when the matched key differs from the key
|
||||
|
||||
dist-lib
|
||||
- job: build-linux
|
||||
- path:
|
||||
- `dist-lib/*`
|
||||
- `!dist-lib/src`
|
||||
- key: `dist-lib-<target>-<branch>-<sha>`
|
||||
- restore key 1: `dist-lib-<target>-<branch>-`
|
||||
- restore key 2: `dist-lib-<target>-`
|
||||
- restored on pull requests that change the chat core, `libs` matrix entries, x86_64
|
||||
- saved when the library build succeeds and the matched key differs from the key
|
||||
|
||||
store hash
|
||||
- first 16 hex characters of sha256 of the store's package directory listing
|
||||
|
||||
target
|
||||
- build-linux: `ubuntu-<os>-<arch>-ghc<ghc>-<toolchain>`
|
||||
- build-linux-postgres: `ubuntu-22.04-x86_64-postgres-ghc<ghc>-<toolchain>`
|
||||
- build-macos: `<os>-<arch>-ghc<ghc>-<toolchain>`
|
||||
- build-windows: `windows-x86_64-ghc<ghc>-<toolchain>`
|
||||
|
||||
toolchain
|
||||
- Linux: first 16 hex characters of sha256 of the installed `binutils*`, `cpp*`, `gcc*` and `*-dev` package versions in the container
|
||||
- macOS: first 16 hex characters of sha256 of `sw_vers -productVersion` and `clang --version`
|
||||
- Windows: first 16 hex characters of sha256 of MSYS2 `gcc --version`
|
||||
|
||||
Release tags: caches are neither restored nor saved.
|
||||
Each missed restore: `::warning title=Cache miss::` annotation.
|
||||
|
||||
## build-linux
|
||||
|
||||
Job permissions
|
||||
- `contents: write`
|
||||
- `packages: write`
|
||||
|
||||
Build image
|
||||
- registry cache: `ghcr.io/simplex-chat/simplex-chat-build-cache:<os>-<arch>-ghc<ghc>-<ISO week>`
|
||||
- read: every build except release tags
|
||||
- written: pushes to master, `mode=max`, `ignore-error=true`
|
||||
- login: `docker login ghcr.io` with `GITHUB_TOKEN`, every build except release tags
|
||||
|
||||
Container
|
||||
- volume: `~/.cabal:/home/builder/.cabal`
|
||||
- environment: `CABAL_DIR=/home/builder/.cabal`
|
||||
|
||||
Steps after "Start container":
|
||||
1. Compute cache keys.
|
||||
2. Restore deps.
|
||||
3. Restore src.
|
||||
4. Restore dist.
|
||||
5. Restore dist-lib.
|
||||
6. GHC 8.10.7, dist restored: `scripts/ci/source-timestamps.sh restore dist-newstyle/sources.sha256`.
|
||||
7. Prepare cabal.project.local: unchanged.
|
||||
8. Build CLI:
|
||||
- `scripts/ci/prune-old-builds.sh dist-newstyle`
|
||||
- `cabal update`
|
||||
- `cabal build -j --enable-tests <ghc_options>`
|
||||
- binaries copied to `/out`
|
||||
- `cabal clean` removed
|
||||
9. Build library:
|
||||
- `scripts/ci/prune-old-builds.sh dist-lib`
|
||||
- `dist-lib/src`: symlink to `../dist-newstyle/src`
|
||||
- `DIST_DIR=dist-lib`
|
||||
10. GHC 8.10.7: `scripts/ci/source-timestamps.sh record dist-newstyle/sources.sha256`.
|
||||
11. Save deps.
|
||||
12. Save src.
|
||||
13. Save dist.
|
||||
14. Save dist-lib.
|
||||
15. Release and test steps: unchanged.
|
||||
16. -O0 refresh, when `cache_o0`, the run is not cancelled and "Build CLI" succeeded:
|
||||
- remove `dist-newstyle` contents except `src`
|
||||
- restore dist with opt `O0`
|
||||
- GHC 8.10.7: restore source timestamps
|
||||
- `scripts/ci/prune-old-builds.sh dist-newstyle`
|
||||
- `cabal build -j --enable-tests --ghc-options=-O0`
|
||||
- GHC 8.10.7: record source hashes
|
||||
- save dist with opt `O0`
|
||||
|
||||
"Fix permissions for cache": removed.
|
||||
|
||||
## build-linux-postgres
|
||||
|
||||
- condition: unchanged
|
||||
- caches: deps
|
||||
- container: volume and environment as in build-linux
|
||||
- `cabal clean` removed
|
||||
- "Fix permissions for cache": removed
|
||||
|
||||
## build-macos and build-windows
|
||||
|
||||
prepare-build action
|
||||
- inputs `os` and `cache_path`: removed
|
||||
- "Restore cached build": removed
|
||||
- output `cabal-store`: output `cabal-store` of setup-haskell-action
|
||||
|
||||
Steps:
|
||||
1. build-macos: restore toolchain.
|
||||
2. Prepare build.
|
||||
3. build-macos: save toolchain.
|
||||
4. Compute cache keys.
|
||||
5. Restore deps.
|
||||
6. Restore src.
|
||||
7. Restore dist.
|
||||
8. Build CLI:
|
||||
- `scripts/ci/prune-old-builds.sh dist-newstyle`
|
||||
- `<ghc_options>`
|
||||
9. Save deps.
|
||||
10. Save src.
|
||||
11. Save dist.
|
||||
12. build-macos "Run tests": aarch64 only, `cabal test` with `<ghc_options>`.
|
||||
13. Other steps: unchanged.
|
||||
14. -O0 refresh as in build-linux, after all other steps.
|
||||
|
||||
build-macos
|
||||
- step before "Prepare build": `/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk` linked to `xcrun --sdk macosx --show-sdk-path`
|
||||
- Build CLI: `bin_path` from the last line of `cabal list-bin simplex-chat`
|
||||
|
||||
build-windows
|
||||
- cache key and store hash steps: `msys2` shell
|
||||
- store hash: store path converted with `cygpath -u`
|
||||
- keeps the `dist-newstyle/src` removals
|
||||
- -O0 refresh:
|
||||
- skipped on libs release pushes
|
||||
- runs `scripts/desktop/prepare-openssl-windows.sh` before the build
|
||||
|
||||
## prune-old-builds.sh
|
||||
|
||||
Arguments: build directories.
|
||||
In each `<dir>/build/*/*/`:
|
||||
- `simplex-chat-<version>`: deleted unless the version equals `simplex-chat.cabal`
|
||||
|
||||
Version: carriage returns removed.
|
||||
|
||||
## Verification
|
||||
|
||||
Baseline, run 38730229359:
|
||||
- Linux "Restore cached build": 0-4 s
|
||||
- Linux "Build and cache Docker image": 3.5-5 min
|
||||
- Linux "Build CLI", GHC 9.6.3: 23-30 min
|
||||
- Linux "Build CLI", GHC 8.10.7: 80 min
|
||||
- macOS "Build CLI": 1.5-30 min
|
||||
- Windows "Build CLI": 23 min
|
||||
|
||||
Test pull request: `ep/ci-build-cache` into master.
|
||||
|
||||
On the second run of the test pull request:
|
||||
1. build-linux, build-macos and build-windows restore deps and dist.
|
||||
2. "Build CLI" compiles only modules changed since the first run.
|
||||
3. GHC 8.10.7 "Build CLI" compiles only changed modules.
|
||||
4. Tests pass.
|
||||
|
||||
After merge, on the first push to master:
|
||||
1. build-linux, build-macos and build-windows save deps, dist with opt `O2` and dist with opt `O0`.
|
||||
2. build-linux writes the image cache.
|
||||
|
||||
On the next pull request:
|
||||
1. build-linux, build-macos and build-windows restore dist with opt `O0` from master.
|
||||
2. build-linux reads the image cache.
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
version=$(awk '/^version:/ {print $2}' simplex-chat.cabal | tr -d '\r')
|
||||
for dir in "$@"; do
|
||||
if [ -d "$dir/build" ]; then
|
||||
find "$dir/build" -mindepth 3 -maxdepth 3 -type d -name 'simplex-chat-*' ! -name "simplex-chat-$version" -exec rm -rf {} +
|
||||
fi
|
||||
done
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
command=$1
|
||||
hashes=$2
|
||||
|
||||
sources() {
|
||||
git ls-files -z '*.hs' '*.hsc'
|
||||
}
|
||||
|
||||
case "$command" in
|
||||
record)
|
||||
sources | xargs -0 sha256sum > "$hashes"
|
||||
;;
|
||||
restore)
|
||||
sources | xargs -0 touch
|
||||
while read -r hash file; do
|
||||
if [ -f "$file" ] && [ "$(sha256sum "$file" | cut -d' ' -f1)" = "$hash" ]; then
|
||||
touch -d 2000-01-01 "$file"
|
||||
fi
|
||||
done < "$hashes"
|
||||
;;
|
||||
*)
|
||||
echo "usage: $0 record|restore <hashes file>" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -19,7 +19,8 @@ fi
|
||||
|
||||
root_dir="$(dirname "$(dirname "$(readlink "$0")")")"
|
||||
cd $root_dir
|
||||
BUILD_DIR=dist-newstyle/build/$ARCH-$OS/ghc-${GHC_VERSION}/simplex-chat-*
|
||||
DIST_DIR="${DIST_DIR:-dist-newstyle}"
|
||||
BUILD_DIR=$DIST_DIR/build/$ARCH-$OS/ghc-${GHC_VERSION}/simplex-chat-*
|
||||
|
||||
exports=( $(sed 's/foreign export ccall "chat_migrate_init_key"//' src/Simplex/Chat/Mobile.hs | sed 's/foreign export ccall "chat_reopen_store"//' |grep "foreign export ccall" | cut -d '"' -f2) )
|
||||
for elem in "${exports[@]}"; do count=$(grep -R "$elem$" libsimplex.dll.def | wc -l); if [ $count -ne 1 ]; then echo Wrong exports in libsimplex.dll.def. Add \"$elem\" to that file; exit 1; fi ; done
|
||||
@@ -28,10 +29,10 @@ for elem in "${exports[@]}"; do count=$(grep -R "\"$elem\"" flake.nix | wc -l);
|
||||
#rm -rf $BUILD_DIR
|
||||
if [[ "$DATABASE_BACKEND" == "postgres" ]]; then
|
||||
echo "Building with postgres backend..."
|
||||
cabal build lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN -optl-Wl,-soname,libsimplex.so -flink-rts -threaded' --constraint 'simplexmq +client_library +client_postgres' --constraint 'simplex-chat +client_library +client_postgres'
|
||||
cabal build --builddir="$DIST_DIR" lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN -optl-Wl,-soname,libsimplex.so -flink-rts -threaded' --constraint 'simplexmq +client_library +client_postgres' --constraint 'simplex-chat +client_library +client_postgres'
|
||||
else
|
||||
echo "Building with sqlite backend..."
|
||||
cabal build lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN -optl-Wl,-soname,libsimplex.so -flink-rts -threaded' --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library'
|
||||
cabal build --builddir="$DIST_DIR" lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN -optl-Wl,-soname,libsimplex.so -flink-rts -threaded' --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library'
|
||||
fi
|
||||
cd $BUILD_DIR/build
|
||||
mv libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so libsimplex.so 2> /dev/null || true
|
||||
|
||||
@@ -1526,10 +1526,6 @@ Query: UPDATE ratchets SET ratchet_state = ? WHERE conn_id = ?
|
||||
Plan:
|
||||
SEARCH ratchets USING PRIMARY KEY (conn_id=?)
|
||||
|
||||
Query: UPDATE rcv_file_chunk_replicas SET delay = ?, retries = retries + 1, updated_at = ? WHERE rcv_file_chunk_replica_id = ?
|
||||
Plan:
|
||||
SEARCH rcv_file_chunk_replicas USING INTEGER PRIMARY KEY (rowid=?)
|
||||
|
||||
Query: UPDATE rcv_file_chunk_replicas SET received = 1, updated_at = ? WHERE rcv_file_chunk_replica_id = ?
|
||||
Plan:
|
||||
SEARCH rcv_file_chunk_replicas USING INTEGER PRIMARY KEY (rowid=?)
|
||||
|
||||
@@ -6306,6 +6306,8 @@ testGroupHistoryGroupLink =
|
||||
bob #> "#team hey!"
|
||||
alice <# "#team bob> hey!"
|
||||
|
||||
threadDelay 1000000
|
||||
|
||||
alice ##> "/create link #team"
|
||||
gLink <- getGroupLink alice "team" GRMember True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user