diff --git a/.github/actions/swap/action.yml b/.github/actions/swap/action.yml new file mode 100644 index 000000000..87d670b14 --- /dev/null +++ b/.github/actions/swap/action.yml @@ -0,0 +1,44 @@ +name: 'Set Swap Space' +description: 'Add moar swap' +branding: + icon: 'crop' + color: 'orange' +inputs: + swap-size-gb: + description: 'Swap space to create, in Gigabytes.' + required: false + default: '10' +runs: + using: "composite" + steps: + - name: Swap space report before modification + shell: bash + run: | + echo "Memory and swap:" + free -h + echo + swapon --show + echo + - name: Set Swap + shell: bash + run: | + export SWAP_FILE=$(swapon --show=NAME | tail -n 1) + echo "Swap file: $SWAP_FILE" + if [ -z "$SWAP_FILE" ]; then + SWAP_FILE=/opt/swapfile + else + sudo swapoff $SWAP_FILE + sudo rm $SWAP_FILE + fi + sudo fallocate -l ${{ inputs.swap-size-gb }}G $SWAP_FILE + sudo chmod 600 $SWAP_FILE + sudo mkswap $SWAP_FILE + sudo swapon $SWAP_FILE + - name: Swap space report after modification + shell: bash + run: | + echo "Memory and swap:" + free -h + echo + swapon --show + echo diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ee1d75ec..69f568a78 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,8 +10,56 @@ on: pull_request: jobs: + +# ============================= +# Create release +# ============================= + +# Create release, but only if it's triggered by tag push. +# On pull requests/commits push, this job will always complete. + + maybe-release: + runs-on: ubuntu-latest + steps: + - name: Clone project + if: startsWith(github.ref, 'refs/tags/v') + uses: actions/checkout@v3 + + - name: Build changelog + id: build_changelog + if: startsWith(github.ref, 'refs/tags/v') + uses: simplex-chat/release-changelog-builder-action@v5 + with: + configuration: .github/changelog_conf.json + failOnError: true + ignorePreReleases: true + commitMode: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create release + if: startsWith(github.ref, 'refs/tags/v') + uses: simplex-chat/action-gh-release@v2 + with: + body: | + See full changelog [here](https://github.com/simplex-chat/simplexmq/blob/master/CHANGELOG.md). + + Commits: + ${{ steps.build_changelog.outputs.changelog }} + prerelease: true + files: | + LICENSE + fail_on_unmatched_files: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +# ============================= +# Main build job +# ============================= + build: - name: "Ubuntu: ${{ matrix.os }}, GHC: ${{ matrix.ghc }}" + name: "ubuntu-${{ matrix.os }}, GHC: ${{ matrix.ghc }}" + needs: maybe-release env: apps: "smp-server xftp-server ntf-server xftp" runs-on: ubuntu-${{ matrix.os }} @@ -35,21 +83,32 @@ jobs: - os: 22.04 ghc: "8.10.7" platform_name: 22_04-8.10.7 + should_run: ${{ !(github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }} - os: 22.04 ghc: "9.6.3" platform_name: 22_04-x86-64 + should_run: true - os: 24.04 ghc: "9.6.3" platform_name: 24_04-x86-64 + should_run: true steps: - name: Clone project + if: matrix.should_run == true uses: actions/checkout@v3 - name: Set up Docker Buildx + if: matrix.should_run == true uses: simplex-chat/docker-setup-buildx-action@v3 + - name: Setup swap + if: matrix.ghc == '8.10.7' && matrix.should_run == true + uses: ./.github/actions/swap + with: + swap-size-gb: 20 + - name: Install PostgreSQL 15 client tools - if: matrix.os == '22.04' + if: matrix.os == '22.04' && matrix.should_run == true shell: bash run: | # Import the repository signing key @@ -62,6 +121,7 @@ jobs: sudo apt -y install postgresql-client-15 - name: Build and cache Docker image + if: matrix.should_run == true uses: simplex-chat/docker-build-push-action@v6 with: context: . @@ -77,6 +137,7 @@ jobs: GHC=${{ matrix.ghc }} - name: Cache dependencies + if: matrix.should_run == true uses: actions/cache@v4 with: path: | @@ -85,6 +146,7 @@ jobs: key: ${{ matrix.os }}-${{ hashFiles('cabal.project', 'simplexmq.cabal') }} - name: Start container + if: matrix.should_run == true shell: bash run: | docker run -t -d \ @@ -95,6 +157,7 @@ jobs: build/${{ matrix.platform_name }}:latest - name: Build smp-server (postgresql) and tests + if: matrix.should_run == true shell: docker exec -t builder sh -eu {0} run: | cabal update @@ -108,17 +171,27 @@ jobs: strip /out/smp-server - name: Copy simplexmq-test from container + if: matrix.should_run == true shell: bash run: | docker cp builder:/out/simplexmq-test . - name: Copy smp-server (postgresql) from container and prepare it - if: startsWith(github.ref, 'refs/tags/v') + if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true + id: prepare-postgres shell: bash run: | - docker cp builder:/out/smp-server ./smp-server-postgres-ubuntu-${{ matrix.platform_name }} + name="smp-server-postgres-ubuntu-${{ matrix.platform_name }}" + docker cp builder:/out/smp-server $name + + path="${{ github.workspace }}/$name" + echo "bin=$path" >> $GITHUB_OUTPUT + + hash="SHA2-256($name)= $(openssl sha256 $path | cut -d' ' -f 2)" + printf 'hash=%s' "$hash" >> $GITHUB_OUTPUT - name: Build everything else (standard) + if: matrix.should_run == true shell: docker exec -t builder sh -eu {0} run: | cabal build --jobs=$(nproc) @@ -131,48 +204,64 @@ jobs: done - name: Copy binaries from container and prepare them - if: startsWith(github.ref, 'refs/tags/v') + id: prepare-regular + if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true shell: bash run: | docker cp builder:/out . - for i in ${{ env.apps }}; do mv ./out/$i ./$i-ubuntu-${{ matrix.platform_name }}; done - - name: Build changelog - if: startsWith(github.ref, 'refs/tags/v') - id: build_changelog - uses: simplex-chat/release-changelog-builder-action@v5 - with: - configuration: .github/changelog_conf.json - failOnError: true - ignorePreReleases: true - commitMode: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + printf 'bins< bins.output + printf 'hashes< hashes.output + for i in ${{ env.apps }}; do + mv ./out/$i ./$i-ubuntu-${{ matrix.platform_name }} - - name: Create release - if: startsWith(github.ref, 'refs/tags/v') && matrix.ghc != '8.10.7' + name="$i-ubuntu-${{ matrix.platform_name }}" + + path="${{ github.workspace }}/$name" + hash="SHA2-256($name)= $(openssl sha256 $path | cut -d' ' -f 2)" + + printf '%s\n' "$path" >> bins.output + printf '%s\n\n' "$hash" >> hashes.output + done + printf 'EOF\n' >> bins.output + printf 'EOF\n' >> hashes.output + + cat bins.output >> "$GITHUB_OUTPUT" + cat hashes.output >> "$GITHUB_OUTPUT" + + - name: Upload binaries + if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true uses: simplex-chat/action-gh-release@v2 with: - body: | - See full changelog [here](https://github.com/simplex-chat/simplexmq/blob/master/CHANGELOG.md). - - Commits: - ${{ steps.build_changelog.outputs.changelog }} + append_body: true prerelease: true - files: | - LICENSE - smp-server-ubuntu-${{ matrix.platform_name }} - smp-server-postgres-ubuntu-${{ matrix.platform_name }} - ntf-server-ubuntu-${{ matrix.platform_name }} - xftp-server-ubuntu-${{ matrix.platform_name }} - xftp-ubuntu-${{ matrix.platform_name }} fail_on_unmatched_files: true + body: | + ${{ steps.prepare-regular.outputs.hashes }} + ${{ steps.prepare-postgres.outputs.hash }} + files: | + ${{ steps.prepare-regular.outputs.bins }} + ${{ steps.prepare-postgres.outputs.bin }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Test + if: matrix.should_run == true shell: bash env: PGHOST: localhost run: | - ./simplexmq-test + i=1 + while [ "$i" -le 10 ]; do + if ./simplexmq-test; then + break + else + echo "Attempt $i failed, retrying..." + i=$((i + 1)) + sleep 1 + fi + done + if [ "$i" -gt 10 ]; then + echo "All 10 attempts failed." + exit 1 + fi diff --git a/scripts/reproduce-builds.sh b/scripts/simplexmq-reproduce-builds.sh similarity index 81% rename from scripts/reproduce-builds.sh rename to scripts/simplexmq-reproduce-builds.sh index 70e672247..54fbfad8e 100755 --- a/scripts/reproduce-builds.sh +++ b/scripts/simplexmq-reproduce-builds.sh @@ -6,7 +6,8 @@ TAG="$1" tempdir="$(mktemp -d)" init_dir="$PWD" -repo="https://github.com/simplex-chat/simplexmq" +repo_name="simplexmq" +repo="https://github.com/simplex-chat/${repo_name}" export DOCKER_BUILDKIT=1 cleanup() { @@ -18,10 +19,10 @@ cleanup() { } trap 'cleanup' EXIT INT -mkdir -p "$init_dir/$TAG/from-source" "$init_dir/$TAG/prebuilt" +mkdir -p "$init_dir/$TAG-$repo_name/from-source" "$init_dir/$TAG-$repo_name/prebuilt" git -C "$tempdir" clone "$repo.git" &&\ - cd "$tempdir/simplexmq" &&\ + cd "$tempdir/${repo_name}" &&\ git checkout "$TAG" for os in 22.04 24.04; do @@ -39,7 +40,7 @@ for os in 22.04 24.04; do # Run container in background docker run -t -d \ --name builder \ - -v "$tempdir/simplexmq:/project" \ + -v "$tempdir/${repo_name}:/project" \ local # PostgreSQL build (only smp-server) @@ -51,11 +52,11 @@ for os in 22.04 24.04; do # Copy smp-server postgresql binary and prepare it docker cp \ builder:/out/smp-server \ - "$init_dir/$TAG/from-source/smp-server-postgres-ubuntu-${os_url}-x86-64" + "$init_dir/$TAG-$repo_name/from-source/smp-server-postgres-ubuntu-${os_url}-x86-64" # Download prebuilt postgresql binary curl -L \ - --output-dir "$init_dir/$TAG/prebuilt/" \ + --output-dir "$init_dir/$TAG-$repo_name/prebuilt/" \ -O \ "$repo/releases/download/${TAG}/smp-server-postgres-ubuntu-${os_url}-x86-64" @@ -76,11 +77,11 @@ for os in 22.04 24.04; do # Prepare regular binaries and download the prebuilt ones for app in $apps; do curl -L \ - --output-dir "$init_dir/$TAG/prebuilt/" \ + --output-dir "$init_dir/$TAG-$repo_name/prebuilt/" \ -O \ "$repo/releases/download/${TAG}/${app}-ubuntu-${os_url}-x86-64" - mv "./out-${os}/$app" "$init_dir/$TAG/from-source/${app}-ubuntu-${os_url}-x86-64" + mv "./out-${os}/$app" "$init_dir/$TAG-$repo_name/from-source/${app}-ubuntu-${os_url}-x86-64" done # Important! Remove dist-newstyle for the next interation @@ -105,7 +106,7 @@ cd "$init_dir" # Final stage: compare hashes # Path to binaries -path_bin="$init_dir/$TAG" +path_bin="$init_dir/$TAG-$repo_name" # Assume everything is okay for now bad=0 @@ -122,7 +123,7 @@ for file in "$path_bin"/from-source/*; do # Compare if [ "$compiled" != "$prebuilt" ]; then - # If hashes doesn't match, sed bad... + # If hashes doesn't match, set bad... bad=1 # ... and print affected binary @@ -132,7 +133,7 @@ done # If everything is still okay, compute checksums file if [ "$bad" = 0 ]; then - sha256sum "$path_bin"/from-source/* | sed -e "s|$PWD/||g" -e 's|from-source/||g' > "$path_bin/_sha256sums" + sha256sum "$path_bin"/from-source/* | sed -e "s|$PWD/||g" -e 's|from-source/||g' -e "s|-$repo_name||g" > "$path_bin/_sha256sums" printf 'Checksums computed - %s\n' "$path_bin/_sha256sums" fi