synapse/.github/workflows/complement_tests.yml

# Re-usable workflow (https://docs.github.com/en/actions/how-tos/reuse-automations/reuse-workflows)
name: Reusable Complement testing

on:
  workflow_call:
    inputs:
      use_latest_deps:
        type: boolean
        default: false
      use_twisted_trunk:
        type: boolean
        default: false

# Control the permissions granted to `GITHUB_TOKEN`.
permissions:
  # `actions/checkout` reads the repository (also see
  # https://github.com/actions/checkout/tree/de0fac2e4500dabe0009e67214ff5f5447ce83dd/#recommended-permissions)
  contents: read

env:
  RUST_VERSION: 1.87.0

jobs:
  complement:
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        include:
          - arrangement: monolith
            database: SQLite

          - arrangement: monolith
            database: Postgres

          - arrangement: workers
            database: Postgres

    steps:
      - name: Checkout synapse codebase
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          path: synapse

      # Log Docker system info for debugging (compare with your local environment) and
      # tracking GitHub runner changes over time (can easily compare a run from last
      # week with the current one in question).
      - run: docker system info
        shell: bash

      - name: Install Rust
        uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master
        with:
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2

      # We use `poetry` in `complement.sh`
      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
        with:
          poetry-version: "2.4.1"
          # Matches the `path` where we checkout Synapse above
          working-directory: "synapse"

      - name: Prepare Complement's Prerequisites
        run: synapse/.ci/scripts/setup_complement_prerequisites.sh

      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
        with:
          cache-dependency-path: complement/go.sum
          go-version-file: complement/go.mod

      # This step is specific to the 'Twisted trunk' test run:
      - name: Patch dependencies
        if: ${{ inputs.use_twisted_trunk }}
        run: |
          set -x
          DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx
          pipx install poetry==2.4.1

          poetry remove -n twisted
          poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk
          poetry lock
        working-directory: synapse

      # Run the image sanity check test first as this is the first thing we want to know
      # about (are we actually testing what we expect?) and we don't want to debug
      # downstream failures (wild goose chase).
      - name: Sanity check Complement image
        id: run_sanity_check_complement_image_test
        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
        #     are underpowered and don't like running tons of Synapse instances at once.
        # -json: Output JSON format so that gotestfmt can parse it.
        #
        # tee /tmp/gotest-sanity-check-complement.log: We tee the raw JSON to a
        # file so we can re-process it later for better formatting with gotestfmt
        # (and upload it as an artifact).
        #
        # We deliberately do not dump the raw JSON to the terminal as it is huge
        # and unreadable. Instead we pipe it through `jq` to print a compact,
        # real-time progress view: for each `go test -json` event we print only
        # completed tests (`pass`/`skip`/`fail`, including subtests), dropping the
        # noisy `run`/`output` events and package-level lines. The full detail
        # lives in the raw log we tee to the file above. Non-JSON lines (e.g. the
        # image build output and `set -x` traces that `go test -json` does not
        # wrap) are passed through unchanged rather than parsed, so we still see
        # them and the filter never aborts the pipeline.
        run: |
          set -o pipefail
          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json -run 'TestSynapseVersion/Synapse_version_matches_current_git_checkout' 2>&1 \
            | tee /tmp/gotest-sanity-check-complement.log \
            | jq -rR --unbuffered '. as $raw | (try fromjson catch $raw) | if type == "object" then (select(.Action=="pass" or .Action=="fail" or .Action=="skip") | select(.Test) | "\(.Action|ascii_upcase) \(.Test) \(.Elapsed)s") else $raw end'
        shell: bash
        env:
          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}

      - name: Formatted sanity check Complement test logs
        # Always run this step if we attempted to run the Complement tests.
        if: always() && steps.run_sanity_check_complement_image_test.outcome != 'skipped'
        # We do not hide successful tests in `gotestfmt` here as the list of sanity
        # check tests is so short. Feel free to change this when we get more tests.
        #
        # Note that the `-hide` argument is interpreted by `gotestfmt`. From it,
        # it derives several values under `$settings` and passes them to our
        # custom `.ci/complement_package.gotpl` template to render the output.
        run: cat /tmp/gotest-sanity-check-complement.log | gotestfmt -hide "successful-downloads,empty-packages"

      - name: Run Complement Tests
        id: run_complement_tests
        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
        #     are underpowered and don't like running tons of Synapse instances at once.
        # -json: Output JSON format so that gotestfmt can parse it.
        #
        # tee /tmp/gotest-complement.log: We tee the raw JSON to a file so we can
        # re-process it later for better formatting with gotestfmt (and upload it
        # as an artifact), and pipe it through `jq` for a compact, real-time
        # progress view. See the sanity check step above for details on the jq
        # filter.
        run: |
          set -o pipefail
          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 \
            | tee /tmp/gotest-complement.log \
            | jq -rR --unbuffered '. as $raw | (try fromjson catch $raw) | if type == "object" then (select(.Action=="pass" or .Action=="fail" or .Action=="skip") | select(.Test) | "\(.Action|ascii_upcase) \(.Test) \(.Elapsed)s") else $raw end'
        shell: bash
        env:
          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
          TEST_ONLY_IGNORE_POETRY_LOCKFILE: ${{ inputs.use_latest_deps && 1 || '' }}
          TEST_ONLY_SKIP_DEP_HASH_VERIFICATION: ${{ inputs.use_twisted_trunk  && 1 || '' }}

      - name: Formatted Complement test logs (only failing are shown)
        # Always run this step if we attempted to run the Complement tests.
        if: always() && steps.run_complement_tests.outcome != 'skipped'
        # Hide successful tests in order to reduce the verbosity of the otherwise very large output.
        #
        # Note that the `-hide` argument is interpreted by `gotestfmt`. From it,
        # it derives several values under `$settings` and passes them to our
        # custom `.ci/complement_package.gotpl` template to render the output.
        run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages"

      - name: Run in-repo Complement Tests
        # Always run as long as we attempted to run the original Complement tests.
        # This lets us get a status report on the in-repo tests, even if the
        # standard Complement test suite fails.
        if: always() && steps.run_complement_tests.outcome != 'skipped'
        id: run_in_repo_complement_tests
        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
        #     are underpowered and don't like running tons of Synapse instances at once.
        # -json: Output JSON format so that gotestfmt can parse it.
        #
        # tee /tmp/gotest-in-repo-complement.log: We tee the raw JSON to a file so
        # we can re-process it later for better formatting with gotestfmt (and
        # upload it as an artifact), and pipe it through `jq` for a compact,
        # real-time progress view. See the sanity check step above for details on
        # the jq filter.
        run: |
          set -o pipefail
          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 \
            | tee /tmp/gotest-in-repo-complement.log \
            | jq -rR --unbuffered '. as $raw | (try fromjson catch $raw) | if type == "object" then (select(.Action=="pass" or .Action=="fail" or .Action=="skip") | select(.Test) | "\(.Action|ascii_upcase) \(.Test) \(.Elapsed)s") else $raw end'
        shell: bash
        env:
          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
          TEST_ONLY_IGNORE_POETRY_LOCKFILE: ${{ inputs.use_latest_deps && 1 || '' }}
          TEST_ONLY_SKIP_DEP_HASH_VERIFICATION: ${{ inputs.use_twisted_trunk  && 1 || '' }}

      - name: Formatted in-repo Complement test logs (only failing are shown)
        # Always run this step if we attempted to run the Complement tests.
        if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped'
        # Hide successful tests in order to reduce the verbosity of the otherwise very large output.
        #
        # Note that the `-hide` argument is interpreted by `gotestfmt`. From it,
        # it derives several values under `$settings` and passes them to our
        # custom `.ci/complement_package.gotpl` template to render the output.
        run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages"

      - name: Upload Complement logs
        # Always upload the logs (as artifacts) if we attempted to run the
        # Complement tests, so we can debug failures without scrolling through
        # the (very large) raw output in the build log.
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() }}
        with:
          # The matrix values keep the name unique across the three matrix
          # combinations (upload-artifact rejects duplicate names within a run).
          name: Complement Logs - ${{ job.status }} - (${{ matrix.arrangement }}, ${{ matrix.database }})
          path: |
            /tmp/gotest-sanity-check-complement.log
            /tmp/gotest-complement.log
            /tmp/gotest-in-repo-complement.log