synapse/.github/workflows/tests.yml

name: Tests

on:
  push:
    branches: ["develop", "release-*"]
  pull_request:
  merge_group:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  RUST_VERSION: 1.87.0

jobs:
  # Job to detect what has changed so we don't run e.g. Rust checks on PRs that
  # don't modify Rust code.
  changes:
    runs-on: ubuntu-latest
    outputs:
      rust: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.rust }}
      trial: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.trial }}
      integration: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.integration }}
      linting: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting }}
      linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting_readme }}
    steps:
      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        id: filter
        # We only check on PRs
        if: startsWith(github.ref, 'refs/pull/')
        with:
          filters: |
            rust:
              - 'rust/**'
              - 'Cargo.toml'
              - 'Cargo.lock'
              - '.rustfmt.toml'
              - '.github/workflows/tests.yml'

            trial:
              - 'synapse/**'
              - 'tests/**'
              - 'rust/**'
              - '.ci/scripts/calculate_jobs.py'
              - 'Cargo.toml'
              - 'Cargo.lock'
              - 'pyproject.toml'
              - 'poetry.lock'
              - '.github/workflows/tests.yml'

            integration:
              - 'synapse/**'
              - 'rust/**'
              - 'docker/**'
              - 'Cargo.toml'
              - 'Cargo.lock'
              - 'pyproject.toml'
              - 'poetry.lock'
              - 'docker/**'
              - '.ci/**'
              - 'scripts-dev/complement.sh'
              - '.github/workflows/tests.yml'

            linting:
              - 'synapse/**'
              - 'docker/**'
              - 'tests/**'
              - 'scripts-dev/**'
              - 'contrib/**'
              - 'synmark/**'
              - 'stubs/**'
              - '.ci/**'
              - 'mypy.ini'
              - 'pyproject.toml'
              - 'poetry.lock'
              - '.github/workflows/tests.yml'

            linting_readme:
              - 'README.rst'

  check-sampleconfig:
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ needs.changes.outputs.linting == 'true' }}

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
        with:
          python-version: "3.x"
          poetry-version: "2.2.1"
          extras: "all"
      - run: poetry run scripts-dev/generate_sample_config.sh --check
      - run: poetry run scripts-dev/config-lint.sh

  check-schema-delta:
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ needs.changes.outputs.linting == 'true' }}

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.x"
      - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20' 'sqlglot>=28.0.0'"
      - run: scripts-dev/check_schema_delta.py --force-colors

  check-lockfile:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.x"
      - run: .ci/scripts/check_lockfile.py

  lint:
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ needs.changes.outputs.linting == 'true' }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Poetry
        uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
        with:
          poetry-version: "2.2.1"
          install-project: "false"

      - name: Run ruff check
        run: poetry run ruff check --output-format=github .

      - name: Run ruff format
        run: poetry run ruff format --check .

  lint-mypy:
    runs-on: ubuntu-latest
    name: Typechecking
    needs: changes
    if: ${{ needs.changes.outputs.linting == 'true' }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      - name: Setup Poetry
        uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
        with:
          # We want to make use of type hints in optional dependencies too.
          extras: all
          # We have seen odd mypy failures that were resolved when we started
          # installing the project again:
          # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775
          # To make CI green, err towards caution and install the project.
          install-project: "true"
          poetry-version: "2.2.1"

      # Cribbed from
      # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
      - name: Restore/persist mypy's cache
        uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
        with:
          path: |
            .mypy_cache
          key: mypy-cache-${{ github.context.sha }}
          restore-keys: mypy-cache-

      - name: Run mypy
        run: poetry run mypy

  lint-crlf:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Check line endings
        run: scripts-dev/check_line_terminators.sh

  lint-newsfile:
    # Only run on pull_request events, targeting develop/release branches, and skip when the PR author is dependabot[bot].
    if: ${{ github.event_name == 'pull_request' && (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.event.pull_request.user.login != 'dependabot[bot]' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.x"
      - run: "pip install 'towncrier>=18.6.0rc1'"
      - run: scripts-dev/check-newsfragment.sh
        env:
          PULL_REQUEST_NUMBER: ${{ github.event.number }}

  lint-clippy:
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ needs.changes.outputs.rust == 'true' }}

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          components: clippy
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      - run: cargo clippy -- -D warnings

  # We also lint against a nightly rustc so that we can lint the benchmark
  # suite, which requires a nightly compiler.
  lint-clippy-nightly:
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ needs.changes.outputs.rust == 'true' }}

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: nightly-2026-02-01
          components: clippy
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      - run: cargo clippy --all-features -- -D warnings

  lint-rust:
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ needs.changes.outputs.rust == 'true' }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      - name: Setup Poetry
        uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
        with:
          # Install like a normal project from source with all optional dependencies
          extras: all
          install-project: "true"
          poetry-version: "2.2.1"

      - name: Ensure `Cargo.lock` is up to date (no stray changes after install)
        # The `::error::` syntax is using GitHub Actions' error annotations, see
        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions
        run: |
          if git diff --quiet Cargo.lock; then
            echo "Cargo.lock is up to date"
          else
            echo "::error::Cargo.lock has uncommitted changes after install. Please run 'poetry install --extras all' and commit the Cargo.lock changes."
            git diff --exit-code Cargo.lock
            exit 1
          fi

  # This job is split from `lint-rust` because it requires a nightly Rust toolchain
  # for some of the unstable options we use in `.rustfmt.toml`.
  lint-rustfmt:
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ needs.changes.outputs.rust == 'true' }}

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          # We use nightly so that we can use some unstable options that we use in
          # `.rustfmt.toml`.
          toolchain: nightly-2025-04-23
          components: rustfmt
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      - run: cargo fmt --check

  # This is to detect issues with the rst file, which can otherwise cause issues
  # when uploading packages to PyPi.
  lint-readme:
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ needs.changes.outputs.linting_readme == 'true' }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.x"
      - run: "pip install rstcheck"
      - run: "rstcheck --report-level=WARNING README.rst"

  # Dummy step to gate other tests on without repeating the whole list
  linting-done:
    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
    needs:
      - lint
      - lint-mypy
      - lint-crlf
      - lint-newsfile
      - check-sampleconfig
      - check-schema-delta
      - check-lockfile
      - lint-clippy
      - lint-clippy-nightly
      - lint-rust
      - lint-rustfmt
      - lint-readme
    runs-on: ubuntu-latest
    steps:
      - uses: matrix-org/done-action@3409aa904e8a2aaf2220f09bc954d3d0b0a2ee67 # v3
        with:
          needs: ${{ toJSON(needs) }}

          # Various bits are skipped if there was no applicable changes.
          skippable: |
            check-sampleconfig
            check-schema-delta
            lint
            lint-mypy
            lint-newsfile
            lint-clippy
            lint-clippy-nightly
            lint-rust
            lint-rustfmt
            lint-readme

  calculate-test-jobs:
    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
    needs: linting-done
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.x"
      - id: get-matrix
        run: .ci/scripts/calculate_jobs.py
    outputs:
      trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }}
      sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }}

  trial:
    if: ${{ !cancelled() && !failure() && needs.changes.outputs.trial == 'true' }} # Allow previous steps to be skipped, but not fail
    needs:
      - calculate-test-jobs
      - changes
    runs-on: ubuntu-latest
    strategy:
      matrix:
        job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }}

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - run: sudo apt-get -qq install xmlsec1
      - name: Set up PostgreSQL ${{ matrix.job.postgres-version }}
        if: ${{ matrix.job.postgres-version }}
        # 1. Mount postgres data files onto a tmpfs in-memory filesystem to reduce overhead of docker's overlayfs layer.
        # 2. Expose the unix socket for postgres. This removes latency of using docker-proxy for connections.
        run: |
          docker run -d -p 5432:5432 \
            --tmpfs /var/lib/postgres:rw,size=6144m \
            --mount 'type=bind,src=/var/run/postgresql,dst=/var/run/postgresql' \
            -e POSTGRES_PASSWORD=postgres \
            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
            postgres:${{ matrix.job.postgres-version }}

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
        with:
          python-version: ${{ matrix.job.python-version }}
          poetry-version: "2.2.1"
          extras: ${{ matrix.job.extras }}
      - name: Await PostgreSQL
        if: ${{ matrix.job.postgres-version }}
        timeout-minutes: 2
        run: until pg_isready -h localhost; do sleep 1; done
      - run: poetry run trial --jobs=6 tests
        env:
          SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }}
          SYNAPSE_POSTGRES_HOST: /var/run/postgresql
          SYNAPSE_POSTGRES_USER: postgres
          SYNAPSE_POSTGRES_PASSWORD: postgres
      - name: Dump logs
        # Logs are most useful when the command fails, always include them.
        if: ${{ always() }}
        # Note: Dumps to workflow logs instead of using actions/upload-artifact
        #       This keeps logs colocated with failing jobs
        #       It also ignores find's exit code; this is a best effort affair
        run: >-
          find _trial_temp -name '*.log'
          -exec echo "::group::{}" \;
          -exec cat {} \;
          -exec echo "::endgroup::" \;
          || true

  trial-olddeps:
    # Note: sqlite only; no postgres
    if: ${{ !cancelled() && !failure() && needs.changes.outputs.trial == 'true' }} # Allow previous steps to be skipped, but not fail
    needs:
      - linting-done
      - changes
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      # There aren't wheels for some of the older deps, so we need to install
      # their build dependencies
      - run: |
          sudo apt-get -qq update
          sudo apt-get -qq install build-essential libffi-dev python3-dev \
            libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.10"

      - name: Prepare old deps
        # Note: we install using `uv` here, not poetry or pip to allow us to test with the
        # minimum version of all dependencies, both those explicitly specified and those
        # implicitly brought in by the explicit dependencies.
        run: |
          pip install uv
          uv pip install --system --resolution=lowest .[all,test]

      # We nuke the local copy, as we've installed synapse into the virtualenv
      # (rather than use an editable install, which we no longer support). If we
      # don't do this then python can't find the native lib.
      - run: rm -rf synapse/

      # Sanity check we can import/run Synapse
      - run: python -m synapse.app.homeserver --help

      - run: python -m twisted.trial -j6 tests
      - name: Dump logs
        # Logs are most useful when the command fails, always include them.
        if: ${{ always() }}
        # Note: Dumps to workflow logs instead of using actions/upload-artifact
        #       This keeps logs colocated with failing jobs
        #       It also ignores find's exit code; this is a best effort affair
        run: >-
          find _trial_temp -name '*.log'
          -exec echo "::group::{}" \;
          -exec cat {} \;
          -exec echo "::endgroup::" \;
          || true

  trial-pypy:
    # Very slow; only run if the branch name includes 'pypy'
    # Note: sqlite only; no postgres. Completely untested since poetry move.
    if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() && needs.changes.outputs.trial == 'true' }}
    needs:
      - linting-done
      - changes
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["pypy-3.10"]
        extras: ["all"]

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      # Install libs necessary for PyPy to build binary wheels for dependencies
      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
        with:
          python-version: ${{ matrix.python-version }}
          poetry-version: "2.2.1"
          extras: ${{ matrix.extras }}
      - run: poetry run trial --jobs=2 tests
      - name: Dump logs
        # Logs are most useful when the command fails, always include them.
        if: ${{ always() }}
        # Note: Dumps to workflow logs instead of using actions/upload-artifact
        #       This keeps logs colocated with failing jobs
        #       It also ignores find's exit code; this is a best effort affair
        run: >-
          find _trial_temp -name '*.log'
          -exec echo "::group::{}" \;
          -exec cat {} \;
          -exec echo "::endgroup::" \;
          || true

  sytest:
    if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true' }}
    needs:
      - calculate-test-jobs
      - changes
    runs-on: ubuntu-latest
    container:
      image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }}
      volumes:
        - ${{ github.workspace }}:/src
      env:
        # If this is a pull request to a release branch, use that branch as default branch for sytest, else use develop
        # This works because the release script always create a branch on the sytest repo with the same name as the release branch
        SYTEST_DEFAULT_BRANCH: ${{ startsWith(github.base_ref, 'release-') && github.base_ref || 'develop' }}
        SYTEST_BRANCH: ${{ github.head_ref }}
        POSTGRES: ${{ matrix.job.postgres && 1}}
        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }}
        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }}
        WORKERS: ${{ matrix.job.workers && 1 }}
        BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
        TOP: ${{ github.workspace }}

    strategy:
      fail-fast: false
      matrix:
        job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }}

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Prepare test blacklist
        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      - name: Run SyTest
        run: /bootstrap.sh synapse
        working-directory: /src
      - name: Summarise results.tap
        if: ${{ always() }}
        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
      - name: Upload SyTest logs
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        if: ${{ always() }}
        with:
          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }})
          path: |
            /logs/results.tap
            /logs/**/*.log*

  export-data:
    if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true'}} # Allow previous steps to be skipped, but not fail
    needs: [linting-done, portdb, changes]
    runs-on: ubuntu-latest
    env:
      TOP: ${{ github.workspace }}

    services:
      postgres:
        image: postgres
        ports:
          - 5432:5432
        env:
          POSTGRES_PASSWORD: "postgres"
          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - run: sudo apt-get -qq install xmlsec1 postgresql-client
      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
        with:
          poetry-version: "2.2.1"
          extras: "postgres"
      - run: .ci/scripts/test_export_data_command.sh
        env:
          PGHOST: localhost
          PGUSER: postgres
          PGPASSWORD: postgres
          PGDATABASE: postgres

  portdb:
    if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true'}} # Allow previous steps to be skipped, but not fail
    needs:
      - linting-done
      - changes
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          - python-version: "3.10"
            postgres-version: "14"

          - python-version: "3.14"
            postgres-version: "17"

    services:
      postgres:
        image: postgres:${{ matrix.postgres-version }}
        ports:
          - 5432:5432
        env:
          POSTGRES_PASSWORD: "postgres"
          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Add PostgreSQL apt repository
        # We need a version of pg_dump that can handle the version of
        # PostgreSQL being tested against. The Ubuntu package repository lags
        # behind new releases, so we have to use the PostreSQL apt repository.
        # Steps taken from https://www.postgresql.org/download/linux/ubuntu/
        run: |
          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
          sudo apt-get update
      - run: sudo apt-get -qq install xmlsec1 postgresql-client
      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
        with:
          python-version: ${{ matrix.python-version }}
          poetry-version: "2.2.1"
          extras: "postgres"
      - run: .ci/scripts/test_synapse_port_db.sh
        id: run_tester_script
        env:
          PGHOST: localhost
          PGUSER: postgres
          PGPASSWORD: postgres
          PGDATABASE: postgres
      - name: "Upload schema differences"
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }}
        with:
          name: Schema dumps
          path: |
            unported.sql
            ported.sql
            schema_diff

  complement:
    if: "${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true' }}"
    needs:
      - linting-done
      - changes
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        include:
          - arrangement: monolith
            database: SQLite

          - arrangement: monolith
            database: Postgres

          - arrangement: workers
            database: Postgres

    steps:
      - name: Checkout synapse codebase
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          path: synapse

      # Log Docker system info for debugging (compare with your local environment) and
      # tracking GitHub runner changes over time (can easily compare a run from last
      # week with the current one in question).
      - run: docker system info
        shell: bash

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      # We use `poetry` in `complement.sh`
      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
        with:
          poetry-version: "2.2.1"
          # Matches the `path` where we checkout Synapse above
          working-directory: "synapse"

      - name: Prepare Complement's Prerequisites
        run: synapse/.ci/scripts/setup_complement_prerequisites.sh

      - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
        with:
          cache-dependency-path: complement/go.sum
          go-version-file: complement/go.mod

      # Run the image sanity check test first as this is the first thing we want to know
      # about (are we actually testing what we expect?) and we don't want to debug
      # downstream failures (wild goose chase).
      - name: Sanity check Complement image
        id: run_sanity_check_complement_image_test
        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
        #     are underpowered and don't like running tons of Synapse instances at once.
        # -json: Output JSON format so that gotestfmt can parse it.
        #
        # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it
        # later on for better formatting with gotestfmt. But we still want the command
        # to output to the terminal as it runs so we can see what's happening in
        # real-time.
        run: |
          set -o pipefail
          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json -run 'TestSynapseVersion/Synapse_version_matches_current_git_checkout' 2>&1 | tee /tmp/gotest-sanity-check-complement.log
        shell: bash
        env:
          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}

      - name: Formatted sanity check Complement test logs
        # Always run this step if we attempted to run the Complement tests.
        if: always() && steps.run_sanity_check_complement_image_test.outcome != 'skipped'
        # We do not hide successful tests in `gotestfmt` here as the list of sanity
        # check tests is so short. Feel free to change this when we get more tests.
        #
        # Note that the `-hide` argument is interpreted by `gotestfmt`. From it,
        # it derives several values under `$settings` and passes them to our
        # custom `.ci/complement_package.gotpl` template to render the output.
        run: cat /tmp/gotest-sanity-check-complement.log | gotestfmt -hide "successful-downloads,empty-packages"

      - name: Run Complement Tests
        id: run_complement_tests
        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
        #     are underpowered and don't like running tons of Synapse instances at once.
        # -json: Output JSON format so that gotestfmt can parse it.
        #
        # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it
        # later on for better formatting with gotestfmt. But we still want the command
        # to output to the terminal as it runs so we can see what's happening in
        # real-time.
        run: |
          set -o pipefail
          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log
        shell: bash
        env:
          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}

      - name: Formatted Complement test logs (only failing are shown)
        # Always run this step if we attempted to run the Complement tests.
        if: always() && steps.run_complement_tests.outcome != 'skipped'
        # Hide successful tests in order to reduce the verbosity of the otherwise very large output.
        #
        # Note that the `-hide` argument is interpreted by `gotestfmt`. From it,
        # it derives several values under `$settings` and passes them to our
        # custom `.ci/complement_package.gotpl` template to render the output.
        run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages"

      - name: Run in-repo Complement Tests
        id: run_in_repo_complement_tests
        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
        #     are underpowered and don't like running tons of Synapse instances at once.
        # -json: Output JSON format so that gotestfmt can parse it.
        #
        # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it
        # later on for better formatting with gotestfmt. But we still want the command
        # to output to the terminal as it runs so we can see what's happening in
        # real-time.
        run: |
          set -o pipefail
          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log
        shell: bash
        env:
          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}

      - name: Formatted in-repo Complement test logs (only failing are shown)
        # Always run this step if we attempted to run the Complement tests.
        if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped'
        # Hide successful tests in order to reduce the verbosity of the otherwise very large output.
        #
        # Note that the `-hide` argument is interpreted by `gotestfmt`. From it,
        # it derives several values under `$settings` and passes them to our
        # custom `.ci/complement_package.gotpl` template to render the output.
        run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages"

  cargo-test:
    if: ${{ needs.changes.outputs.rust == 'true' }}
    runs-on: ubuntu-latest
    needs:
      - linting-done
      - changes

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      - run: cargo test

  # We want to ensure that the cargo benchmarks still compile, which requires a
  # nightly compiler.
  cargo-bench:
    if: ${{ needs.changes.outputs.rust == 'true' }}
    runs-on: ubuntu-latest
    needs:
      - linting-done
      - changes

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: nightly-2022-12-01
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      - run: cargo bench --no-run

  # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
  tests-done:
    if: ${{ always() }}
    needs:
      - trial
      - trial-olddeps
      - sytest
      - export-data
      - portdb
      - complement
      - cargo-test
      - cargo-bench
      - linting-done
    runs-on: ubuntu-latest
    steps:
      - uses: matrix-org/done-action@3409aa904e8a2aaf2220f09bc954d3d0b0a2ee67 # v3
        with:
          needs: ${{ toJSON(needs) }}

          # Various bits are skipped if there was no applicable changes.
          # The newsfile lint may be skipped on non PR builds.
          skippable: |
            trial
            trial-olddeps
            sytest
            portdb
            export-data
            complement
            lint-newsfile
            cargo-test
            cargo-bench