synapse/.github/workflows/latest_deps.yml

# People who are freshly `pip install`ing from PyPI will pull in the latest versions of
# dependencies which match the broad requirements. Since most CI runs are against
# the locked poetry environment, run specifically against the latest dependencies to
# know if there's an upcoming breaking change.
#
# As an overview this workflow:
# - checks out develop,
# - installs from source, pulling in the dependencies like a fresh `pip install` would, and
# - runs mypy and test suites in that checkout.
#
# Based on the twisted trunk CI job.

name: Latest dependencies

on:
  schedule:
    - cron: 0 7 * * *
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  RUST_VERSION: 1.87.0

jobs:
  check_repo:
    # Prevent this workflow from running on any fork of Synapse other than element-hq/synapse, as it is
    # only useful to the Synapse core team.
    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
    # of the workflow will be skipped as well.
    runs-on: ubuntu-latest
    outputs:
      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
    steps:
      - id: check_condition
        run: echo "should_run_workflow=${{ github.repository == 'element-hq/synapse' }}" >> "$GITHUB_OUTPUT"

  mypy:
    needs: check_repo
    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      # The dev dependencies aren't exposed in the wheel metadata (at least with current
      # poetry-core versions), so we install with poetry.
      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
        with:
          python-version: "3.x"
          poetry-version: "2.2.1"
          extras: "all"
      # Dump installed versions for debugging.
      - run: poetry run pip list > before.txt
      # Upgrade all runtime dependencies only. This is intended to mimic a fresh
      # `pip install matrix-synapse[all]` as closely as possible.
      - run: poetry update --without dev
      - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true)
      - name: Remove unhelpful options from mypy config
        run: sed -e '/warn_unused_ignores = True/d' -e '/warn_redundant_casts = True/d' -i mypy.ini
      - run: poetry run mypy
  trial:
    needs: check_repo
    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          - database: "sqlite"
          - database: "postgres"
            postgres-version: "14"

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      - run: sudo apt-get -qq install xmlsec1
      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
        if: ${{ matrix.postgres-version }}
        run: |
          docker run -d -p 5432:5432 \
            -e POSTGRES_PASSWORD=postgres \
            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
            postgres:${{ matrix.postgres-version }}
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.x"
      - run: pip install .[all,test]
      - name: Await PostgreSQL
        if: ${{ matrix.postgres-version }}
        timeout-minutes: 2
        run: until pg_isready -h localhost; do sleep 1; done

      # We nuke the local copy, as we've installed synapse into the virtualenv
      # (rather than use an editable install, which we no longer support). If we
      # don't do this then python can't find the native lib.
      - run: rm -rf synapse/

      - run: python -m twisted.trial --jobs=2 tests
        env:
          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
          SYNAPSE_POSTGRES_HOST: localhost
          SYNAPSE_POSTGRES_USER: postgres
          SYNAPSE_POSTGRES_PASSWORD: postgres
      - name: Dump logs
        # Logs are most useful when the command fails, always include them.
        if: ${{ always() }}
        # Note: Dumps to workflow logs instead of using actions/upload-artifact
        #       This keeps logs colocated with failing jobs
        #       It also ignores find's exit code; this is a best effort affair
        run: >-
          find _trial_temp -name '*.log'
          -exec echo "::group::{}" \;
          -exec cat {} \;
          -exec echo "::endgroup::" \;
          || true

  sytest:
    needs: check_repo
    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    container:
      image: matrixdotorg/sytest-synapse:testing
      volumes:
        - ${{ github.workspace }}:/src
    strategy:
      fail-fast: false
      matrix:
        include:
          - sytest-tag: bookworm

          - sytest-tag: bookworm
            postgres: postgres
            workers: workers
            redis: redis
    env:
      POSTGRES: ${{ matrix.postgres && 1}}
      WORKERS: ${{ matrix.workers && 1 }}
      REDIS: ${{ matrix.redis && 1 }}
      BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Rust
        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master
        with:
          toolchain: ${{ env.RUST_VERSION }}
      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      - name: Ensure sytest runs `pip install`
        # Delete the lockfile so sytest will `pip install` rather than `poetry install`
        run: rm /src/poetry.lock
        working-directory: /src
      - name: Prepare test blacklist
        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
      - name: Run SyTest
        run: /bootstrap.sh synapse
        working-directory: /src
      - name: Summarise results.tap
        if: ${{ always() }}
        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
      - name: Upload SyTest logs
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        if: ${{ always() }}
        with:
          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
          path: |
            /logs/results.tap
            /logs/**/*.log*

  complement:
    needs: check_repo
    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        include:
          - arrangement: monolith
            database: SQLite

          - arrangement: monolith
            database: Postgres

          - arrangement: workers
            database: Postgres

    steps:
      - name: Check out synapse codebase
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          path: synapse

      - name: Prepare Complement's Prerequisites
        run: synapse/.ci/scripts/setup_complement_prerequisites.sh

      - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
        with:
          cache-dependency-path: complement/go.sum
          go-version-file: complement/go.mod

      - name: Run Complement Tests
        id: run_complement_tests
        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
        #     are underpowered and don't like running tons of Synapse instances at once.
        # -json: Output JSON format so that gotestfmt can parse it.
        #
        # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it
        # later on for better formatting with gotestfmt. But we still want the command
        # to output to the terminal as it runs so we can see what's happening in
        # real-time.
        run: |
          set -o pipefail
          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log
        shell: bash
        env:
          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
          TEST_ONLY_IGNORE_POETRY_LOCKFILE: 1

      - name: Formatted Complement test logs
        # Always run this step if we attempted to run the Complement tests.
        if: always() && steps.run_complement_tests.outcome != 'skipped'
        run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,empty-packages"

      - name: Run in-repo Complement Tests
        id: run_in_repo_complement_tests
        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
        #     are underpowered and don't like running tons of Synapse instances at once.
        # -json: Output JSON format so that gotestfmt can parse it.
        #
        # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it
        # later on for better formatting with gotestfmt. But we still want the command
        # to output to the terminal as it runs so we can see what's happening in
        # real-time.
        run: |
          set -o pipefail
          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log
        shell: bash
        env:
          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
          TEST_ONLY_IGNORE_POETRY_LOCKFILE: 1

      - name: Formatted in-repo Complement test logs
        # Always run this step if we attempted to run the Complement tests.
        if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped'
        run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,empty-packages"

  # Open an issue if the build fails, so we know about it.
  # Only do this if we're not experimenting with this action in a PR.
  open-issue:
    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request' && needs.check_repo.outputs.should_run_workflow == 'true'"
    needs:
      # TODO: should mypy be included here? It feels more brittle than the others.
      - mypy
      - trial
      - sytest
      - complement

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          update_existing: true
          filename: .ci/latest_deps_build_failed_issue_template.md