Add CI check that upgrades old schema and checks its identical

Signed-off-by: Olivier 'reivilibre <oliverw@matrix.org>

.ci/scripts/check_schema_upgrade.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+#
+# Checks that you get the same resultant database schema whether you
+# start at the latest Synapse version and set up a fresh database,
+# or whether you set up a fresh database on an old version and then
+# run the upgrade steps.
+#
+# Usage:
+#   PGUSER=postgres PGPASSWORD=postgres .ci/scripts/check_schema_upgrade.sh
+
+set -eu
+
+# The full schema version to upgrade from.
+BASE_SCHEMA="54"
+# A git ref (e.g. tag) to check out the full schema from.
+# Needed because old full schemas have since been removed.
+BASE_REF="v1.62.0"
+
+# Check Postgres env is set
+: "${PGUSER:?must be set}"
+: "${PGPASSWORD:?must be set}"
+
+# Change to repo root
+cd "$(dirname "$0")/../.."
+
+# Names of the database splits we have
+DATABASE_SPLITS=(common main state)
+
+if [ ! -z "$(git status --porcelain)" ]; then
+  echo "The repository has uncommitted changes. Refusing to run."
+  exit 1
+fi
+
+OUTPUT_DIR="$(mktemp -d)"
+
+echo "Checking out full schema $BASE_SCHEMA from $BASE_REF..."
+for db in "${DATABASE_SPLITS[@]}"; do
+  git checkout "$BASE_REF" -- "synapse/storage/schema/$db/full_schemas/$BASE_SCHEMA"
+done
+
+# Synapse will naturally set up a fresh database using the highest version full schema it can find.
+# To avoid that, remove all other full schema versions.
+echo "Removing other full schemas..."
+for db in "${DATABASE_SPLITS[@]}"; do
+  for dir in "synapse/storage/schema/$db/full_schemas"/*; do
+    [ -d "$dir" ] || continue
+    if [ "$(basename "$dir")" != "$BASE_SCHEMA" ]; then
+      echo "  $dir"
+      rm -rf "$dir"
+    fi
+  done
+done
+
+echo "Building the database as a new install..."
+echo "$PGPASSWORD" | scripts-dev/make_full_schema.sh \
+  -c -p "$PGUSER" -o "$OUTPUT_DIR/create"
+
+echo "Building the database as an upgrade from schema $BASE_SCHEMA..."
+echo "$PGPASSWORD" | scripts-dev/make_full_schema.sh \
+  --test-upgrade-from="$BASE_SCHEMA" \
+  -c -p "$PGUSER" -o "$OUTPUT_DIR/upgrade"
+
+# Give a generous 10 lines of context, hopefully enough to show the table name if it's
+# e.g. a column embedded within a CREATE TABLE statement.
+if diff --recursive --unified=10 "$OUTPUT_DIR/create" "$OUTPUT_DIR/upgrade"; then
+  echo "OK: a database upgraded from schema $BASE_SCHEMA matches a new install." >&2
+else
+  echo
+  echo "ERROR: upgrading a database from schema $BASE_SCHEMA does not produce the same" >&2
+  echo "schema as installing one at that version and applying the deltas." >&2
+  echo >&2
+  echo " Left (-): Fresh install     |     Right (+): Simulated upgraded install"
+  exit 1
+fi

.github/workflows/tests.yml

@@ -35,6 +35,7 @@ jobs:
       integration: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.integration }}
       linting: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting }}
       linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting_readme }}
+      schema_upgrade: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.schema_upgrade }}
       golangci: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.golangci }}
     steps:
       - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
@@ -90,6 +91,12 @@ jobs:
               - 'poetry.lock'
               - '.github/workflows/tests.yml'
 
+            schema_upgrade:
+              - 'synapse/storage/**'
+              - 'scripts-dev/make_full_schema.sh'
+              - '.ci/scripts/check_schema_upgrade.sh'
+              - '.github/workflows/tests.yml'
+
             golangci:
               - 'complement/**/*.go'
               - 'complement/.golangci.yml'
@@ -133,6 +140,67 @@ jobs:
       - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20' 'sqlglot>=28.0.0'"
       - run: scripts-dev/check_schema_delta.py --force-colors
 
+  # Checks that a schema upgrade is possible and that it produces the same schema as
+  # a fresh database.
+  check-schema-upgrade:
+    name: Check Schema Upgrade
+    runs-on: ubuntu-latest
+    needs: [changes, linting-done]
+    if: ${{ !cancelled() && !failure() && needs.changes.outputs.schema_upgrade == 'true' }} # Allow previous steps to be skipped, but not fail
+
+    steps:
+      - name: Start postgres with a frozen clock
+        # Use faketime here for schema deltas that are wall-clock sensitive under Postgres
+        run: |
+          mkdir /tmp/postgres-faketime
+          cat > /tmp/postgres-faketime/Dockerfile <<'EOF'
+            FROM postgres:14-alpine
+            RUN apk add --no-cache libfaketime
+
+            # It seems like it could be harmful to fake the monotonic timer
+            # as it might prevent deadlock detection, etc.
+            # But not sure, just doing out of precaution.
+            ENV FAKETIME_DONT_FAKE_MONOTONIC=1
+            ENTRYPOINT ["faketime", "-f", "2001-05-25 12:42:42", "docker-entrypoint.sh"]
+            CMD ["postgres"]
+          EOF
+          docker build -t localhost/postgres-faketime /tmp/postgres-faketime
+
+          docker run -d --name postgres -p 5432:5432 \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+            --health-cmd pg_isready --health-interval 10s \
+            --health-timeout 5s --health-retries 5 \
+            localhost/postgres-faketime
+
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          # We need a deep clone so we can check out the old full schema version
+          fetch-depth: 0
+
+      - name: Install PostgreSQL client and faketime
+        run: sudo apt-get -qq install postgresql-client faketime
+
+      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
+        with:
+          poetry-version: "2.4.1"
+          extras: "postgres"
+          python-version: "3.x"
+
+      - name: Wait for Postgres to be up
+        run: |
+          until [ "$(docker inspect -f '{{.State.Health.Status}}' postgres)" = healthy ]; do sleep 2; done
+
+      - name: Check that an old database still upgrades cleanly
+        env:
+          PGHOST: localhost
+          PGUSER: postgres
+          PGPASSWORD: postgres
+        # Use faketime here for schema deltas that are wall-clock sensitive under SQLite
+        run: |
+          faketime -f "2001-05-25 12:42:42" \
+            poetry run .ci/scripts/check_schema_upgrade.sh
+
   check-lockfile:
     runs-on: ubuntu-latest
     steps:
@@ -770,6 +838,7 @@ jobs:
       - cargo-test
       - cargo-bench
       - linting-done
+      - check-schema-upgrade
     runs-on: ubuntu-latest
     steps:
       - uses: matrix-org/done-action@3409aa904e8a2aaf2220f09bc954d3d0b0a2ee67 # v3
@@ -788,3 +857,4 @@ jobs:
             lint-newsfile
             cargo-test
             cargo-bench
+            check-schema-upgrade