synapse/scripts-dev/make_full_schema.sh

#!/usr/bin/env bash
#
# This script generates SQL files for creating a brand new Synapse DB with the latest
# schema, on both SQLite3 and Postgres.

export PGHOST="localhost"
POSTGRES_MAIN_DB_NAME="synapse_full_schema_main.$$"
POSTGRES_COMMON_DB_NAME="synapse_full_schema_common.$$"
POSTGRES_STATE_DB_NAME="synapse_full_schema_state.$$"

# Python package names that must be importable
REQUIRED_DEPS=("synapse" "sqlite3" "psycopg2")

usage() {
  echo
  echo "Usage: $0 -p <postgres_username> -o <path> [-c] [-n <schema number>] [-h]"
  echo "It is the caller's responsibility to be in the correct Python environment (e.g. using \`poetry run scripts-dev/make_full_schema.sh\`)."
  echo
  echo "-p <postgres_username>"
  echo "  Username to connect to local postgres instance. The password will be requested"
  echo "  during script execution."
  echo "-c"
  echo "  CI mode. Prints every command that the script runs."
  echo "-o <path>"
  echo "  Directory to output full schema files to. You probably want to use"
  echo "  '-o synapse/storage/schema'"
  echo "-n <schema number>"
  echo "  Schema number for the new snapshot. Used to set the location of files within "
  echo "  the output directory, mimicking that of synapse/storage/schemas."
  echo "  NOTE: This does not influence which schema deltas are applied to build the full schema."
  echo "        Schema deltas past this version will still be applied; if you want to build a"
  echo "        full schema at a particular version, later deltas first need to be"
  echo "        temporarily deleted (as well as homeserver code temporarily tweaked"
  echo "        not to rely on any of those changes when applying background updates)"
  echo "  Defaults to 9999."
  echo "-h"
  echo "  Display this help text."
  echo ""
  echo ""
  echo "You probably want to invoke this with something like"
  echo "  docker run --rm -e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres -e POSTGRES_DB=synapse -p 5432:5432 postgres:14-alpine"
  echo "  echo postgres | poetry run scripts-dev/make_full_schema.sh -p postgres -n MY_SCHEMA_NUMBER -o synapse/storage/schema"
  echo ""
  echo "  NB: make sure to run this against the *oldest* supported version of postgres,"
  echo "  or else pg_dump might output non-backwards-compatible syntax."
}

SCHEMA_NUMBER="9999"
while getopts "p:co:hn:" opt; do
  case $opt in
    p)
      export PGUSER=$OPTARG
      ;;
    c)
      # Print all commands that are being executed
      set -x
      ;;
    o)
      command -v realpath > /dev/null || (echo "The -o flag requires the 'realpath' binary to be installed" && exit 1)
      OUTPUT_DIR="$(realpath "$OPTARG")"
      ;;
    h)
      usage
      exit
      ;;
    n)
      SCHEMA_NUMBER="$OPTARG"
      ;;
    \?)
      echo "ERROR: Invalid option: -$OPTARG" >&2
      usage
      exit
      ;;
  esac
done

# Check that required dependencies are installed
unsatisfied_requirements=()
for dep in "${REQUIRED_DEPS[@]}"; do
  python -c "import $dep" &> /dev/null || unsatisfied_requirements+=("$dep")
done
if [ ${#unsatisfied_requirements} -ne 0 ]; then
  echo 'Please `poetry install --extras postgres` first as the following Python modules are not importable: '"${unsatisfied_requirements[*]}"
  exit 1
fi

if [ -z "$PGUSER" ]; then
  echo "No postgres username supplied"
  usage
  exit 1
fi

if [ -z "$OUTPUT_DIR" ]; then
  echo "No output directory supplied"
  usage
  exit 1
fi

# Create the output directory if it doesn't exist
mkdir -p "$OUTPUT_DIR"

read -rsp "Postgres password for '$PGUSER': " PGPASSWORD
echo ""
export PGPASSWORD

# Exit immediately if a command fails
set -e

# cd to root of the synapse directory
cd "$(dirname "$0")/.."

# Create temporary SQLite and Postgres homeserver db configs and key file
TMPDIR=$(mktemp -d)
KEY_FILE=$TMPDIR/test.signing.key # default Synapse signing key path
SQLITE_CONFIG=$TMPDIR/sqlite.conf
SQLITE_MAIN_DB=$TMPDIR/main.db
SQLITE_STATE_DB=$TMPDIR/state.db
SQLITE_COMMON_DB=$TMPDIR/common.db
POSTGRES_CONFIG=$TMPDIR/postgres.conf

# Ensure these files are delete on script exit
cleanup() {
  echo "Cleaning up temporary sqlite database and config files..."
  rm -r "$TMPDIR"
  echo "Cleaning up temporary Postgres database..."
  dropdb --if-exists "$POSTGRES_COMMON_DB_NAME"
  dropdb --if-exists "$POSTGRES_MAIN_DB_NAME"
  dropdb --if-exists "$POSTGRES_STATE_DB_NAME"
}
trap 'cleanup' EXIT

cat > "$SQLITE_CONFIG" <<EOF
server_name: "test"

signing_key_path: "$KEY_FILE"
macaroon_secret_key: "abcde"

report_stats: false

databases:
  common:
    name: "sqlite3"
    data_stores: []
    args:
      database: "$SQLITE_COMMON_DB"
  main:
    name: "sqlite3"
    data_stores: ["main"]
    args:
      database: "$SQLITE_MAIN_DB"
  state:
    name: "sqlite3"
    data_stores: ["state"]
    args:
      database: "$SQLITE_STATE_DB"

# Suppress the key server warning.
trusted_key_servers: []
EOF

cat > "$POSTGRES_CONFIG" <<EOF
server_name: "test"

signing_key_path: "$KEY_FILE"
macaroon_secret_key: "abcde"

report_stats: false

databases:
  common:
    name: "psycopg2"
    data_stores: []
    args:
      user: "$PGUSER"
      host: "$PGHOST"
      password: "$PGPASSWORD"
      database: "$POSTGRES_COMMON_DB_NAME"
  main:
    name: "psycopg2"
    data_stores: ["main"]
    args:
      user: "$PGUSER"
      host: "$PGHOST"
      password: "$PGPASSWORD"
      database: "$POSTGRES_MAIN_DB_NAME"
  state:
    name: "psycopg2"
    data_stores: ["state"]
    args:
      user: "$PGUSER"
      host: "$PGHOST"
      password: "$PGPASSWORD"
      database: "$POSTGRES_STATE_DB_NAME"


# Suppress the key server warning.
trusted_key_servers: []
EOF

# Generate the server's signing key.
echo "Generating SQLite3 db schema..."
python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG"

# Make sure the SQLite3 database is using the latest schema and has no pending background update.
echo "Running db background jobs..."
poetry run python synapse/_scripts/update_synapse_database.py --database-config "$SQLITE_CONFIG" --run-background-updates

# Create the PostgreSQL database.
echo "Creating postgres databases..."
createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_COMMON_DB_NAME"
createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_MAIN_DB_NAME"
createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_STATE_DB_NAME"

echo "Running db background jobs..."
poetry run python synapse/_scripts/update_synapse_database.py --database-config "$POSTGRES_CONFIG" --run-background-updates


echo "Dropping unwanted db tables..."

# Some common tables are created and updated by Synapse itself and do not belong in the
# schema.
DROP_APP_MANAGED_TABLES="
DROP TABLE schema_version;
DROP TABLE schema_compat_version;
DROP TABLE applied_schema_deltas;
DROP TABLE applied_module_schemas;
"
# Other common tables are not created by Synapse and do belong in the schema.
# TODO: we could derive DROP_COMMON_TABLES from the dump of the common-only DB. But
#       since there's only one table there, I haven't bothered to do so.
DROP_COMMON_TABLES="$DROP_APP_MANAGED_TABLES
DROP TABLE background_updates;
"

sqlite3 "$SQLITE_COMMON_DB" <<< "$DROP_APP_MANAGED_TABLES"
sqlite3 "$SQLITE_MAIN_DB" <<< "$DROP_COMMON_TABLES"
sqlite3 "$SQLITE_STATE_DB" <<< "$DROP_COMMON_TABLES"
psql "$POSTGRES_COMMON_DB_NAME" -w <<< "$DROP_APP_MANAGED_TABLES"
psql "$POSTGRES_MAIN_DB_NAME" -w <<< "$DROP_COMMON_TABLES"
psql "$POSTGRES_STATE_DB_NAME" -w <<< "$DROP_COMMON_TABLES"

# For Reasons(TM), SQLite's `.schema` also dumps out "shadow tables", the implementation
# details behind full text search tables.
# Previously we omitted these from the dumps by dropping them beforehand,
# but nowadays it seems to be forbidden to drop those.
# The emitted dump adds `IF NOT EXISTS` text for them, so it's harmless.

echo "Dumping SQLite3 schema..."

mkdir -p "$OUTPUT_DIR/"{common,main,state}"/full_schemas/$SCHEMA_NUMBER"
# `--nosys` prevents emitting (some) SQLite internal tables like `sqlite_sequence` that
# aren't managed by the Synapse application and don't need to be part of our full schema.
#
# (SQLite creates `sqlite_sequence` automatically when the database contains at least one
# table with an AUTOINCREMENT column.)
sqlite3 "$SQLITE_COMMON_DB" ".schema --nosys"            > "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite"
sqlite3 "$SQLITE_COMMON_DB" ".dump --data-only --nosys" >> "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite"
sqlite3 "$SQLITE_MAIN_DB"   ".schema --nosys"            > "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite"
sqlite3 "$SQLITE_MAIN_DB"   ".dump --data-only --nosys" >> "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite"
sqlite3 "$SQLITE_STATE_DB"  ".schema --nosys"            > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite"
sqlite3 "$SQLITE_STATE_DB"  ".dump --data-only --nosys" >> "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite"

cleanup_pg_schema() {
  # Cleanup as follows:
  # - Remove empty lines. pg_dump likes to output a lot of these.
  # - Remove the `\restrict` and `\unrestrict` psql meta-commands.
  #   We don't run the pg_dump output through psql so no meta-commands are
  #   supported and so the security feature (which doesn't apply anyway
  #   as we trust the source database) is not relevant.
  # - Remove comment-only lines. pg_dump also likes to output a lot of these to visually
  #   separate tables etc.
  # - Remove "public." prefix --- the schema name.
  # - Remove "SET" commands. Last time I ran this, the output commands were
  #     SET statement_timeout = 0;
  #     SET lock_timeout = 0;
  #     SET idle_in_transaction_session_timeout = 0;
  #     SET client_encoding = 'UTF8';
  #     SET standard_conforming_strings = on;
  #     SET check_function_bodies = false;
  #     SET xmloption = content;
  #     SET client_min_messages = warning;
  #     SET row_security = off;
  #     SET default_table_access_method = heap;
  # - Very carefully remove specific SELECT statements. We CANNOT blanket remove all
  #   SELECT statements because some of those have side-effects which we do want in the
  #   schema. Last time I ran this, the only SELECTS were
  #     SELECT pg_catalog.set_config('search_path', '', false);
  #   and
  #     SELECT pg_catalog.setval(text, bigint, bool);
  #   We do want to remove the former, but the latter is important. If the last argument
  #   is `true` or omitted, this marks the given integer as having been consumed and
  #   will NOT appear as the nextval.
   sed -e '/^$/d' \
   -e '/^\\restrict REMOVEME$/d' \
   -e '/^\\unrestrict REMOVEME$/d' \
   -e '/^--/d' \
   -e 's/public\.//g' \
   -e '/^SET /d' \
   -e '/^SELECT pg_catalog.set_config/d'
}

echo "Dumping Postgres schema..."

# --restrict-key: set the \restrict key to a static value (normally a random string) for easy find/replacement in the code above.
pg_dump --restrict-key=REMOVEME --format=plain --schema-only         --no-tablespaces --no-acl --no-owner "$POSTGRES_COMMON_DB_NAME" | cleanup_pg_schema  > "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"
pg_dump --restrict-key=REMOVEME --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_COMMON_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"
pg_dump --restrict-key=REMOVEME --format=plain --schema-only         --no-tablespaces --no-acl --no-owner "$POSTGRES_MAIN_DB_NAME"   | cleanup_pg_schema  > "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"
pg_dump --restrict-key=REMOVEME --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_MAIN_DB_NAME"   | cleanup_pg_schema >> "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"
pg_dump --restrict-key=REMOVEME --format=plain --schema-only         --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME"  | cleanup_pg_schema  > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"
pg_dump --restrict-key=REMOVEME --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME"  | cleanup_pg_schema >> "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"

if [[ "$OUTPUT_DIR" == *synapse/storage/schema ]]; then
  echo "Updating contrib/datagrip symlinks..."
  ln -sf "../../synapse/storage/schema/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" "contrib/datagrip/common.sql"
  ln -sf "../../synapse/storage/schema/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"   "contrib/datagrip/main.sql"
  ln -sf "../../synapse/storage/schema/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"  "contrib/datagrip/state.sql"
else
  echo "Not updating contrib/datagrip symlinks (unknown output directory)"
fi
echo "Done! Files dumped to: $OUTPUT_DIR"