Introduce support for MSC4429: Profile Updates for Legacy Sync (#19556)

Implements support for [MSC4429: Profile Updates for Legacy
Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429).

Paired with https://github.com/matrix-org/complement/pull/849 and
https://github.com/matrix-org/sytest/tree/anoa/msc4429

Tracking issue for removing unstable identifiers in Synapse:
https://github.com/element-hq/synapse/issues/19891
Further improvements tracked in:
https://github.com/element-hq/synapse/issues/19981

---------

Co-authored-by: Half-Shot <will@half-shot.uk>
Co-authored-by: Jason Robinson <jasonr@element.io>
Co-authored-by: Olivier 'reivilibre' <oliverw@element.io>

scripts-dev/check_schema_delta.py

@@ -14,6 +14,11 @@ import sqlglot.expressions
 
 SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$")
 
+# Keep this in sync with synapse.storage.engines._base. The CI job for this
+# script deliberately installs only its lightweight parsing dependencies, so we
+# avoid importing Synapse here.
+AUTO_INCREMENT_PRIMARY_KEYPLACEHOLDER = "$%AUTO_INCREMENT_PRIMARY_KEY%$"
+
 # The base branch we want to check against. We use the main development branch
 # on the assumption that is what we are developing against.
 DEVELOP_BRANCH = "develop"
@@ -81,7 +86,7 @@ def main(force_colors: bool) -> None:
     bad_delta_files = []
     changed_delta_files = []
     for diff in diffs:
-        if diff.b_path is None:
+        if diff.deleted_file or diff.b_path is None:
             # We don't lint deleted files.
             continue
 
@@ -196,6 +201,10 @@ def check_schema_delta(delta_files: list[str], force_colors: bool) -> bool:
             )
             return True
 
+        delta_contents = _replace_auto_increment_primary_key_placeholder(
+            delta_contents, sql_lang
+        )
+
         statements = sqlglot.parse(delta_contents, read=sql_lang)
 
         for statement in statements:
@@ -244,5 +253,18 @@ def check_schema_delta(delta_files: list[str], force_colors: bool) -> bool:
     return success
 
 
+def _replace_auto_increment_primary_key_placeholder(
+    delta_contents: str, sql_lang: str
+) -> str:
+    """Replace Synapse's auto-increment PK placeholder with parseable SQL."""
+
+    if sql_lang == "sqlite":
+        replacement = "INTEGER PRIMARY KEY AUTOINCREMENT"
+    else:
+        replacement = "BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY"
+
+    return delta_contents.replace(AUTO_INCREMENT_PRIMARY_KEYPLACEHOLDER, replacement)
+
+
 if __name__ == "__main__":
     main()

scripts-dev/complement.sh

@@ -286,6 +286,7 @@ main() {
     ./tests/msc4155
     ./tests/msc4306
     ./tests/msc4222
+    ./tests/msc4429
   )
 
   # Export the list of test packages as a space-separated environment variable, so other

scripts-dev/mypy_synapse_plugin.py

@@ -45,6 +45,7 @@ from mypy.types import (
     AnyType,
     CallableType,
     Instance,
+    LiteralType,
     NoneType,
     Options,
     TupleType,
@@ -813,6 +814,10 @@ def is_cacheable(
     if isinstance(rt, AnyType):
         return True, ("may be mutable" if verbose else None)
 
+    elif isinstance(rt, LiteralType):
+        # Literal[True] etc
+        return True, None
+
     elif isinstance(rt, Instance):
         if (
             rt.type.fullname in IMMUTABLE_VALUE_TYPES